[clang] [llvm] [X86] Add ACE v1 (AI Compute Extensions) support (PR #208408)
via cfe-commits
cfe-commits at lists.llvm.org
Sun Nov 15 19:53:31 PST 2026
https://github.com/ganeshgit updated https://github.com/llvm/llvm-project/pull/208408
>From f537f625b75eb32b5456c16b6de5cc244892f186 Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Wed, 8 Jul 2026 00:41:01 +0530
Subject: [PATCH 1/6] [X86] Add ACE v1 (AI Compute Extensions) support
ACE v1 introduces new tile-based matrix operations with fixed 16x64
tile dimensions (Palette 2), unlike AMX's configurable dimensions.
Key features:
- New __acetile type with fixed 16x64 dimensions
- __tile_ace_* intrinsics for ACE tile operations
- Outer product instructions: TOP4BUUD, TOP4BUSD, TOP4BSSD, TOP4BSUD, TOP2BF16PS
- Mixed precision FP8 instructions: TOP4MX variants
- Tile movement: TILEMOVROW, TILEMOVCOL for ZMM<->tile transfers
- BSR (Block Scale Register) operations for scaling factors
- ACE tile spill/reload using TILEMOVROW + VMOVUPS row-by-row
(ACE doesn't have TILELOADD/TILESTORED instructions)
Backend changes:
- X86LowerTileCopy handles ACE tile copies
- X86PreTileConfig/X86FastPreTileConfig for ACE register allocation
- Palette 2 tile configuration support
Reused from AMX:
- The x86_amx type
- AMX-AVX512f definitions and TILE instruction definitions
- Passes for lowering and handling the x86_amx type
- Wrappers internally use rows and cols to use amx definitions
This PR builds on PR #206888 and should be reviewed after/with it.
---
clang/include/clang/Basic/BuiltinsX86_64.td | 71 +-
clang/include/clang/Options/Options.td | 2 +
clang/lib/Basic/Targets/X86.cpp | 6 +
clang/lib/Basic/Targets/X86.h | 1 +
clang/lib/Headers/CMakeLists.txt | 1 +
clang/lib/Headers/acev1intrin.h | 766 ++++++++++++++++++
clang/lib/Headers/cpuid.h | 2 +
clang/lib/Headers/immintrin.h | 2 +
clang/lib/Sema/SemaX86.cpp | 5 +
clang/test/CodeGen/X86/ace-api.c | 145 ++++
clang/test/CodeGen/X86/ace-builtins.c | 119 +++
clang/test/CodeGen/X86/ace-inline-asm.c | 85 ++
clang/test/CodeGen/X86/acev1.c | 38 +
clang/test/CodeGen/X86/acev1_api.c | 138 ++++
clang/test/CodeGen/X86/acev1_bsr.c | 63 ++
clang/test/CodeGen/X86/acev1_errors.c | 17 +
clang/test/CodeGen/X86/acev1_inline_asm.c | 76 ++
clang/test/CodeGen/X86/acev1_mxfp8.c | 51 ++
clang/test/CodeGen/X86/acev1_tile_movement.c | 113 +++
clang/test/CodeGen/attr-target-x86.c | 4 +-
clang/test/Preprocessor/x86_target_features.c | 7 +
llvm/include/llvm/IR/IntrinsicsX86.td | 198 +++++
.../llvm/TargetParser/X86TargetParser.def | 1 +
.../lib/Target/X86/AsmParser/X86AsmParser.cpp | 50 ++
.../X86/MCTargetDesc/X86ATTInstPrinter.cpp | 92 ++-
.../X86/MCTargetDesc/X86ATTInstPrinter.h | 1 +
.../X86/MCTargetDesc/X86IntelInstPrinter.cpp | 88 +-
.../X86/MCTargetDesc/X86IntelInstPrinter.h | 1 +
llvm/lib/Target/X86/X86.td | 3 +
llvm/lib/Target/X86/X86ExpandPseudo.cpp | 89 ++
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 101 ++-
llvm/lib/Target/X86/X86FastTileConfig.cpp | 17 +-
llvm/lib/Target/X86/X86FrameLowering.cpp | 5 +-
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 19 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 290 ++++++-
llvm/lib/Target/X86/X86InstrACE.td | 417 ++++++++++
llvm/lib/Target/X86/X86InstrAMX.td | 67 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 8 +-
llvm/lib/Target/X86/X86InstrInfo.td | 3 +
llvm/lib/Target/X86/X86InstrPredicates.td | 3 +
llvm/lib/Target/X86/X86LowerAMXType.cpp | 144 +++-
llvm/lib/Target/X86/X86LowerTileCopy.cpp | 200 +++--
.../lib/Target/X86/X86MachineFunctionInfo.cpp | 4 +-
llvm/lib/Target/X86/X86MachineFunctionInfo.h | 22 +
llvm/lib/Target/X86/X86PreTileConfig.cpp | 10 +-
llvm/lib/Target/X86/X86RegisterInfo.cpp | 17 +-
llvm/lib/Target/X86/X86RegisterInfo.td | 9 +
llvm/lib/Target/X86/X86TileConfig.cpp | 10 +-
llvm/lib/TargetParser/X86TargetParser.cpp | 1 +
llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll | 244 ++++++
llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll | 99 +++
.../CodeGen/X86/ACE/acev1-outer-product.ll | 204 +++++
llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll | 94 +++
llvm/test/CodeGen/X86/ipra-reg-usage.ll | 4 +-
llvm/test/MC/X86/ACE/ace-att.s | 109 +++
llvm/test/MC/X86/ACE/ace-error.s | 39 +
llvm/test/MC/X86/ACE/ace-intel.s | 109 +++
llvm/test/TableGen/x86-fold-tables.inc | 10 +
58 files changed, 4364 insertions(+), 130 deletions(-)
create mode 100644 clang/lib/Headers/acev1intrin.h
create mode 100644 clang/test/CodeGen/X86/ace-api.c
create mode 100644 clang/test/CodeGen/X86/ace-builtins.c
create mode 100644 clang/test/CodeGen/X86/ace-inline-asm.c
create mode 100644 clang/test/CodeGen/X86/acev1.c
create mode 100644 clang/test/CodeGen/X86/acev1_api.c
create mode 100644 clang/test/CodeGen/X86/acev1_bsr.c
create mode 100644 clang/test/CodeGen/X86/acev1_errors.c
create mode 100644 clang/test/CodeGen/X86/acev1_inline_asm.c
create mode 100644 clang/test/CodeGen/X86/acev1_mxfp8.c
create mode 100644 clang/test/CodeGen/X86/acev1_tile_movement.c
create mode 100644 llvm/lib/Target/X86/X86InstrACE.td
create mode 100644 llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
create mode 100644 llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
create mode 100644 llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
create mode 100644 llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
create mode 100644 llvm/test/MC/X86/ACE/ace-att.s
create mode 100644 llvm/test/MC/X86/ACE/ace-error.s
create mode 100644 llvm/test/MC/X86/ACE/ace-intel.s
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index bf8ddd6559cde7..19bf9e01189920 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -221,6 +221,9 @@ let Features = "amx-int8", Attributes = [NoThrow] in {
let Features = "amx-tile", Attributes = [NoThrow] in {
def tilestored64_internal : X86Builtin<"void(unsigned short, unsigned short, void *, size_t, _Vector<256, int>)">;
+}
+
+let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
def tilezero_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short)">;
}
@@ -237,7 +240,7 @@ let Features = "amx-complex", Attributes = [NoThrow] in {
def tcmmrlfp16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-avx512,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+let Features = "amx-avx512,avx10.2|acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def tcvtrowd2ps_internal : X86Builtin<"_Vector<16, float>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
def tcvtrowps2bf16h_internal : X86Builtin<"_Vector<32, __bf16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
def tcvtrowps2bf16l_internal : X86Builtin<"_Vector<32, __bf16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
@@ -253,7 +256,7 @@ let Features = "amx-fp8", Attributes = [NoThrow] in {
def tdphf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-tile", Attributes = [NoThrow] in {
+let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
def tile_loadconfig : X86Builtin<"void(void const *)">;
def tile_storeconfig : X86Builtin<"void(void const *)">;
def tilerelease : X86Builtin<"void()">;
@@ -291,7 +294,7 @@ let Features = "amx-complex", Attributes = [NoThrow] in {
def tcmmrlfp16ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
}
-let Features = "amx-avx512,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+let Features = "amx-avx512,avx10.2|acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def tcvtrowd2ps : X86Builtin<"_Vector<16, float>(_Constant unsigned char, unsigned int)">;
def tcvtrowps2bf16h : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, unsigned int)">;
def tcvtrowps2bf16l : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, unsigned int)">;
@@ -387,3 +390,65 @@ let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<256>]
let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def vmovrsw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short const *>)">;
}
+
+// ACE (AI Compute Extensions) Builtins
+// Tile movement - single builtin handles both immediate and register index forms
+// The lowering code checks if index is constant and selects appropriate instruction
+let Features = "acev1", Attributes = [NoThrow] in {
+ def tilesetcol : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow writes TO tile (AMX tilemovrow reads FROM tile)
+ def tilesetrow : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+}
+
+// Tile movement internal (for IR-level operations)
+let Features = "acev1", Attributes = [NoThrow] in {
+ def tilesetcol_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow_to_tile writes TO tile (AMX tilemovrow reads FROM tile)
+ def tilesetrow_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+}
+
+// BSR management
+let Features = "acev1", Attributes = [NoThrow] in {
+ def bsrinit : X86Builtin<"void()">;
+ def bsrmovf : X86Builtin<"void(_Vector<16, int>, _Vector<16, int>)">;
+ def bsrmovh_set : X86Builtin<"void(_Vector<16, int>)">;
+ def bsrmovh_get : X86Builtin<"_Vector<16, int>()">;
+ def bsrmovl_set : X86Builtin<"void(_Vector<16, int>)">;
+ def bsrmovl_get : X86Builtin<"_Vector<16, int>()">;
+}
+
+// Outer products - user-facing (immediate tile IDs)
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4buud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4busd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4bssd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4bsud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+}
+
+// Mixed precision outer products - user-facing
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+}
+
+// Outer products - internal (for IR-level operations with tile values)
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top2bf16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
+ def top4buud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4busd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bssd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bsud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+}
+
+// Mixed precision internal
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top4mxhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxbhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxhbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxbssps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+}
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index db1648c2d55d87..c7cc6ca5a145c6 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -7462,6 +7462,8 @@ def mamx_tile : Flag<["-"], "mamx-tile">, Group<m_x86_Features_Group>;
def mno_amx_tile : Flag<["-"], "mno-amx-tile">, Group<m_x86_Features_Group>;
def mamx_movrs: Flag<["-"], "mamx-movrs">, Group<m_x86_Features_Group>;
def mno_amx_movrs: Flag<["-"], "mno-amx-movrs">, Group<m_x86_Features_Group>;
+def macev1 : Flag<["-"], "macev1">, Group<m_x86_Features_Group>;
+def mno_acev1 : Flag<["-"], "mno-acev1">, Group<m_x86_Features_Group>;
def mcmpccxadd : Flag<["-"], "mcmpccxadd">, Group<m_x86_Features_Group>;
def mno_cmpccxadd : Flag<["-"], "mno-cmpccxadd">, Group<m_x86_Features_Group>;
def msse : Flag<["-"], "msse">, Group<m_x86_Features_Group>;
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 670c85a769d71b..120324c31d550d 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -410,6 +410,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
HasAMXMOVRS = true;
} else if (Feature == "+amx-avx512") {
HasAMXAVX512 = true;
+ } else if (Feature == "+acev1") {
+ HasACEV1 = true;
} else if (Feature == "+cmpccxadd") {
HasCMPCCXADD = true;
} else if (Feature == "+raoint") {
@@ -960,6 +962,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__AMX_MOVRS__");
if (HasAMXAVX512)
Builder.defineMacro("__AMX_AVX512__");
+ if (HasACEV1)
+ Builder.defineMacro("__ACEV1__");
if (HasCMPCCXADD)
Builder.defineMacro("__CMPCCXADD__");
if (HasRAOINT)
@@ -1099,6 +1103,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
.Case("amx-int8", true)
.Case("amx-movrs", true)
.Case("amx-tile", true)
+ .Case("acev1", true)
.Case("avx", true)
.Case("avx10.1", true)
.Case("avx10.2", true)
@@ -1222,6 +1227,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
.Case("amx-int8", HasAMXINT8)
.Case("amx-movrs", HasAMXMOVRS)
.Case("amx-tile", HasAMXTILE)
+ .Case("acev1", HasACEV1)
.Case("avx", SSELevel >= AVX)
.Case("avx10.1", HasAVX10_1)
.Case("avx10.2", HasAVX10_2)
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 536168212a8cb6..7080a5656a4634 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -141,6 +141,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
bool HasAMXFP8 = false;
bool HasAMXMOVRS = false;
bool HasAMXAVX512 = false;
+ bool HasACEV1 = false;
bool HasSERIALIZE = false;
bool HasTSXLDTRK = false;
bool HasUSERMSR = false;
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 8d5b959bd24e58..7122e36ec062de 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -169,6 +169,7 @@ set(webassembly_files
set(x86_files
# Intrinsics
+ acev1intrin.h
adcintrin.h
adxintrin.h
ammintrin.h
diff --git a/clang/lib/Headers/acev1intrin.h b/clang/lib/Headers/acev1intrin.h
new file mode 100644
index 00000000000000..0491c7f0b383d7
--- /dev/null
+++ b/clang/lib/Headers/acev1intrin.h
@@ -0,0 +1,766 @@
+/*===-------------- acev1intrin.h - ACEV1 intrinsics -*- C/C++ -*------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===------------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <acev1intrin.h> directly; include <immintrin.h> instead."
+#endif /* __IMMINTRIN_H */
+
+#ifndef __ACEV1INTRIN_H
+#define __ACEV1INTRIN_H
+#ifdef __x86_64__
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS_ACE \
+ __attribute__((__always_inline__, __nodebug__, __target__("acev1")))
+
+/// Load tile configuration from a 64-byte memory location. For ACE
+/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1),
+/// ACE tiles have fixed dimensions of 16 rows × 64 bytes, so per-tile
+/// row/column configuration bytes are ignored. If palette_id is zero,
+/// tiles return to init state and are zeroed. Invalid configurations
+/// result in #GP fault.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> LDTILECFG </c> instruction.
+///
+/// \param __config
+/// A pointer to 64-byte tile configuration (use __ace_tile_config).
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_tile_ace_loadconfig(const void *__config) {
+ __builtin_ia32_tile_loadconfig(__config);
+}
+
+/// Store the current tile configuration to a 64-byte memory location.
+/// For ACE (Palette 2), the stored palette_id will be 2 and per-tile
+/// configuration bytes reflect the fixed 16×64 dimensions. If tiles
+/// are not configured, all zeroes are stored.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> STTILECFG </c> instruction.
+///
+/// \param __config
+/// A pointer to 64-byte tile configuration buffer.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_tile_ace_storeconfig(void *__config) {
+ __builtin_ia32_tile_storeconfig(__config);
+}
+
+/// Release the tile configuration to return to init state, releasing
+/// all tile storage. After this, tiles must be reconfigured with
+/// _tile_ace_loadconfig before use.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILERELEASE </c> instruction.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
+ __builtin_ia32_tilerelease();
+}
+
+/// Zero the ACE tile specified by "tile". Sets all 1024 bytes
+/// (16 rows × 64 bytes) of the tile register to zero.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEZERO </c> instruction.
+///
+/// \param tile
+/// Destination tile register ID (0-7).
+#define _tile_ace_zero(tile) __builtin_ia32_tilezero((tile))
+
+/// Move a 64-byte ZMM vector to a tile column. The 16 doublewords from
+/// the ZMM are written as a vertical column in the tile at the specified
+/// index.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVCOL </c> instruction.
+///
+/// \param dst
+/// Destination tile register ID (0-7).
+/// \param src
+/// Source ZMM register ID (0-31).
+/// \param idx
+/// Column index (0-15). Immediate or register form selected automatically.
+#define _tile_setcol(dst, src, idx) \
+ __builtin_ia32_tilesetcol((dst), (src), (idx))
+
+/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written
+/// as a horizontal row in the tile at the specified index.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param dst
+/// Destination tile register ID (0-7).
+/// \param src
+/// Source ZMM register ID (0-31).
+/// \param idx
+/// Row index (0-15). Immediate or register form selected automatically.
+#define _tile_setrow(dst, src, idx) \
+ __builtin_ia32_tilesetrow((dst), (src), (idx))
+
+/// Initialize the Block Scale Register (BSR) to zero. The BSR holds
+/// 32 scaling factors used by mixed-precision outer product instructions
+/// (TOP4MX* variants). Must be called before using BSR-scaled operations.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRINIT </c> instruction.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) {
+ __builtin_ia32_bsrinit();
+}
+
+/// Load the full BSR (32 scale factors) from two ZMM registers. The low
+/// half (16 factors) comes from __src1, high half from __src2. Each
+/// doubleword contains one scale factor.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVF </c> instruction.
+///
+/// \param __src1
+/// ZMM with scale factors for BSR low half (factors 0-15).
+/// \param __src2
+/// ZMM with scale factors for BSR high half (factors 16-31).
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1,
+ __m512i __src2) {
+ __builtin_ia32_bsrmovf((__v16si)__src1, (__v16si)__src2);
+}
+
+/// Load the high half of BSR (scale factors 16-31) from a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
+///
+/// \param __src
+/// ZMM with 16 scale factors to write to BSR high half.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) {
+ __builtin_ia32_bsrmovh_set((__v16si)__src);
+}
+
+/// Read the high half of BSR (scale factors 16-31) to a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
+///
+/// \returns
+/// ZMM containing 16 scale factors from BSR high half.
+static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) {
+ return (__m512i)__builtin_ia32_bsrmovh_get();
+}
+
+/// Load the low half of BSR (scale factors 0-15) from a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
+///
+/// \param __src
+/// ZMM with 16 scale factors to write to BSR low half.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) {
+ __builtin_ia32_bsrmovl_set((__v16si)__src);
+}
+
+/// Read the low half of BSR (scale factors 0-15) to a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
+///
+/// \returns
+/// ZMM containing 16 scale factors from BSR low half.
+static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
+ return (__m512i)__builtin_ia32_bsrmovl_get();
+}
+
+/// Compute 2-way outer product of BF16 pairs, accumulating to FP32.
+/// Each BF16 pair from src1 and src2 produces two FP32 products that
+/// are accumulated into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP2BF16PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 32 BF16 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 32 BF16 values.
+#define _tile_top2bf16ps(dst, src1, src2) \
+ __builtin_ia32_top2bf16ps((dst), (src1), (src2))
+
+/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Each group of 4 unsigned byte pairs from src1 and src2 produces
+/// 4 products accumulated into the destination tile as 32-bit integers.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUUD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+#define _tile_top4buud(dst, src1, src2) \
+ __builtin_ia32_top4buud((dst), (src1), (src2))
+
+/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Each group of 4 byte pairs (unsigned from src1, signed from src2)
+/// produces 4 products accumulated into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUSD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+#define _tile_top4busd(dst, src1, src2) \
+ __builtin_ia32_top4busd((dst), (src1), (src2))
+
+/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Each group of 4 signed byte pairs produces 4 products accumulated
+/// into the destination tile as 32-bit integers.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSSD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+#define _tile_top4bssd(dst, src1, src2) \
+ __builtin_ia32_top4bssd((dst), (src1), (src2))
+
+/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Each group of 4 byte pairs (signed from src1, unsigned from src2)
+/// produces 4 products accumulated into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSUD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+#define _tile_top4bsud(dst, src1, src2) \
+ __builtin_ia32_top4bsud((dst), (src1), (src2))
+
+/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
+/// Multiplies HF8 values from src1 with BF8 values from src2, converting
+/// to FP32 and accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing HF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing BF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxhf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxhf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product with BF8/HF8 format.
+/// Multiplies BF8 values from src1 with HF8 (E4M3) values from src2,
+/// converting to FP32 and accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBHF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing BF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing HF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxbhf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbhf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product with HF8/BF8 format.
+/// Multiplies HF8 (E4M3) values from src1 with BF8 values from src2,
+/// converting to FP32 and accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHBF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing HF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing BF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxhbf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxhbf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
+/// Multiplies BF8 values from both sources, converting to FP32 and
+/// accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing BF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing BF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxbf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product of signed INT8 with BSR
+/// scaling. Multiplies signed bytes, applies scale factors from the BSR,
+/// and accumulates FP32 results into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBSSPS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing signed bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing signed bytes.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define _tile_top4mxbssps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbssps((dst), (src1), (src2), (imm))
+
+/// ACE tile type with fixed dimensions (16 rows × 64 bytes = 1024 bytes).
+/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1.
+typedef int __acetile __attribute__((__vector_size__(1024), __aligned__(64)));
+
+/// ACE Palette 2 tile configuration structure (64 bytes).
+/// For ACE, only byte 0 (palette_id = 2) is significant; bytes 1-63
+/// must be zero. Unlike AMX Palette 1, tile dimensions are fixed.
+typedef struct __attribute__((__packed__, __aligned__(64))) {
+ unsigned char palette_id;
+ unsigned char reserved[63];
+} __ace_tile_config;
+
+/// Initialize an ACE tile configuration structure for Palette 2.
+/// Sets palette_id to 2 and clears all reserved bytes to zero.
+/// Use with _tile_ace_loadconfig to configure tiles for ACE operations.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+__ace_init_config(__ace_tile_config *cfg) {
+ for (int i = 0; i < 64; i++)
+ ((unsigned char *)cfg)[i] = 0;
+ cfg->palette_id = 2;
+}
+
+/// Zero an ACE tile variable. Sets all 1024 bytes to zero.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEZERO </c> instruction.
+///
+/// \param dst
+/// Pointer to __acetile variable to be zeroed.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_zero(__acetile *dst) {
+ *dst = __builtin_ia32_tilezero_internal(16, 64);
+}
+
+/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Multiplies 64 unsigned bytes from each ZMM source, producing a
+/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUUD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 unsigned bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4buud(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4buud_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Multiplies 64 bytes from each ZMM source (unsigned from src1,
+/// signed from src2), producing a 16×16 grid of 32-bit sums.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUSD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 signed bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4busd(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4busd_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Multiplies 64 signed bytes from each ZMM source, producing a
+/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSSD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 signed bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4bssd(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4bssd_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Multiplies 64 bytes from each ZMM source (signed from src1,
+/// unsigned from src2), producing a 16×16 grid of 32-bit sums.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSUD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 unsigned bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4bsud(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4bsud_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 2-way outer product of BF16 to FP32.
+/// Multiplies 32 BF16 values from each source, producing a 16×16 grid
+/// of FP32 sums accumulated into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP2BF16PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector (__m512bh) containing 32 BF16 values.
+/// \param src2
+/// Second source ZMM vector (__m512bh) containing 32 BF16 values.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
+ __m512bh src2) {
+ *dst = __builtin_ia32_top2bf16ps_internal(16, 64, 64, *dst, (__v32bf)src1,
+ (__v32bf)src2);
+}
+
+/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
+/// Multiplies HF8 values from src1 with HF8 values from src2, applies
+/// BSR scaling, converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing HF8 (E4M3) values.
+/// \param src2
+/// Second source ZMM vector containing HF8 (E4M3) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxhf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxhf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product with BF8/HF8 format.
+/// Multiplies BF8 (E5M2) values from src1 with HF8 (E4M3) values from src2,
+/// applies BSR scaling, converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBHF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing BF8 (E5M2) values.
+/// \param src2
+/// Second source ZMM vector containing HF8 (E4M3) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxbhf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxbhf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product with HF8/BF8 format.
+/// Multiplies HF8 (E4M3) values from src1 with BF8 (E5M2) values from src2,
+/// applies BSR scaling, converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHBF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing HF8 (E4M3) values.
+/// \param src2
+/// Second source ZMM vector containing BF8 (E5M2) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxhbf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxhbf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
+/// Multiplies BF8 values from both sources, applies BSR scaling,
+/// converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing BF8 (E5M2) values.
+/// \param src2
+/// Second source ZMM vector containing BF8 (E5M2) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxbf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxbf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product of MX INT8 with BSR scaling.
+/// Multiplies signed MX INT8 bytes from both sources, applies BSR scaling,
+/// converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBSSPS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing signed MX INT8 values.
+/// \param src2
+/// Second source ZMM vector containing signed MX INT8 values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxbssps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxbssps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Write a ZMM vector as a column in an ACE tile.
+/// The 16 doublewords from src are written vertically at column idx.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVCOL </c> instruction.
+///
+/// \param dst
+/// Pointer to destination __acetile.
+/// \param src
+/// Source ZMM vector (__m512i) with 16 doublewords.
+/// \param idx
+/// Column index (0-15).
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
+ unsigned int idx) {
+ *dst = __builtin_ia32_tilesetcol_internal(16, 64, (__v16si)src, idx);
+}
+
+/// Write a ZMM vector as a row in an ACE tile.
+/// The 64 bytes from src are written horizontally at row idx.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param dst
+/// Pointer to destination __acetile.
+/// \param src
+/// Source ZMM vector (__m512i) with 64 bytes.
+/// \param idx
+/// Row index (0-15).
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_setrow(__acetile *dst, __m512i src,
+ unsigned int idx) {
+ *dst = __builtin_ia32_tilesetrow_internal(16, 64, (__v16si)src, idx);
+}
+
+/// Read a row from an ACE tile to a ZMM vector.
+/// Returns the 64 bytes at row idx as a ZMM vector.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512i) containing the 64-byte row.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512i __tile_ace_getrow(__acetile *src, unsigned int idx) {
+ return (__m512i)__builtin_ia32_tilemovrow_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert INT32 elements to FP32.
+/// Each of the 16 INT32 elements in the row is converted to FP32.
+/// Rounding uses RTNE (round to nearest even).
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWD2PS </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing INT32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512) containing 16 FP32 values.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512 __tile_ace_cvtrowd2ps(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowd2ps_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to BF16 (high).
+/// Each FP32 element is converted to BF16 and placed in the high 16 bits
+/// of each destination dword; low 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2BF16H </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512bh) with BF16 values in high half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512bh __tile_ace_cvtrowps2bf16h(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2bf16h_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to BF16 (low).
+/// Each FP32 element is converted to BF16 and placed in the low 16 bits
+/// of each destination dword; high 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2BF16L </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512bh) with BF16 values in low half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512bh __tile_ace_cvtrowps2bf16l(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2bf16l_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to FP16 (high).
+/// Each FP32 element is converted to FP16 and placed in the high 16 bits
+/// of each destination dword; low 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2PHH </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512h) with FP16 values in high half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512h __tile_ace_cvtrowps2phh(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2phh_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to FP16 (low).
+/// Each FP32 element is converted to FP16 and placed in the low 16 bits
+/// of each destination dword; high 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2PHL </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512h) with FP16 values in low half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512h __tile_ace_cvtrowps2phl(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2phl_internal(16, 64, *src, idx);
+}
+
+#undef __DEFAULT_FN_ATTRS_ACE
+
+#endif /* __x86_64__ */
+#endif /* __ACEV1INTRIN_H */
diff --git a/clang/lib/Headers/cpuid.h b/clang/lib/Headers/cpuid.h
index 57c6ab9ba7362e..24715bf288b612 100644
--- a/clang/lib/Headers/cpuid.h
+++ b/clang/lib/Headers/cpuid.h
@@ -225,6 +225,8 @@
/* Features in %ecx for leaf 0x24 sub-leaf 1 */
#define bit_AVX10_V2_AUX 0x00000008
+/* Features in %ecx for leaf 7 sub-leaf 1 */
+#define bit_ACEV1 0x00000800
/* Features in %eax for leaf 13 sub-leaf 1 */
#define bit_XSAVEOPT 0x00000001
#define bit_XSAVEC 0x00000002
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index ef3e638d4d0439..087b4659f1b7dd 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -483,6 +483,8 @@ _storebe_i64(void * __P, long long __D) {
#include <amxavx512intrin.h>
+#include <acev1intrin.h>
+
#include <avx512vp2intersectintrin.h>
#include <avx512vlvp2intersectintrin.h>
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index 0c2e6281729589..1599672ec5404b 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -548,6 +548,11 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tdpbhf8ps:
case X86::BI__builtin_ia32_tdphbf8ps:
case X86::BI__builtin_ia32_tdphf8ps:
+ case X86::BI__builtin_ia32_top2bf16ps:
+ case X86::BI__builtin_ia32_top4buud:
+ case X86::BI__builtin_ia32_top4busd:
+ case X86::BI__builtin_ia32_top4bssd:
+ case X86::BI__builtin_ia32_top4bsud:
return CheckBuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
case X86::BI__builtin_ia32_tcvtrowps2bf16hi:
case X86::BI__builtin_ia32_tcvtrowps2bf16li:
diff --git a/clang/test/CodeGen/X86/ace-api.c b/clang/test/CodeGen/X86/ace-api.c
new file mode 100644
index 00000000000000..8f82fbf66e39ad
--- /dev/null
+++ b/clang/test/CodeGen/X86/ace-api.c
@@ -0,0 +1,145 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
+// RUN: -emit-llvm -o - -Werror | FileCheck %s
+
+// Test ACE struct-based API functions using __acetile type
+// ACE v1 uses fixed 16x64 tile dimensions (Palette 2)
+
+#include <immintrin.h>
+
+// CHECK-LABEL: @test_ace_top4buud
+// CHECK: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %{{.*}})
+// CHECK: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+// CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %{{.*}})
+void test_ace_top4buud(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4buud(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4busd
+// CHECK: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+void test_ace_top4busd(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4busd(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4bssd
+// CHECK: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+void test_ace_top4bssd(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4bssd(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4bsud
+// CHECK: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+void test_ace_top4bsud(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4bsud(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top2bf16ps
+// CHECK: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
+void test_ace_top2bf16ps(__acetile *dst, __m512bh a, __m512bh b) {
+ __tile_ace_top2bf16ps(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4mxhf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxhf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxhf8ps(dst, a, b, 5);
+}
+
+// CHECK-LABEL: @test_ace_top4mxbhf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxbhf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxbhf8ps(dst, a, b, 3);
+}
+
+// CHECK-LABEL: @test_ace_top4mxhbf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxhbf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxhbf8ps(dst, a, b, 7);
+}
+
+// CHECK-LABEL: @test_ace_top4mxbf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxbf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxbf8ps(dst, a, b, 1);
+}
+
+// CHECK-LABEL: @test_ace_top4mxbssps
+// CHECK: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxbssps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxbssps(dst, a, b, 9);
+}
+
+// CHECK-LABEL: @test_ace_setcol
+// CHECK: call x86_amx @llvm.x86.tilesetcol.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
+void test_ace_setcol(__acetile *dst, __m512i src) {
+ __tile_ace_setcol(dst, src, 3);
+}
+
+// CHECK-LABEL: @test_ace_setrow
+// CHECK: call x86_amx @llvm.x86.tilesetrow.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
+void test_ace_setrow(__acetile *dst, __m512i src) {
+ __tile_ace_setrow(dst, src, 5);
+}
+
+// CHECK-LABEL: @test_ace_getrow
+// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512i test_ace_getrow(__acetile *src) {
+ return __tile_ace_getrow(src, 7);
+}
+
+// CHECK-LABEL: @test_ace_zero
+// CHECK: call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+void test_ace_zero(__acetile *dst) {
+ __tile_ace_zero(dst);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowd2ps
+// CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512 test_ace_cvtrowd2ps(__acetile *src) {
+ return __tile_ace_cvtrowd2ps(src, 2);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2bf16h
+// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512bh test_ace_cvtrowps2bf16h(__acetile *src) {
+ return __tile_ace_cvtrowps2bf16h(src, 4);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2bf16l
+// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512bh test_ace_cvtrowps2bf16l(__acetile *src) {
+ return __tile_ace_cvtrowps2bf16l(src, 6);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2phh
+// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512h test_ace_cvtrowps2phh(__acetile *src) {
+ return __tile_ace_cvtrowps2phh(src, 8);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2phl
+// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512h test_ace_cvtrowps2phl(__acetile *src) {
+ return __tile_ace_cvtrowps2phl(src, 10);
+}
+
+// CHECK-LABEL: @test_ace_workflow
+// CHECK: call x86_amx @llvm.x86.tilezero.internal
+// CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+// CHECK: call x86_amx @llvm.x86.top4buud.internal
+// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+void test_ace_workflow(__m512i *input, __m512i *output) {
+ __acetile acc;
+
+ // Zero the tile
+ __tile_ace_zero(&acc);
+
+ // Initialize with data via setrow
+ __tile_ace_setrow(&acc, input[0], 0);
+
+ // Perform outer product computation
+ __tile_ace_top4buud(&acc, input[1], input[2]);
+
+ // Extract result via getrow
+ output[0] = __tile_ace_getrow(&acc, 0);
+}
diff --git a/clang/test/CodeGen/X86/ace-builtins.c b/clang/test/CodeGen/X86/ace-builtins.c
new file mode 100644
index 00000000000000..edaa80c78486f0
--- /dev/null
+++ b/clang/test/CodeGen/X86/ace-builtins.c
@@ -0,0 +1,119 @@
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +acev1 -target-feature +avx512f -emit-llvm -o - | FileCheck %s
+
+#include <immintrin.h>
+
+// BSR Management Tests
+
+// CHECK-LABEL: @test_bsrinit
+// CHECK: call void @llvm.x86.bsrinit()
+void test_bsrinit(void) {
+ _bsr0_init();
+}
+
+// CHECK-LABEL: @test_bsrmovf
+// CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_bsrmovf(__m512i a, __m512i b) {
+ _bsr0_movf(a, b);
+}
+
+// CHECK-LABEL: @test_bsrmovh_set
+// CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
+void test_bsrmovh_set(__m512i a) {
+ _bsr0_movh_set(a);
+}
+
+// CHECK-LABEL: @test_bsrmovh_get
+// CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+__m512i test_bsrmovh_get(void) {
+ return _bsr0_movh_get();
+}
+
+// CHECK-LABEL: @test_bsrmovl_set
+// CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
+void test_bsrmovl_set(__m512i a) {
+ _bsr0_movl_set(a);
+}
+
+// CHECK-LABEL: @test_bsrmovl_get
+// CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+__m512i test_bsrmovl_get(void) {
+ return _bsr0_movl_get();
+}
+
+// Tile Movement Tests
+
+// CHECK-LABEL: @test_tilemovcol
+// CHECK: call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %{{.*}}, i32 5)
+void test_tilemovcol(__m512i src) {
+ _tile_setcol(0, src, 5);
+}
+
+// CHECK-LABEL: @test_tilemovrow_to_tile
+// CHECK: call void @llvm.x86.tilesetrow(i8 0, <16 x i32> %{{.*}}, i32 5)
+void test_tilemovrow_to_tile(__m512i src) {
+ _tile_setrow(0, src, 5);
+}
+
+// Outer Product Tests
+
+// CHECK-LABEL: @test_top2bf16ps
+// CHECK: call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
+void test_top2bf16ps(void) {
+ _tile_top2bf16ps(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4buud
+// CHECK: call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
+void test_top4buud(void) {
+ _tile_top4buud(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4busd
+// CHECK: call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
+void test_top4busd(void) {
+ _tile_top4busd(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4bssd
+// CHECK: call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
+void test_top4bssd(void) {
+ _tile_top4bssd(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4bsud
+// CHECK: call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
+void test_top4bsud(void) {
+ _tile_top4bsud(0, 1, 2);
+}
+
+// Mixed Precision Outer Product Tests
+
+// CHECK-LABEL: @test_top4mxhf8ps
+// CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxhf8ps(void) {
+ _tile_top4mxhf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxbhf8ps
+// CHECK: call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxbhf8ps(void) {
+ _tile_top4mxbhf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxhbf8ps
+// CHECK: call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxhbf8ps(void) {
+ _tile_top4mxhbf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxbf8ps
+// CHECK: call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxbf8ps(void) {
+ _tile_top4mxbf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxbssps
+// CHECK: call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxbssps(void) {
+ _tile_top4mxbssps(0, 1, 2, 5);
+}
diff --git a/clang/test/CodeGen/X86/ace-inline-asm.c b/clang/test/CodeGen/X86/ace-inline-asm.c
new file mode 100644
index 00000000000000..27e36d84149fcc
--- /dev/null
+++ b/clang/test/CodeGen/X86/ace-inline-asm.c
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f \
+// RUN: -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+// Test ACE instructions in inline assembly
+
+// BSR Management Instructions
+
+// CHECK-LABEL: @test_bsrinit_asm
+// CHECK: call void asm sideeffect "bsrinit", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrinit_asm(void) {
+ __asm__ volatile("bsrinit");
+}
+
+// CHECK-LABEL: @test_bsrmovf_asm
+// CHECK: call void asm sideeffect "bsrmovf %zmm1, %zmm0", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrmovf_asm(void) {
+ __asm__ volatile("bsrmovf %%zmm1, %%zmm0" :::);
+}
+
+// CHECK-LABEL: @test_bsrmovh_asm
+// CHECK: call void asm sideeffect "bsrmovh %zmm2", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrmovh_asm(void) {
+ __asm__ volatile("bsrmovh %%zmm2" :::);
+}
+
+// CHECK-LABEL: @test_bsrmovl_asm
+// CHECK: call void asm sideeffect "bsrmovl %zmm3", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrmovl_asm(void) {
+ __asm__ volatile("bsrmovl %%zmm3" :::);
+}
+
+// Tile Movement Instructions
+
+// CHECK-LABEL: @test_tilemovcol_asm
+// CHECK: call void asm sideeffect "tilemovcol $$$$5, %zmm0, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
+void test_tilemovcol_asm(void) {
+ __asm__ volatile("tilemovcol $$5, %%zmm0, %%tmm1" ::: "tmm1");
+}
+
+// CHECK-LABEL: @test_tilemovrow_asm
+// CHECK: call void asm sideeffect "tilemovrow $$$$3, %zmm0, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
+void test_tilemovrow_asm(void) {
+ __asm__ volatile("tilemovrow $$3, %%zmm0, %%tmm2" ::: "tmm2");
+}
+
+// Outer Product Instructions
+
+// CHECK-LABEL: @test_top4buud_asm
+// CHECK: call void asm sideeffect "top4buud %zmm1, %zmm0, %tmm0", "~{tmm0},~{dirflag},~{fpsr},~{flags}"()
+void test_top4buud_asm(void) {
+ __asm__ volatile("top4buud %%zmm1, %%zmm0, %%tmm0" ::: "tmm0");
+}
+
+// CHECK-LABEL: @test_top4bssd_asm
+// CHECK: call void asm sideeffect "top4bssd %zmm2, %zmm1, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
+void test_top4bssd_asm(void) {
+ __asm__ volatile("top4bssd %%zmm2, %%zmm1, %%tmm1" ::: "tmm1");
+}
+
+// CHECK-LABEL: @test_top2bf16ps_asm
+// CHECK: call void asm sideeffect "top2bf16ps %zmm3, %zmm2, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
+void test_top2bf16ps_asm(void) {
+ __asm__ volatile("top2bf16ps %%zmm3, %%zmm2, %%tmm2" ::: "tmm2");
+}
+
+// Mixed Precision Outer Product Instructions
+
+// CHECK-LABEL: @test_top4mxhfbps_asm
+// CHECK: call void asm sideeffect "top4mxhfbps $$$$5, %zmm1, %zmm0, %tmm3", "~{tmm3},~{dirflag},~{fpsr},~{flags}"()
+void test_top4mxhfbps_asm(void) {
+ __asm__ volatile("top4mxhfbps $$5, %%zmm1, %%zmm0, %%tmm3" ::: "tmm3");
+}
+
+// Full ACE operation sequence in inline assembly
+// CHECK-LABEL: @test_ace_sequence_asm
+// CHECK: call void asm sideeffect
+void test_ace_sequence_asm(void) {
+ __asm__ volatile(
+ "bsrinit \n\t"
+ "top4buud %%zmm1, %%zmm0, %%tmm0 \n\t"
+ "top4bssd %%zmm3, %%zmm2, %%tmm1 \n\t"
+ ::: "memory", "tmm0", "tmm1"
+ );
+}
diff --git a/clang/test/CodeGen/X86/acev1.c b/clang/test/CodeGen/X86/acev1.c
new file mode 100644
index 00000000000000..8f487b8ea2f51a
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1.c
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests macro-based ACE v1 intrinsics emit correct LLVM IR
+
+#include <immintrin.h>
+
+void test_acev1_outer_products(void) {
+ // CHECK-LABEL: @test_acev1_outer_products
+ // CHECK: call void @llvm.x86.top2bf16ps(i8 0,
+ // CHECK: call void @llvm.x86.top4buud(i8 1,
+ // CHECK: call void @llvm.x86.top4busd(i8 2,
+ // CHECK: call void @llvm.x86.top4bssd(i8 3,
+ // CHECK: call void @llvm.x86.top4bsud(i8 4,
+ _tile_top2bf16ps(0, 1, 2);
+ _tile_top4buud(1, 2, 3);
+ _tile_top4busd(2, 3, 4);
+ _tile_top4bssd(3, 4, 5);
+ _tile_top4bsud(4, 5, 6);
+}
+
+void test_acev1_tile_config(void *data) {
+ // CHECK-LABEL: @test_acev1_tile_config
+ // CHECK: call void @llvm.x86.ldtilecfg(ptr %{{.*}})
+ // CHECK: call void @llvm.x86.sttilecfg(ptr %{{.*}})
+ // CHECK: call void @llvm.x86.tilerelease()
+ // CHECK: call void @llvm.x86.tilezero(i8 0)
+ _tile_ace_loadconfig(data);
+ _tile_ace_storeconfig(data);
+ _tile_ace_release();
+ _tile_ace_zero(0);
+}
+
+void test_acev1_bsr(void) {
+ // CHECK-LABEL: @test_acev1_bsr
+ // CHECK: call void @llvm.x86.bsrinit()
+ _bsr0_init();
+}
diff --git a/clang/test/CodeGen/X86/acev1_api.c b/clang/test/CodeGen/X86/acev1_api.c
new file mode 100644
index 00000000000000..acb12dae30cc4e
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_api.c
@@ -0,0 +1,138 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
+// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 APIs using __acetile type (fixed 16x64 dimensions)
+// Note: ACE v1 does not have TILELOADD/TILESTORED - uses ACE-specific tile operations
+
+#include <immintrin.h>
+
+// Test __acetile zero
+void test_tile_ace_zero(void) {
+ // CHECK-LABEL: @test_tile_ace_zero
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __acetile c;
+ __tile_ace_zero(&c);
+}
+
+// Test TOP4BUUD ACE API
+void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4buud
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4buud(&dst, src1, src2);
+}
+
+// Test TOP4BUSD ACE API
+void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4busd
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4busd(&dst, src1, src2);
+}
+
+// Test TOP4BSSD ACE API
+void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4bssd
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4bssd(&dst, src1, src2);
+}
+
+// Test TOP4BSUD ACE API
+void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4bsud
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4bsud(&dst, src1, src2);
+}
+
+// Test TOP2BF16PS ACE API
+void test_tile_ace_top2bf16ps(__acetile dst, __m512bh src1, __m512bh src2) {
+ // CHECK-LABEL: @test_tile_ace_top2bf16ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top2bf16ps(&dst, src1, src2);
+}
+
+// Test TOP4MXHF8PS ACE API (HF8 x HF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxhf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxhf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxhf8ps(&dst, src1, src2, 5);
+}
+
+// Test TOP4MXBHF8PS ACE API (BF8 x HF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxbhf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxbhf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxbhf8ps(&dst, src1, src2, 3);
+}
+
+// Test TOP4MXHBF8PS ACE API (HF8 x BF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxhbf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxhbf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxhbf8ps(&dst, src1, src2, 7);
+}
+
+// Test TOP4MXBF8PS ACE API (BF8 x BF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxbf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxbf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxbf8ps(&dst, src1, src2, 1);
+}
+
+// Test TOP4MXBSSPS ACE API (MX INT8 SxS -> FP32 with BSR scaling)
+void test_tile_ace_top4mxbssps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxbssps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxbssps(&dst, src1, src2, 9);
+}
+
+// Test tile setrow (ACE-specific: ZMM to tile row)
+void test_tile_ace_setrow(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setrow
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __tile_ace_setrow(&dst, src, 0);
+}
+
+// Test tile setcol (ACE-specific: ZMM to tile column)
+void test_tile_ace_setcol(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setcol
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __tile_ace_setcol(&dst, src, 0);
+}
+
+// Test tile getrow (ACE-specific: tile row to ZMM)
+__m512i test_tile_ace_getrow(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_getrow
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ return __tile_ace_getrow(&src, 0);
+}
+
+// Integration test: ACE computation workflow
+void test_ace_workflow(__m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_ace_workflow
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.top4buud.internal
+ __acetile acc;
+ __tile_ace_zero(&acc);
+ __tile_ace_top4buud(&acc, src1, src2);
+}
diff --git a/clang/test/CodeGen/X86/acev1_bsr.c b/clang/test/CodeGen/X86/acev1_bsr.c
new file mode 100644
index 00000000000000..d68ecdcb59f7d2
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_bsr.c
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 BSR (Block Sparse Register) operations emit correct LLVM IR
+
+#include <immintrin.h>
+#include <stdint.h>
+
+void test_bsr_init(void) {
+ // CHECK-LABEL: @test_bsr_init
+ // CHECK: call void @llvm.x86.bsrinit()
+ _bsr0_init();
+}
+
+void test_bsr_movf(__m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_bsr_movf
+ // CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+ _bsr0_movf(src1, src2);
+}
+
+void test_bsr_movh_set(__m512i src) {
+ // CHECK-LABEL: @test_bsr_movh_set
+ // CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
+ _bsr0_movh_set(src);
+}
+
+__m512i test_bsr_movh_get(void) {
+ // CHECK-LABEL: @test_bsr_movh_get
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+ return _bsr0_movh_get();
+}
+
+void test_bsr_movl_set(__m512i src) {
+ // CHECK-LABEL: @test_bsr_movl_set
+ // CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
+ _bsr0_movl_set(src);
+}
+
+__m512i test_bsr_movl_get(void) {
+ // CHECK-LABEL: @test_bsr_movl_get
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ return _bsr0_movl_get();
+}
+
+// Test BSR operations sequence
+void test_bsr_sequence(void) {
+ // CHECK-LABEL: @test_bsr_sequence
+ // CHECK: call void @llvm.x86.bsrinit()
+ // CHECK: call void @llvm.x86.bsrmovf
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ _bsr0_init();
+
+ __m512i zmm1 = _mm512_set1_epi32(1);
+ __m512i zmm2 = _mm512_set1_epi32(2);
+ _bsr0_movf(zmm1, zmm2);
+
+ __m512i high = _bsr0_movh_get();
+ __m512i low = _bsr0_movl_get();
+
+ (void)high;
+ (void)low;
+}
diff --git a/clang/test/CodeGen/X86/acev1_errors.c b/clang/test/CodeGen/X86/acev1_errors.c
new file mode 100644
index 00000000000000..c928882b62f4a3
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_errors.c
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -fsyntax-only -verify
+
+// Tests compile-time semantic errors for ACE v1 intrinsics
+
+#include <immintrin.h>
+
+void test_tile_range_errors(void) {
+ // Tile index must be in range [0, 7]
+ _tile_ace_zero(8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_zero(16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+
+ // Outer product tile index errors
+ _tile_top4buud(8, 0, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_top4bssd(16, 0, 0); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ _tile_top2bf16ps(9, 0, 0); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
+}
diff --git a/clang/test/CodeGen/X86/acev1_inline_asm.c b/clang/test/CodeGen/X86/acev1_inline_asm.c
new file mode 100644
index 00000000000000..1dbf7d76a53554
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_inline_asm.c
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
+
+// Tests inline assembly with ACE v1 tile registers
+
+#include <immintrin.h>
+
+void test_acev1_inline_asm_outer_product(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_outer_product
+ // CHECK: call void asm sideeffect "tilezero %tmm0
+ // CHECK: top4buud %zmm0, %zmm1, %tmm0
+ // CHECK: tilestored %tmm0
+ __asm__ volatile (
+ "tilezero %%tmm0 \n\t"
+ "top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
+ "tilestored %%tmm0, 0(%%rdi) \n\t"
+ ::: "memory", "tmm0", "zmm0", "zmm1"
+ );
+}
+
+void test_acev1_inline_asm_bf16(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_bf16
+ // CHECK: call void asm sideeffect "tilezero %tmm1
+ // CHECK: top2bf16ps %zmm2, %zmm3, %tmm1
+ __asm__ volatile (
+ "tilezero %%tmm1 \n\t"
+ "top2bf16ps %%zmm2, %%zmm3, %%tmm1 \n\t"
+ ::: "memory", "tmm1", "zmm2", "zmm3"
+ );
+}
+
+void test_acev1_inline_asm_bsr(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_bsr
+ // CHECK: call void asm sideeffect "bsrinit %bsr0
+ // CHECK: bsrmovf %zmm0, %zmm1
+ __asm__ volatile (
+ "bsrinit %%bsr0 \n\t"
+ "bsrmovf %%zmm0, %%zmm1 \n\t"
+ ::: "memory", "zmm0", "zmm1"
+ );
+}
+
+void test_acev1_inline_asm_tile_config(void *cfg) {
+ // CHECK-LABEL: @test_acev1_inline_asm_tile_config
+ // CHECK: call void asm sideeffect "ldtilecfg ($0)
+ // CHECK: tilerelease
+ __asm__ volatile (
+ "ldtilecfg (%0) \n\t"
+ "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm1 \n\t"
+ "tilerelease \n\t"
+ :: "r"(cfg) : "memory", "tmm0", "tmm1"
+ );
+}
+
+void test_acev1_inline_asm_all_outer_products(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_all_outer_products
+ // CHECK: top4buud
+ // CHECK: top4busd
+ // CHECK: top4bssd
+ // CHECK: top4bsud
+ // CHECK: top2bf16ps
+ __asm__ volatile (
+ "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm1 \n\t"
+ "tilezero %%tmm2 \n\t"
+ "tilezero %%tmm3 \n\t"
+ "tilezero %%tmm4 \n\t"
+ "top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
+ "top4busd %%zmm0, %%zmm1, %%tmm1 \n\t"
+ "top4bssd %%zmm0, %%zmm1, %%tmm2 \n\t"
+ "top4bsud %%zmm0, %%zmm1, %%tmm3 \n\t"
+ "top2bf16ps %%zmm0, %%zmm1, %%tmm4 \n\t"
+ ::: "memory", "tmm0", "tmm1", "tmm2", "tmm3", "tmm4", "zmm0", "zmm1"
+ );
+}
diff --git a/clang/test/CodeGen/X86/acev1_mxfp8.c b/clang/test/CodeGen/X86/acev1_mxfp8.c
new file mode 100644
index 00000000000000..e99ca9dda4067f
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_mxfp8.c
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 MX FP8 outer product macro-based intrinsics
+
+#include <immintrin.h>
+
+void test_top4mxhf8ps(void) {
+ // CHECK-LABEL: @test_top4mxhf8ps
+ // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 0)
+ _tile_top4mxhf8ps(0, 1, 2, 0);
+}
+
+void test_top4mxbhf8ps(void) {
+ // CHECK-LABEL: @test_top4mxbhf8ps
+ // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, i8 2, i8 3, i8 1)
+ _tile_top4mxbhf8ps(1, 2, 3, 1);
+}
+
+void test_top4mxhbf8ps(void) {
+ // CHECK-LABEL: @test_top4mxhbf8ps
+ // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, i8 3, i8 4, i8 0)
+ _tile_top4mxhbf8ps(2, 3, 4, 0);
+}
+
+void test_top4mxbf8ps(void) {
+ // CHECK-LABEL: @test_top4mxbf8ps
+ // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, i8 4, i8 5, i8 1)
+ _tile_top4mxbf8ps(3, 4, 5, 1);
+}
+
+void test_top4mxbssps(void) {
+ // CHECK-LABEL: @test_top4mxbssps
+ // CHECK: call void @llvm.x86.top4mxbssps(i8 4, i8 5, i8 6, i8 0)
+ _tile_top4mxbssps(4, 5, 6, 0);
+}
+
+// Test all MX FP8 variants in sequence
+void test_all_mxfp8_variants(void) {
+ // CHECK-LABEL: @test_all_mxfp8_variants
+ // CHECK: call void @llvm.x86.top4mxhf8ps
+ // CHECK: call void @llvm.x86.top4mxbhf8ps
+ // CHECK: call void @llvm.x86.top4mxhbf8ps
+ // CHECK: call void @llvm.x86.top4mxbf8ps
+ // CHECK: call void @llvm.x86.top4mxbssps
+ _tile_top4mxhf8ps(0, 1, 2, 0);
+ _tile_top4mxbhf8ps(3, 4, 5, 0);
+ _tile_top4mxhbf8ps(6, 7, 0, 0);
+ _tile_top4mxbf8ps(1, 2, 3, 1);
+ _tile_top4mxbssps(4, 5, 6, 1);
+}
diff --git a/clang/test/CodeGen/X86/acev1_tile_movement.c b/clang/test/CodeGen/X86/acev1_tile_movement.c
new file mode 100644
index 00000000000000..f1383104f5db23
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_tile_movement.c
@@ -0,0 +1,113 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f \
+// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 tile movement operations (ZMM <-> Tile row/col)
+// Note: ACE v1 does not have TILELOADD/TILESTORED - uses TILEMOVROW/TILEMOVCOL for data movement
+
+#include <immintrin.h>
+
+// Test tile setrow - write ZMM to tile row (ACE-specific)
+void test_tile_ace_setrow(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setrow
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __tile_ace_setrow(&dst, src, 0);
+}
+
+// Test tile setrow with different row indices
+void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setrow_indices
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __tile_ace_setrow(&dst, src, 0);
+ __tile_ace_setrow(&dst, src, 8);
+ __tile_ace_setrow(&dst, src, 15);
+}
+
+// Test tile setcol - write ZMM to tile column (ACE-specific)
+void test_tile_ace_setcol(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setcol
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __tile_ace_setcol(&dst, src, 0);
+}
+
+// Test tile setcol with different column indices
+void test_tile_ace_setcol_indices(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setcol_indices
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __tile_ace_setcol(&dst, src, 0);
+ __tile_ace_setcol(&dst, src, 8);
+}
+
+// Test tile getrow - read tile row to ZMM (ACE-specific)
+__m512i test_tile_ace_getrow(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_getrow
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ return __tile_ace_getrow(&src, 0);
+}
+
+// Test tile getrow with different row indices
+void test_tile_ace_getrow_indices(__acetile src, __m512i *out) {
+ // CHECK-LABEL: @test_tile_ace_getrow_indices
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ out[0] = __tile_ace_getrow(&src, 0);
+ out[1] = __tile_ace_getrow(&src, 8);
+ out[2] = __tile_ace_getrow(&src, 15);
+}
+
+// Integration test: initialize tile using ACE tile zero and setrow
+void test_tile_init_via_setrow(void) {
+ // CHECK-LABEL: @test_tile_init_via_setrow
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __acetile tile;
+ __tile_ace_zero(&tile);
+
+ __m512i zmm = _mm512_set1_epi32(0x12345678);
+ __tile_ace_setrow(&tile, zmm, 0);
+}
+
+// Test combined tile operations
+void test_tile_movement_combined(void) {
+ // CHECK-LABEL: @test_tile_movement_combined
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __acetile tile;
+ __tile_ace_zero(&tile);
+
+ __m512i zmm1 = _mm512_set1_epi32(1);
+ __m512i zmm2 = _mm512_set1_epi32(2);
+
+ // Set multiple rows
+ __tile_ace_setrow(&tile, zmm1, 0);
+ __tile_ace_setrow(&tile, zmm2, 1);
+
+ // Set columns
+ __tile_ace_setcol(&tile, zmm1, 0);
+}
+
+// Integration test: complete ACE tile workflow with data movement
+void test_ace_data_workflow(__m512i *input, __m512i *output) {
+ // CHECK-LABEL: @test_ace_data_workflow
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ __acetile tile;
+ __tile_ace_zero(&tile);
+
+ // Initialize first row with input
+ __tile_ace_setrow(&tile, input[0], 0);
+
+ // Perform computation
+ __tile_ace_top4buud(&tile, input[1], input[2]);
+
+ // Extract result row
+ output[0] = __tile_ace_getrow(&tile, 0);
+}
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index d8fd922a06ebd9..9824ffe76c0a3c 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -147,7 +147,7 @@ __attribute__((target("fpmath=387")))
void f_fpmath_387(void) {}
// CHECK-NOT: tune-cpu
-// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
+// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-acev1,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse2")))
void f_no_sse2(void) {}
@@ -155,7 +155,7 @@ void f_no_sse2(void) {}
__attribute__((target("sse4")))
void f_sse4(void) {}
-// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
+// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-acev1,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse4")))
void f_no_sse4(void) {}
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 90b785d3c0694e..636e995b3f6f1a 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -547,6 +547,13 @@
// NO-AMX-AVX512-NOT: #define __AMX_AVX512__ 1
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 -x c \
+// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 %s
+// ACEV1: #define __ACEV1__ 1
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mno-acev1 -x c \
+// RUN: -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+// NO-ACEV1-NOT: #define __ACEV1__ 1
+
// RUN: %clang -target i386-unknown-unknown -march=atom -mavxvnni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVXVNNI %s
// AVXVNNI: #define __AVX2__ 1
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 83214d1601d807..43a0885bd8bcd8 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5736,6 +5736,204 @@ let TargetPrefix = "x86" in {
[]>;
}
+//===----------------------------------------------------------------------===//
+// ACE (AI Compute Extensions) Intrinsics
+//===----------------------------------------------------------------------===//
+let TargetPrefix = "x86" in {
+ // BSRINIT - Initialize Block Scale Register
+ def int_x86_bsrinit : ClangBuiltin<"__builtin_ia32_bsrinit">,
+ Intrinsic<[], [], []>;
+
+ // BSRMOVF - Move Full to BSR
+ // Takes two ZMM registers as input (doubleword integers), writes to BSR (implicit)
+ def int_x86_bsrmovf : ClangBuiltin<"__builtin_ia32_bsrmovf">,
+ Intrinsic<[], [llvm_v16i32_ty, llvm_v16i32_ty], []>;
+
+ // BSRMOVH - Move Half to/from BSR
+ // Load variant: read from ZMM (doubleword integers), update BSR
+ def int_x86_bsrmovh_set : ClangBuiltin<"__builtin_ia32_bsrmovh_set">,
+ Intrinsic<[], [llvm_v16i32_ty], []>;
+ // Store variant: read from BSR, return ZMM (doubleword integers)
+ def int_x86_bsrmovh_get : ClangBuiltin<"__builtin_ia32_bsrmovh_get">,
+ Intrinsic<[llvm_v16i32_ty], [], []>;
+
+ // BSRMOVL - Move Low to/from BSR
+ // Load variant: read from ZMM (doubleword integers), write to BSR
+ def int_x86_bsrmovl_set : ClangBuiltin<"__builtin_ia32_bsrmovl_set">,
+ Intrinsic<[], [llvm_v16i32_ty], []>;
+ // Store variant: read from BSR, return ZMM (doubleword integers)
+ def int_x86_bsrmovl_get : ClangBuiltin<"__builtin_ia32_bsrmovl_get">,
+ Intrinsic<[llvm_v16i32_ty], [], []>;
+
+ // TILEMOVCOL - Move Column from Vector to Tile
+ // Single intrinsic handles both immediate and register index forms.
+ // Lowering checks if index is constant and selects appropriate instruction.
+ def int_x86_tilesetcol : ClangBuiltin<"__builtin_ia32_tilesetcol">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>]>;
+
+ // ACE TILEMOVROW - Move Row from Vector TO Tile (extends AMX TILEMOVROW)
+ // Note: AMX has int_x86_tilemovrow which reads FROM tile to vector
+ // ACE adds the reverse direction: writing TO tile from vector
+ // Single intrinsic handles both immediate and register index forms.
+ // Lowering checks if index is constant and selects appropriate instruction.
+ def int_x86_tilesetrow : ClangBuiltin<"__builtin_ia32_tilesetrow">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>]>;
+
+ // TOP2BF16PS - Outer Product BF16 to Single Precision
+ def int_x86_top2bf16ps : ClangBuiltin<"__builtin_ia32_top2bf16ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4 - Tile Outer Product (4x4 block operations)
+ // TOP4BUUD - Unsigned/Unsigned
+ def int_x86_top4buud : ClangBuiltin<"__builtin_ia32_top4buud">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4BUSD - Unsigned/Signed
+ def int_x86_top4busd : ClangBuiltin<"__builtin_ia32_top4busd">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4BSSD - Signed/Signed
+ def int_x86_top4bssd : ClangBuiltin<"__builtin_ia32_top4bssd">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4BSUD - Signed/Unsigned
+ def int_x86_top4bsud : ClangBuiltin<"__builtin_ia32_top4bsud">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4MX - Mixed Precision Outer Products
+ // TOP4MXHFBPS - FP16/BF16 to FP32
+ def int_x86_top4mxhf8ps : ClangBuiltin<"__builtin_ia32_top4mxhf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBHFBPS - BF16/FP16 to FP32
+ def int_x86_top4mxbhf8ps : ClangBuiltin<"__builtin_ia32_top4mxbhf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXHBFBPS - FP16/BF16/BF16 to FP32
+ def int_x86_top4mxhbf8ps : ClangBuiltin<"__builtin_ia32_top4mxhbf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBF8PS - MX FP8 BF8xBF8 to FP32
+ def int_x86_top4mxbf8ps : ClangBuiltin<"__builtin_ia32_top4mxbf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBSSPS - MX INT8 SxS to FP32 (BSR scaled)
+ def int_x86_top4mxbssps : ClangBuiltin<"__builtin_ia32_top4mxbssps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // ACE - internal intrinsics (for IR-level optimization)
+ // These take tile/ZMM values directly instead of immediate tile IDs
+
+ // TILEMOVCOL internal - takes dimensions and returns tile
+ def int_x86_tilesetcol_internal :
+ ClangBuiltin<"__builtin_ia32_tilesetcol_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
+ []>;
+
+ // ACE TILEMOVROW internal - takes dimensions, vector, index and returns tile
+ // Note: AMX has int_x86_tilemovrow_internal which extracts from tile to vector
+ // ACE adds the reverse: write vector to tile row
+ def int_x86_tilesetrow_internal :
+ ClangBuiltin<"__builtin_ia32_tilesetrow_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
+ []>;
+
+ // TOP2BF16PS internal - operates on BF16 data (32 BF16 elements per ZMM)
+ def int_x86_top2bf16ps_internal :
+ ClangBuiltin<"__builtin_ia32_top2bf16ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v32bf16_ty, llvm_v32bf16_ty], []>;
+
+ // TOP4BUUD internal - unsigned bytes (64 elements per ZMM)
+ def int_x86_top4buud_internal :
+ ClangBuiltin<"__builtin_ia32_top4buud_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4BUSD internal - unsigned/signed bytes (64 elements per ZMM)
+ def int_x86_top4busd_internal :
+ ClangBuiltin<"__builtin_ia32_top4busd_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4BSSD internal - signed bytes (64 elements per ZMM)
+ def int_x86_top4bssd_internal :
+ ClangBuiltin<"__builtin_ia32_top4bssd_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4BSUD internal - signed/unsigned bytes (64 elements per ZMM)
+ def int_x86_top4bsud_internal :
+ ClangBuiltin<"__builtin_ia32_top4bsud_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4MXHFBPS internal - 4th arg is BSR index (i8 immediate)
+ // FP8 data packed in dwords (16 dwords per ZMM)
+ def int_x86_top4mxhf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxhf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBHFBPS internal - FP8 data packed in dwords
+ def int_x86_top4mxbhf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxbhf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXHBFBPS internal - FP8 data packed in dwords
+ def int_x86_top4mxhbf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxhbf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBF8PS internal - MX FP8 BF8xBF8 data packed in dwords
+ def int_x86_top4mxbf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxbf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBSSPS internal - MX INT8 SxS to FP32 (BSR scaled)
+ def int_x86_top4mxbssps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxbssps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+}
+
//===----------------------------------------------------------------------===//
let TargetPrefix = "x86" in {
// CMPCCXADD
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index 2cf0a1feab13f2..a1e8e444f45460 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -264,6 +264,7 @@ X86_FEATURE (NDD, "ndd")
X86_FEATURE (EGPR, "egpr")
X86_FEATURE (ZU, "zu")
X86_FEATURE (JMPABS, "jmpabs")
+X86_FEATURE (ACEV1, "acev1")
// These features aren't really CPU features, but the frontend can set them.
X86_FEATURE (RETPOLINE_EXTERNAL_THUNK, "retpoline-external-thunk")
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 2173a2b4b5e23b..255a2a70aea103 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -3975,6 +3975,56 @@ bool X86AsmParser::parseInstruction(ParseInstructionInfo &Info, StringRef Name,
}
}
+ // ACE BSR instructions: Strip %bsr0 operands and distinguish load/store.
+ // BSR (Block Scale Register) is implicit in the encoding (always BSR0),
+ // but the spec requires explicit %bsr0 in assembly syntax for disambiguation.
+ //
+ // For BSRMOVH and BSRMOVL, load vs store is determined by operand order:
+ // - ATT load: zmm/mem, bsr0 (ZMM first, BSR last) - move TO bsr0
+ // - ATT store: bsr0, zmm/mem (BSR first, ZMM last) - move FROM bsr0
+ // - Intel syntax reverses source/dest from ATT
+ //
+ // Note: Check for specific ACE BSR instructions to avoid matching standard
+ // x86 "bsrl"/"bsrq" (bit scan reverse).
+ if (Name == "bsrinit" || Name == "bsrmovf" || Name == "bsrmovh" ||
+ Name == "bsrmovl") {
+ // Single pass: track positions and filter operands simultaneously
+ int FirstBsrPos = -1;
+ int FirstNonBsrPos = -1;
+ SmallVector<std::unique_ptr<MCParsedAsmOperand>, 4> NewOperands;
+ NewOperands.push_back(std::move(Operands[0])); // Keep the mnemonic token
+
+ for (size_t I = 1; I < Operands.size(); ++I) {
+ X86Operand &Op = static_cast<X86Operand &>(*Operands[I]);
+ if (Op.isReg() && Op.getReg() == X86::BSR0) {
+ if (FirstBsrPos < 0)
+ FirstBsrPos = I;
+ // Skip BSR operands (don't add to NewOperands)
+ } else {
+ if (FirstNonBsrPos < 0)
+ FirstNonBsrPos = I;
+ NewOperands.push_back(std::move(Operands[I]));
+ }
+ }
+
+ // BSR instructions require explicit %bsr0 operand
+ if (FirstBsrPos < 0)
+ return Error(NameLoc, "BSR instruction requires explicit %bsr0 operand");
+
+ Operands = std::move(NewOperands);
+
+ // For BSRMOVH and BSRMOVL, detect set vs get by operand order
+ if (Name == "bsrmovh" || Name == "bsrmovl") {
+ // Intel: BSR first = set (ZMM→BSR); ATT: non-BSR first = set
+ bool IsLoad = isParsingIntelSyntax() ? (FirstBsrPos < FirstNonBsrPos)
+ : (FirstNonBsrPos < FirstBsrPos);
+ StringRef NewMnemonic =
+ IsLoad ? (Name == "bsrmovh" ? "bsrmovh_set" : "bsrmovl_set")
+ : (Name == "bsrmovh" ? "bsrmovh_get" : "bsrmovl_get");
+ static_cast<X86Operand &>(*Operands[0]).setTokenValue(NewMnemonic);
+ }
+ }
+
if (Flags)
Operands.push_back(X86Operand::CreatePrefix(Flags, NameLoc, NameLoc));
return false;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
index 564636959f00fe..e310276f4d75f8 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
@@ -84,8 +84,9 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, uint64_t Address,
STI.hasFeature(X86::Is16Bit)) {
OS << "\tdata32";
}
- // Try to print any aliases first.
- else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
+ // Try to print any aliases first, then BSR instructions, then compare instrs.
+ else if (!printAliasInstr(MI, Address, OS) && !printBSRInstr(MI, OS) &&
+ !printVecCompareInstr(MI, OS))
printInstruction(MI, Address, OS);
// Next always print the annotation.
@@ -402,6 +403,93 @@ bool X86ATTInstPrinter::printVecCompareInstr(const MCInst *MI,
return false;
}
+bool X86ATTInstPrinter::printBSRInstr(const MCInst *MI, raw_ostream &OS) {
+ // ACE BSR instructions: Print with explicit %bsr0 operand as per spec.
+ // BSR is implicit in the encoding, but the spec requires explicit syntax.
+ if (!MI)
+ return false;
+ unsigned Opcode = MI->getOpcode();
+
+ switch (Opcode) {
+ default:
+ return false;
+
+ case X86::BSRINIT:
+ OS << "\tbsrinit\t%bsr0";
+ return true;
+
+ case X86::BSRMOVFrr:
+ // Syntax: bsrmovf %zmm2, %zmm1, %bsr0
+ OS << "\tbsrmovf\t";
+ printOperand(MI, 1, OS); // src2
+ OS << ", ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVFrm:
+ // Syntax: bsrmovf mem, %zmm1, %bsr0
+ OS << "\tbsrmovf\t";
+ printMemReference(MI, 1, OS); // src2 (memory)
+ OS << ", ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVHrr_set:
+ // Syntax: bsrmovh %zmm1, %bsr0 (set BSR high from zmm)
+ OS << "\tbsrmovh\t";
+ printOperand(MI, 0, OS); // src
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVHrm_set:
+ // Syntax: bsrmovh mem, %bsr0 (set BSR high from memory)
+ OS << "\tbsrmovh\t";
+ printMemReference(MI, 0, OS); // src (memory)
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVHrr_get:
+ // Syntax: bsrmovh %bsr0, %zmm1 (get BSR high to zmm)
+ OS << "\tbsrmovh\t%bsr0, ";
+ printOperand(MI, 0, OS); // dst
+ return true;
+
+ case X86::BSRMOVHmr_get:
+ // Syntax: bsrmovh %bsr0, mem
+ OS << "\tbsrmovh\t%bsr0, ";
+ printMemReference(MI, 0, OS); // dst (memory)
+ return true;
+
+ case X86::BSRMOVLrr_set:
+ // Syntax: bsrmovl %zmm2, %bsr0 (set BSR low from zmm)
+ OS << "\tbsrmovl\t";
+ printOperand(MI, 0, OS); // src
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVLrm_set:
+ // Syntax: bsrmovl mem, %bsr0
+ OS << "\tbsrmovl\t";
+ printMemReference(MI, 0, OS); // src (memory)
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVLrr_get:
+ // Syntax: bsrmovl %bsr0, %zmm3 (get BSR low to zmm)
+ OS << "\tbsrmovl\t%bsr0, ";
+ printOperand(MI, 0, OS); // dst
+ return true;
+
+ case X86::BSRMOVLmr_get:
+ // Syntax: bsrmovl %bsr0, mem
+ OS << "\tbsrmovl\t%bsr0, ";
+ printMemReference(MI, 0, OS); // dst (memory)
+ return true;
+ }
+}
+
void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
index 1452622ebcea84..f28024a50a78c0 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
@@ -28,6 +28,7 @@ class X86ATTInstPrinter final : public X86InstPrinterCommon {
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
+ bool printBSRInstr(const MCInst *MI, raw_ostream &OS);
// Autogenerated by tblgen, returns true if we successfully printed an
// alias.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
index ff27005f52ea8f..def7784ea7b5f8 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
@@ -46,7 +46,8 @@ void X86IntelInstPrinter::printInst(const MCInst *MI, uint64_t Address,
if (MI->getOpcode() == X86::DATA16_PREFIX &&
STI.hasFeature(X86::Is16Bit)) {
OS << "\tdata32";
- } else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
+ } else if (!printAliasInstr(MI, Address, OS) && !printBSRInstr(MI, OS) &&
+ !printVecCompareInstr(MI, OS))
printInstruction(MI, Address, OS);
// Next always print the annotation.
@@ -363,6 +364,91 @@ bool X86IntelInstPrinter::printVecCompareInstr(const MCInst *MI, raw_ostream &OS
return false;
}
+bool X86IntelInstPrinter::printBSRInstr(const MCInst *MI, raw_ostream &OS) {
+ // ACE BSR instructions: Print with explicit bsr0 operand as per spec.
+ // BSR is implicit in the encoding, but the spec requires explicit syntax.
+ if (!MI)
+ return false;
+ unsigned Opcode = MI->getOpcode();
+
+ switch (Opcode) {
+ default:
+ return false;
+
+ case X86::BSRINIT:
+ OS << "\tbsrinit\tbsr0";
+ return true;
+
+ case X86::BSRMOVFrr:
+ // Intel Syntax: bsrmovf bsr0, zmm1, zmm2
+ OS << "\tbsrmovf\tbsr0, ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", ";
+ printOperand(MI, 1, OS); // src2
+ return true;
+
+ case X86::BSRMOVFrm:
+ // Intel Syntax: bsrmovf bsr0, zmm1, zmmword ptr [mem]
+ OS << "\tbsrmovf\tbsr0, ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", ";
+ printzmmwordmem(MI, 1, OS); // src2 (memory)
+ return true;
+
+ case X86::BSRMOVHrr_set:
+ // Intel Syntax: bsrmovh bsr0, zmm1 (set BSR high from zmm)
+ OS << "\tbsrmovh\tbsr0, ";
+ printOperand(MI, 0, OS); // src
+ return true;
+
+ case X86::BSRMOVHrm_set:
+ // Intel Syntax: bsrmovh bsr0, zmmword ptr [mem] (set BSR high from memory)
+ OS << "\tbsrmovh\tbsr0, ";
+ printzmmwordmem(MI, 0, OS); // src (memory)
+ return true;
+
+ case X86::BSRMOVHrr_get:
+ // Intel Syntax: bsrmovh zmm1, bsr0 (get BSR high to zmm)
+ OS << "\tbsrmovh\t";
+ printOperand(MI, 0, OS); // dst
+ OS << ", bsr0";
+ return true;
+
+ case X86::BSRMOVHmr_get:
+ // Intel Syntax: bsrmovh zmmword ptr [mem], bsr0
+ OS << "\tbsrmovh\t";
+ printzmmwordmem(MI, 0, OS); // dst (memory)
+ OS << ", bsr0";
+ return true;
+
+ case X86::BSRMOVLrr_set:
+ // Intel Syntax: bsrmovl bsr0, zmm2 (set BSR low from zmm)
+ OS << "\tbsrmovl\tbsr0, ";
+ printOperand(MI, 0, OS); // src
+ return true;
+
+ case X86::BSRMOVLrm_set:
+ // Intel Syntax: bsrmovl bsr0, zmmword ptr [mem]
+ OS << "\tbsrmovl\tbsr0, ";
+ printzmmwordmem(MI, 0, OS); // src (memory)
+ return true;
+
+ case X86::BSRMOVLrr_get:
+ // Intel Syntax: bsrmovl zmm3, bsr0 (get BSR low to zmm)
+ OS << "\tbsrmovl\t";
+ printOperand(MI, 0, OS); // dst
+ OS << ", bsr0";
+ return true;
+
+ case X86::BSRMOVLmr_get:
+ // Intel Syntax: bsrmovl zmmword ptr [mem], bsr0
+ OS << "\tbsrmovl\t";
+ printzmmwordmem(MI, 0, OS); // dst (memory)
+ OS << ", bsr0";
+ return true;
+ }
+}
+
void X86IntelInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
index 324c56c1329da1..85830cc0c02cb0 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
@@ -28,6 +28,7 @@ class X86IntelInstPrinter final : public X86InstPrinterCommon {
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
+ bool printBSRInstr(const MCInst *MI, raw_ostream &OS);
// Autogenerated by tblgen, returns true if we successfully printed an
// alias.
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index b24cdd737024be..6d1133a0d5d397 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -296,6 +296,9 @@ def FeatureAMXAVX512 : SubtargetFeature<"amx-avx512",
"HasAMXAVX512", "true",
"Support AMX-AVX512 instructions",
[FeatureAMXTILE]>;
+def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
+ "Support ACEV1 (AI Compute Extensions v1) instructions",
+ [FeatureAMXAVX512]>;
def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true",
"Support CMPCCXADD instructions">;
def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true",
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 75bed8fb1e50a6..478f750e662ace 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -714,6 +714,95 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
MI.tieOperands(0, 1);
return true;
}
+ // ACE internal pseudo instructions
+ // Pattern: Remove dimension operands (1-3), change opcode, re-tie operands
+ case X86::PTOP2BF16PSV:
+ case X86::PTOP4BUUDV:
+ case X86::PTOP4BUSDV:
+ case X86::PTOP4BSSDV:
+ case X86::PTOP4BSUDV: {
+ MI.untieRegOperand(4);
+ for (unsigned i = 3; i > 0; --i)
+ MI.removeOperand(i);
+ unsigned Opc = 0;
+ switch (Opcode) {
+ case X86::PTOP2BF16PSV:
+ Opc = X86::TOP2BF16PSrrr;
+ break;
+ case X86::PTOP4BUUDV:
+ Opc = X86::TOP4BUUDrrr;
+ break;
+ case X86::PTOP4BUSDV:
+ Opc = X86::TOP4BUSDrrr;
+ break;
+ case X86::PTOP4BSSDV:
+ Opc = X86::TOP4BSSDrrr;
+ break;
+ case X86::PTOP4BSUDV:
+ Opc = X86::TOP4BSUDrrr;
+ break;
+ default:
+ llvm_unreachable("Unexpected ACE opcode");
+ }
+ MI.setDesc(TII->get(Opc));
+ MI.tieOperands(0, 1);
+ return true;
+ }
+ // ACE TOP4MX internal pseudo instructions - have 4 dimension operands (row,
+ // col, k, imm)
+ case X86::PTOP4MXHF8PSV:
+ case X86::PTOP4MXBHF8PSV:
+ case X86::PTOP4MXHBF8PSV:
+ case X86::PTOP4MXBF8PSV:
+ case X86::PTOP4MXBSSPSV: {
+ MI.untieRegOperand(5);
+ // Remove dimension operands 1-3, keep imm at position 4
+ for (unsigned i = 3; i > 0; --i)
+ MI.removeOperand(i);
+ // Now operand layout is: dst, imm, src1(acc), src2, src3
+ // Real instruction expects: dst, src1(acc), src2, src3, imm
+ // Need to move imm from position 1 to end
+ unsigned ImmVal = MI.getOperand(1).getImm();
+ MI.removeOperand(1);
+ unsigned Opc = 0;
+ switch (Opcode) {
+ case X86::PTOP4MXHF8PSV:
+ Opc = X86::TOP4MXHF8PSrrri;
+ break;
+ case X86::PTOP4MXBHF8PSV:
+ Opc = X86::TOP4MXBHF8PSrrri;
+ break;
+ case X86::PTOP4MXHBF8PSV:
+ Opc = X86::TOP4MXHBF8PSrrri;
+ break;
+ case X86::PTOP4MXBF8PSV:
+ Opc = X86::TOP4MXBF8PSrrri;
+ break;
+ case X86::PTOP4MXBSSPSV:
+ Opc = X86::TOP4MXBSSPSrrri;
+ break;
+ default:
+ llvm_unreachable("Unexpected ACE TOP4MX opcode");
+ }
+ MI.setDesc(TII->get(Opc));
+ MI.addOperand(MachineOperand::CreateImm(ImmVal));
+ MI.tieOperands(0, 1);
+ return true;
+ }
+ // ACE TILEMOV internal pseudo instructions - output TILE, remove dimension
+ // operands
+ case X86::PTILEMOVCOLV:
+ case X86::PTILEMOVROWV: {
+ // Operand layout: dst, row, col, src(VR512), idx(GR32)
+ // Remove row, col (operands 1, 2)
+ for (unsigned i = 2; i > 0; --i)
+ MI.removeOperand(i);
+ // Now layout: dst, src, idx - matches real instruction
+ unsigned Opc =
+ (Opcode == X86::PTILEMOVCOLV) ? X86::TILEMOVCOLrr : X86::TILEMOVROWrr;
+ MI.setDesc(TII->get(Opc));
+ return true;
+ }
case X86::PTILESTOREDV: {
for (int i = 1; i >= 0; --i)
MI.removeOperand(i);
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 0312ba39c3cff5..a924360caad7ee 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -210,8 +210,10 @@ void X86FastPreTileConfigImpl::InitializeTileConfigStackSpace() {
.addReg(Xmm);
}
// Fill in the palette first.
+ // Use Palette 2 for ACE v1, Palette 1 for AMX.
+ int PaletteId = ST->hasACEV1() ? 2 : 1;
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), CfgSS)
- .addImm(1);
+ .addImm(PaletteId);
}
/// Insert spill instruction for \p AssignedReg before \p Before.
@@ -223,9 +225,33 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');
const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
- // Don't need shape information for tile store, becasue it is adjacent to
- // the tile def instruction.
- TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, Register());
+
+ // ACE v1 doesn't have TILESTORED - use TILEMOVROW row-by-row
+ if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ const DebugLoc &DL = Before->getDebugLoc();
+ Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+
+ const unsigned NumRows = 16;
+ const unsigned RowSize = 64;
+
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // TILEMOVROW zmm, tmm, imm8 (read from tile)
+ BuildMI(*MBB, Before, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
+ .addReg(VirtReg)
+ .addImm(Row);
+
+ // VMOVUPS [stack + row*64], zmm
+ MachineInstrBuilder MIB =
+ BuildMI(*MBB, Before, DL, TII->get(X86::VMOVUPSZmr));
+ addFrameReference(MIB, FI, Row * RowSize);
+ MIB.addReg(ScratchZMM);
+ }
+ } else {
+ // AMX: Use storeRegToStackSlot (TILESTORED)
+ // Don't need shape information for tile store, because it is adjacent to
+ // the tile def instruction.
+ TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, Register());
+ }
++NumStores;
// TODO: update DBG_VALUEs
@@ -238,6 +264,7 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
int FI = getStackSpaceFor(OrigReg);
const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg);
Register TileReg;
+
// Fold copy to tileload
// BB1:
// spill src to s
@@ -250,24 +277,49 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
TileReg = UseMI->getOperand(0).getReg();
else
TileReg = MRI->createVirtualRegister(&RC);
- // Can't use TII->loadRegFromStackSlot(), because we need the shape
- // information for reload.
- // tileloadd (%sp, %idx), %tmm
- unsigned Opc = X86::PTILELOADDV;
- Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
- // FIXME: MBB is not the parent of UseMI.
- MachineInstr *NewMI = BuildMI(*UseMI->getParent(), UseMI, DebugLoc(),
- TII->get(X86::MOV64ri), StrideReg)
- .addImm(64);
- NewMI = addFrameReference(
- BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg)
- .addReg(RowMO->getReg())
- .addReg(ColMO->getReg()),
- FI);
- MachineOperand &MO = NewMI->getOperand(5);
- MO.setReg(StrideReg);
- RowMO->setIsKill(false);
- ColMO->setIsKill(false);
+
+ // ACE v1 doesn't have TILELOADD - use TILEMOVROW row-by-row
+ if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ const DebugLoc &DL = UseMI->getDebugLoc();
+ Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+
+ const unsigned NumRows = 16;
+ const unsigned RowSize = 64;
+
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // VMOVUPS zmm, [stack + row*64]
+ MachineInstrBuilder MIB = BuildMI(*UseMI->getParent(), UseMI, DL,
+ TII->get(X86::VMOVUPSZrm), ScratchZMM);
+ addFrameReference(MIB, FI, Row * RowSize);
+
+ // TILEMOVROW tmm, zmm, imm8 (write to tile)
+ BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWri),
+ TileReg)
+ .addReg(ScratchZMM)
+ .addImm(Row);
+ }
+ } else {
+ // AMX: Use PTILELOADDV
+ // Can't use TII->loadRegFromStackSlot(), because we need the shape
+ // information for reload.
+ // tileloadd (%sp, %idx), %tmm
+ unsigned Opc = X86::PTILELOADDV;
+ Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
+ // FIXME: MBB is not the parent of UseMI.
+ MachineInstr *NewMI = BuildMI(*UseMI->getParent(), UseMI, DebugLoc(),
+ TII->get(X86::MOV64ri), StrideReg)
+ .addImm(64);
+ NewMI = addFrameReference(
+ BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg)
+ .addReg(RowMO->getReg())
+ .addReg(ColMO->getReg()),
+ FI);
+ MachineOperand &MO = NewMI->getOperand(5);
+ MO.setReg(StrideReg);
+ RowMO->setIsKill(false);
+ ColMO->setIsKill(false);
+ }
+
// Erase copy instruction after it is folded.
if (UseMI->isCopy()) {
UseMI->eraseFromParent();
@@ -685,8 +737,9 @@ bool X86FastPreTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
bool X86FastPreTileConfigImpl::runOnMachineFunction(MachineFunction &MFunc) {
X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
MF = &MFunc;
diff --git a/llvm/lib/Target/X86/X86FastTileConfig.cpp b/llvm/lib/Target/X86/X86FastTileConfig.cpp
index f9bf5575dd44ae..f6ee4313174bb0 100644
--- a/llvm/lib/Target/X86/X86FastTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastTileConfig.cpp
@@ -42,6 +42,7 @@ class X86FastTileConfigImpl {
private:
// context
MachineFunction *MF = nullptr;
+ const X86Subtarget *ST = nullptr;
const TargetInstrInfo *TII = nullptr;
MachineRegisterInfo *MRI = nullptr;
const TargetRegisterInfo *TRI = nullptr;
@@ -131,6 +132,13 @@ bool X86FastTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
} else { // PLDTILECFGV
// Rewrite the shape information to memory. Stack slot should have
// been initialized to zero in pre config.
+ // ACE Palette 2 uses fixed tile dimensions (16x64), so skip writing
+ // shapes - bytes 1-63 must remain zero for ACE.
+ if (ST->hasACEV1()) {
+ ShapeInfos.clear();
+ Change = true;
+ continue;
+ }
int SS = MI.getOperand(0).getIndex(); // tile config stack slot.
for (auto &ShapeInfo : ShapeInfos) {
DebugLoc DL;
@@ -176,15 +184,16 @@ bool X86FastTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
bool X86FastTileConfigImpl::runOnMachineFunction(MachineFunction &MFunc) {
X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
MF = &MFunc;
MRI = &MFunc.getRegInfo();
- const TargetSubtargetInfo *ST = &MFunc.getSubtarget<X86Subtarget>();
+ ST = &MFunc.getSubtarget<X86Subtarget>();
TRI = ST->getRegisterInfo();
- TII = MFunc.getSubtarget().getInstrInfo();
+ TII = ST->getInstrInfo();
bool Change = false;
// Loop over all of the basic blocks, eliminating virtual register references
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 78205b8f395c4f..5e5e4fef0a39ec 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2942,8 +2942,9 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
}
}
- // Emit tilerelease for AMX kernel.
- if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA)
+ // Emit tilerelease for AMX/ACE kernel.
+ if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA ||
+ X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA)
BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
if (NeedsWin64CFI && MF.hasWinCFI())
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 8b995885d23a28..c4f747147b2fcb 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -5778,6 +5778,25 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
ReplaceNode(Node, CNode);
return;
}
+ // AMX-AVX512 internal intrinsics - set program model for struct-based API
+ case Intrinsic::x86_tcvtrowd2ps_internal:
+ case Intrinsic::x86_tcvtrowps2bf16h_internal:
+ case Intrinsic::x86_tcvtrowps2bf16l_internal:
+ case Intrinsic::x86_tcvtrowps2phh_internal:
+ case Intrinsic::x86_tcvtrowps2phl_internal:
+ case Intrinsic::x86_tilemovrow_internal: {
+ if (!Subtarget->hasAMXAVX512())
+ break;
+ auto *MFI =
+ CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
+ // Set program model for struct-based API (ManagedRA)
+ if (Subtarget->hasACEV1())
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
+ else
+ MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ // Let pattern matching handle the rest
+ break;
+ }
}
break;
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 4d081b7ecbf43d..c53f9a4c60aa26 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2700,7 +2700,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
}
- if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
+ if (!Subtarget.useSoftFloat() &&
+ (Subtarget.hasAMXTILE() || Subtarget.hasACEV1())) {
addRegisterClass(MVT::x86amx, &X86::TILERegClass);
}
@@ -28773,6 +28774,42 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
Chain = LockArith.getValue(1);
return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL);
}
+ // ACE Tile Movement Intrinsics - handle both immediate and register index
+ // forms using a single intrinsic. Check if index is constant to select
+ // the appropriate instruction form.
+ case Intrinsic::x86_tilesetrow:
+ case Intrinsic::x86_tilesetcol: {
+ SDLoc DL(Op);
+ SDValue Chain = Op.getOperand(0);
+ unsigned TileID = Op.getConstantOperandVal(2);
+ SDValue Src = Op.getOperand(3);
+ SDValue Idx = Op.getOperand(4);
+
+ unsigned PseudoImm, PseudoReg;
+ if (IntNo == Intrinsic::x86_tilesetrow) {
+ PseudoImm = X86::PTILEMOVROW;
+ PseudoReg = X86::PTILEMOVROW_REG;
+ } else {
+ PseudoImm = X86::PTILEMOVCOL;
+ PseudoReg = X86::PTILEMOVCOL_REG;
+ }
+
+ MachineSDNode *Node;
+ // Check if index is a compile-time constant
+ if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
+ // Use immediate form
+ Node = DAG.getMachineNode(
+ PseudoImm, DL, MVT::Other,
+ {DAG.getTargetConstant(TileID, DL, MVT::i8), Src,
+ DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i8), Chain});
+ } else {
+ // Use register form
+ Node = DAG.getMachineNode(
+ PseudoReg, DL, MVT::Other,
+ {DAG.getTargetConstant(TileID, DL, MVT::i8), Src, Idx, Chain});
+ }
+ return SDValue(Node, 0);
+ }
}
return SDValue();
}
@@ -39090,6 +39127,106 @@ X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
return BB;
}
+// Helper enum and table for centralized tile program model handling
+namespace {
+enum class TileProgModelType {
+ None, // Not a tile instruction
+ ACE_DirectReg, // ACE-only, always ACE_DirectReg
+ ACE_ManagedRA, // ACE-only, always ACE_ManagedRA
+ AMX_DirectReg, // AMX/ACE, DirectReg (checks subtarget)
+ AMX_ManagedRA, // AMX/ACE, ManagedRA (checks subtarget)
+};
+
+struct TileOpcodeModelEntry {
+ unsigned Opcode;
+ TileProgModelType ModelType;
+};
+
+// Centralized table mapping tile opcodes to their program model type
+static const TileOpcodeModelEntry TileOpcodeModels[] = {
+ // AMX/ACE DirectReg (macro API) - checks subtarget for ACE vs AMX
+ {X86::PTILEZERO, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Hrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Lrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHHrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHLrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWD2PSrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTILEMOVROWrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Hrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Lrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHHrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHLrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWD2PSrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTILEMOVROWrte, TileProgModelType::AMX_DirectReg},
+
+ // AMX/ACE ManagedRA (struct API) - checks subtarget for ACE vs AMX
+ {X86::PTILEZEROV, TileProgModelType::AMX_ManagedRA},
+
+ // ACE-only DirectReg (macro API)
+ {X86::PTILEMOVCOL, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVCOL_REG, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVROW, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVROW_REG, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP2BF16PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BUUD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BUSD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BSSD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BSUD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXHF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBHF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXHBF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBSSPS, TileProgModelType::ACE_DirectReg},
+
+ // ACE-only ManagedRA (struct API)
+ {X86::PTILEMOVCOLV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTILEMOVROWV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP2BF16PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BUUDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BUSDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BSSDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BSUDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXHF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBHF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXHBF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBSSPSV, TileProgModelType::ACE_ManagedRA},
+};
+
+// Set tile program model based on opcode. Returns true if model was set.
+static bool setTileProgModelForOpcode(unsigned Opcode,
+ X86MachineFunctionInfo *MFI,
+ const X86Subtarget &Subtarget) {
+ for (const auto &Entry : TileOpcodeModels) {
+ if (Entry.Opcode == Opcode) {
+ switch (Entry.ModelType) {
+ case TileProgModelType::None:
+ return false;
+ case TileProgModelType::ACE_DirectReg:
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_DirectReg);
+ return true;
+ case TileProgModelType::ACE_ManagedRA:
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
+ return true;
+ case TileProgModelType::AMX_DirectReg:
+ if (Subtarget.hasACEV1())
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_DirectReg);
+ else
+ MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
+ return true;
+ case TileProgModelType::AMX_ManagedRA:
+ if (Subtarget.hasACEV1())
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
+ else
+ MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ return true;
+ }
+ }
+ }
+ return false;
+}
+} // anonymous namespace
+
MachineBasicBlock *
X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *BB) const {
@@ -39097,6 +39234,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
const TargetInstrInfo *TII = Subtarget.getInstrInfo();
const MIMetadata MIMD(MI);
+ // Centralized tile program model handling
+ auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
+ setTileProgModelForOpcode(MI.getOpcode(), MFI, Subtarget);
+
auto TMMImmToTMMReg = [](unsigned Imm) {
assert (Imm < 8 && "Illegal tmm index");
return X86::TMM0 + Imm;
@@ -39518,16 +39659,23 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTILEZERO: {
unsigned Imm = MI.getOperand(0).getImm();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
- MI.eraseFromParent(); // The pseudo is gone now.
- auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
- MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
+ MI.eraseFromParent();
return BB;
}
- case X86::PTILEZEROV: {
- auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
- MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ case X86::PTILEZEROV:
+ case X86::PTILEMOVCOLV:
+ case X86::PTILEMOVROWV:
+ case X86::PTOP2BF16PSV:
+ case X86::PTOP4BUUDV:
+ case X86::PTOP4BUSDV:
+ case X86::PTOP4BSSDV:
+ case X86::PTOP4BSUDV:
+ case X86::PTOP4MXHF8PSV:
+ case X86::PTOP4MXBHF8PSV:
+ case X86::PTOP4MXHBF8PSV:
+ case X86::PTOP4MXBF8PSV:
+ case X86::PTOP4MXBSSPSV:
return BB;
- }
case X86::PTILELOADDRS:
case X86::PTILELOADDRST1:
case X86::PTILELOADD:
@@ -39650,6 +39798,132 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MI.eraseFromParent(); // The pseudo is gone now.
return BB;
}
+ // ACE pseudo instruction expansion
+ case X86::PBSRINIT: {
+ BuildMI(*BB, MI, MIMD, TII->get(X86::BSRINIT));
+ MI.eraseFromParent();
+ return BB;
+ }
+ // Note: BSRMOVF, BSRMOVH, BSRMOVL are handled directly by patterns on real
+ // instructions
+ case X86::PTILEMOVCOL: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ unsigned Idx = MI.getOperand(2).getImm();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLri), DstReg)
+ .addReg(SrcReg)
+ .addImm(Idx);
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTILEMOVCOL_REG: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLrr), DstReg)
+ .addReg(SrcReg)
+ .addReg(MI.getOperand(2).getReg());
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTILEMOVROW: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ unsigned Idx = MI.getOperand(2).getImm();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWri), DstReg)
+ .addReg(SrcReg)
+ .addImm(Idx);
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTILEMOVROW_REG: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWrr), DstReg)
+ .addReg(SrcReg)
+ .addReg(MI.getOperand(2).getReg());
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTOP2BF16PS:
+ case X86::PTOP4BUUD:
+ case X86::PTOP4BUSD:
+ case X86::PTOP4BSSD:
+ case X86::PTOP4BSUD: {
+ unsigned Opc;
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case X86::PTOP2BF16PS:
+ Opc = X86::TOP2BF16PSrrr;
+ break;
+ case X86::PTOP4BUUD:
+ Opc = X86::TOP4BUUDrrr;
+ break;
+ case X86::PTOP4BUSD:
+ Opc = X86::TOP4BUSDrrr;
+ break;
+ case X86::PTOP4BSSD:
+ Opc = X86::TOP4BSSDrrr;
+ break;
+ case X86::PTOP4BSUD:
+ Opc = X86::TOP4BSUDrrr;
+ break;
+ }
+ // These instructions: TILE dst (rw), ZMM src1, ZMM src2
+ // The pseudo has: u8imm dst, u8imm src1, u8imm src2
+ // src1 and src2 are ZMM register indices (0-31)
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ unsigned Src1Imm = MI.getOperand(1).getImm();
+ unsigned Src2Imm = MI.getOperand(2).getImm();
+ assert(Src1Imm < 32 && "Illegal ZMM register index");
+ assert(Src2Imm < 32 && "Illegal ZMM register index");
+ BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
+ .addReg(DstReg, RegState::Undef)
+ .addReg(X86::ZMM0 + Src1Imm)
+ .addReg(X86::ZMM0 + Src2Imm);
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTOP4MXHF8PS:
+ case X86::PTOP4MXBHF8PS:
+ case X86::PTOP4MXHBF8PS:
+ case X86::PTOP4MXBF8PS:
+ case X86::PTOP4MXBSSPS: {
+ unsigned Opc;
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case X86::PTOP4MXHF8PS:
+ Opc = X86::TOP4MXHF8PSrrri;
+ break;
+ case X86::PTOP4MXBHF8PS:
+ Opc = X86::TOP4MXBHF8PSrrri;
+ break;
+ case X86::PTOP4MXHBF8PS:
+ Opc = X86::TOP4MXHBF8PSrrri;
+ break;
+ case X86::PTOP4MXBF8PS:
+ Opc = X86::TOP4MXBF8PSrrri;
+ break;
+ case X86::PTOP4MXBSSPS:
+ Opc = X86::TOP4MXBSSPSrrri;
+ break;
+ }
+ // These instructions: TILE dst (rw), ZMM src1, ZMM src2, imm8
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ unsigned Src1Imm = MI.getOperand(1).getImm();
+ unsigned Src2Imm = MI.getOperand(2).getImm();
+ unsigned Imm = MI.getOperand(3).getImm();
+ assert(Src1Imm < 32 && "Illegal ZMM register index");
+ assert(Src2Imm < 32 && "Illegal ZMM register index");
+ BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
+ .addReg(DstReg, RegState::Undef)
+ .addReg(X86::ZMM0 + Src1Imm)
+ .addReg(X86::ZMM0 + Src2Imm)
+ .addImm(Imm);
+ MI.eraseFromParent();
+ return BB;
+ }
}
}
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
new file mode 100644
index 00000000000000..f786f8ca345837
--- /dev/null
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -0,0 +1,417 @@
+//===---- X86InstrACE.td - ACE Instruction Set Extension --*- tablegen -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the instructions that make up the ACE (AI Compute
+// Extensions) instruction set.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// ACE instructions
+
+let Predicates = [HasACEV1, In64BitMode] in {
+
+// BSRINIT - Initialize Block Scale Register
+// VEX.128.F2.0F38.W1 49 11:000:000 - BSRINIT bsr0
+// BSR is implicit destination, ModRM = 11:000:000 (mod=11, reg=000, r/m=000)
+// Note: BSR0 is implicit (via Defs), not in operand list. Asm syntax has no explicit operand.
+let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
+ def BSRINIT : I<0x49, MRM_C0, (outs), (ins),
+ "bsrinit", []>,
+ VEX, XD, T8, REX_W;
+}
+
+// BSRMOVF - Move Full to BSR
+// EVEX.512.NP.MAP6.W1 95 11:000:bbb - BSRMOVF bsr0, zmm1, zmm2/m512
+// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1, ModRM.r/m encodes zmm2
+// BSR is implicit destination (via Defs)
+let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
+ def BSRMOVFrr : I<0x95, MRM0r, (outs),
+ (ins VR512:$src1, VR512:$src2),
+ "bsrmovf\t{$src2, $src1|$src1, $src2}",
+ [(int_x86_bsrmovf (v16i32 VR512:$src1), (v16i32 VR512:$src2))]>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, VVVV;
+
+ let mayLoad = 1 in
+ def BSRMOVFrm : I<0x95, MRM0m, (outs),
+ (ins VR512:$src1, f512mem:$src2),
+ "bsrmovf\t{$src2, $src1|$src1, $src2}", []>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>, VVVV;
+}
+
+// BSRMOVH - Move Half to/from BSR
+// EVEX.512.F2.MAP6.W1 95 /r - BSRMOVH bsr0, zmm1/m512, bsr0 (load, W1)
+// EVEX.512.F2.MAP6.W0 95 /r - BSRMOVH zmm1/m512, bsr0 (store, W0)
+// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
+// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
+// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+let SchedRW = [WriteVecLogic] in {
+ // Set BSR high half (W1): BSR is both input and output (read-modify-write)
+ let Defs = [BSR0], Uses = [BSR0] in {
+ def BSRMOVHrr_set : I<0x95, MRM0r, (outs),
+ (ins VR512:$src),
+ "bsrmovh_set\t$src",
+ [(int_x86_bsrmovh_set (v16i32 VR512:$src))]>,
+ EVEX, XD, T_MAP6, REX_W, EVEX_V512;
+
+ let mayLoad = 1 in
+ def BSRMOVHrm_set : I<0x95, MRM0m, (outs),
+ (ins f512mem:$src),
+ "bsrmovh_set\t$src", []>,
+ EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+
+ // Get BSR high half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ let Uses = [BSR0] in {
+ def BSRMOVHrr_get : I<0x95, MRM0r, (outs VR512:$dst),
+ (ins),
+ "bsrmovh_get\t$dst",
+ [(set (v16i32 VR512:$dst), (int_x86_bsrmovh_get))]>,
+ EVEX, XD, T_MAP6, EVEX_V512;
+
+ let mayStore = 1 in
+ def BSRMOVHmr_get : I<0x95, MRM0m, (outs),
+ (ins f512mem:$dst),
+ "bsrmovh_get\t$dst", []>,
+ EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+}
+
+// BSRMOVL - Move Low to/from BSR
+// EVEX.512.F3.MAP6.W1 95 /r - BSRMOVL bsr0, zmm1/m512 (load, W1)
+// EVEX.512.F3.MAP6.W0 95 /r - BSRMOVL zmm1/m512, bsr0 (store, W0)
+// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
+// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
+// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+let SchedRW = [WriteVecLogic] in {
+ // Set BSR low half (W1): BSR is both input and output (read-modify-write)
+ let Defs = [BSR0], Uses = [BSR0] in {
+ def BSRMOVLrr_set : I<0x95, MRM0r, (outs),
+ (ins VR512:$src),
+ "bsrmovl_set\t$src",
+ [(int_x86_bsrmovl_set (v16i32 VR512:$src))]>,
+ EVEX, XS, T_MAP6, REX_W, EVEX_V512;
+
+ let mayLoad = 1 in
+ def BSRMOVLrm_set : I<0x95, MRM0m, (outs),
+ (ins f512mem:$src),
+ "bsrmovl_set\t$src", []>,
+ EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+
+ // Get BSR low half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ let Uses = [BSR0] in {
+ def BSRMOVLrr_get : I<0x95, MRM0r, (outs VR512:$dst),
+ (ins),
+ "bsrmovl_get\t$dst",
+ [(set (v16i32 VR512:$dst), (int_x86_bsrmovl_get))]>,
+ EVEX, XS, T_MAP6, EVEX_V512;
+
+ let mayStore = 1 in
+ def BSRMOVLmr_get : I<0x95, MRM0m, (outs),
+ (ins f512mem:$dst),
+ "bsrmovl_get\t$dst", []>,
+ EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+}
+
+// TILEMOVCOL - Move Column from Vector to Tile
+// EVEX.512.66.0F3A.W1 2F /r ib - TILEMOVCOL tmm1, zmm2, imm8
+// EVEX.512.66.0F38.W1 4B /r - TILEMOVCOL tmm1, zmm2, r32
+// For r32 variant: GR32 is encoded in EVEX.vvvv (use MRMSrcReg4VOp3 + VVVV)
+let SchedRW = [WriteVecLogic] in {
+ def TILEMOVCOLri : Ii8<0x2F, MRMSrcReg, (outs TILE:$dst),
+ (ins VR512:$src, u8imm:$idx),
+ "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def TILEMOVCOLrr : I<0x4B, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins VR512:$src, GR32:$idx),
+ "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+}
+
+// TILEMOVROW - Move Row from Vector to Tile
+// EVEX.512.66.0F3A.W1 07 /r ib - TILEMOVROW tmm1, zmm2, imm8
+// EVEX.512.66.0F38.W1 4A /r - TILEMOVROW tmm1, zmm2, r32
+// For r32 variant: GR32 is encoded in EVEX.vvvv (use MRMSrcReg4VOp3 + VVVV)
+let SchedRW = [WriteVecLogic] in {
+ def TILEMOVROWri : Ii8<0x07, MRMSrcReg, (outs TILE:$dst),
+ (ins VR512:$src, u8imm:$idx),
+ "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def TILEMOVROWrr : I<0x4A, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins VR512:$src, GR32:$idx),
+ "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+}
+
+// Outer Product Instructions
+// TOP2BF16PS - Outer Product BF16 to Single Precision
+// EVEX.512.F3.0F38.W0 5C /r - TOP2BF16PS tmm1, zmm2, zmm3
+// Operand encoding from ACE spec page 52:
+// Operand 1: ModRM:reg (rw) = tmm1
+// Operand 2: ModRM:r/m (r) = zmm2
+// Operand 3: EVEX.vvvv (r) = zmm3
+// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+ def TOP2BF16PSrrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top2bf16ps\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, XS, T8, EVEX_V512;
+}
+
+// TOP4 Instructions - All use opcode 0x5E, differentiated by prefix
+// EVEX.512.NP.0F38.W0 5E /r - TOP4BUUD tmm1, zmm2, zmm3 (unsigned/unsigned)
+// EVEX.512.66.0F38.W0 5E /r - TOP4BUSD tmm1, zmm2, zmm3 (unsigned/signed)
+// EVEX.512.F2.0F38.W0 5E /r - TOP4BSSD tmm1, zmm2, zmm3 (signed/signed)
+// EVEX.512.F3.0F38.W0 5E /r - TOP4BSUD tmm1, zmm2, zmm3 (signed/unsigned)
+// Operand encoding from ACE spec page 54:
+// Operand 1: ModRM:reg (rw) = tmm1
+// Operand 2: ModRM:r/m (r) = zmm2
+// Operand 3: EVEX.vvvv (r) = zmm3
+// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+ def TOP4BUUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4buud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, T8, EVEX_V512;
+
+ def TOP4BUSDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4busd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, PD, T8, EVEX_V512;
+
+ def TOP4BSSDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4bssd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, XD, T8, EVEX_V512;
+
+ def TOP4BSUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4bsud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, XS, T8, EVEX_V512;
+}
+
+// TOP4MX Instructions - Mixed Precision Outer Products with Immediate
+// All use opcode 0x8D with different prefixes, or 0x8F
+// EVEX.512.NP.0F3A.W0 8D /r ib - TOP4MXBF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.66.0F3A.W0 8D /r ib - TOP4MXHF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.F2.0F3A.W0 8D /r ib - TOP4MXBHF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.F3.0F3A.W0 8D /r ib - TOP4MXHBF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.F2.0F3A.W0 8F /r ib - TOP4MXBSSPS tmm1, zmm2, zmm3, imm8
+// Operand encoding from ACE spec page 47:
+// Operand 1: ModRM:reg (rw) = tmm1
+// Operand 2: ModRM:r/m (r) = zmm2
+// Operand 3: EVEX.vvvv (r) = zmm3
+// Operand 4: imm8
+// MRMSrcReg encodes: src1→reg, src2→r/m, src3→vvvv (so swap to get spec order)
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+ def TOP4MXBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, TA, EVEX_V512;
+
+ def TOP4MXHF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, PD, TA, EVEX_V512;
+
+ def TOP4MXBHF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxbhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, XD, TA, EVEX_V512;
+
+ def TOP4MXHBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxhbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, XS, TA, EVEX_V512;
+
+ def TOP4MXBSSPSrrri : Ii8<0x8F, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxbssps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, XD, TA, EVEX_V512;
+}
+
+//===----------------------------------------------------------------------===//
+// ACE Pseudo Instructions
+// These are used by intrinsics and expanded via custom inserter
+//===----------------------------------------------------------------------===//
+
+// Pseudo instructions for user-facing intrinsics (immediate tile IDs)
+// These use usesCustomInserter to expand to real instructions in X86ISelLowering
+let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
+ // BSRINIT pseudo - no operands, just triggers BSR initialization
+ def PBSRINIT : PseudoI<(outs), (ins),
+ [(int_x86_bsrinit)]>;
+
+ // Note: BSRMOVF, BSRMOVH, BSRMOVL intrinsics are handled directly
+ // by patterns on the real instructions (BSRMOVFrr, BSRMOVHrr_set, etc.)
+
+ // TILEMOVCOL pseudo - immediate index variant
+ // dst: tile ID (imm), src: ZMM value (VR512), idx: column index (imm)
+ // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ def PTILEMOVCOL : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+
+ // TILEMOVCOL pseudo - register index variant
+ def PTILEMOVCOL_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+
+ // TILEMOVROW pseudo - immediate index variant (ACE: write to tile)
+ // dst: tile ID (imm), src: ZMM value (VR512), idx: row index (imm)
+ // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ def PTILEMOVROW : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+
+ // TILEMOVROW pseudo - register index variant (ACE: write to tile)
+ def PTILEMOVROW_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+
+ // TOP2BF16PS pseudo
+ def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top2bf16ps timm:$dst, timm:$src1, timm:$src2)]>;
+
+ // TOP4 pseudos - three tile operands (dst is also accumulator)
+ def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4buud timm:$dst, timm:$src1, timm:$src2)]>;
+
+ def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4busd timm:$dst, timm:$src1, timm:$src2)]>;
+
+ def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4bssd timm:$dst, timm:$src1, timm:$src2)]>;
+
+ def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4bsud timm:$dst, timm:$src1, timm:$src2)]>;
+
+ // TOP4MX pseudos - four operands (dst, src1, src2, imm)
+ def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxbhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxhbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxbssps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+}
+
+// Pseudo instructions for internal intrinsics (register allocation)
+// These are used for IR-level optimization and register allocation
+// usesCustomInserter is needed to set ACEProgModel in EmitInstrWithCustomInserter
+let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
+ // TILEMOVCOL internal - output is TILE, input is ZMM + index
+ def PTILEMOVCOLV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (int_x86_tilesetcol_internal GR16:$row, GR16:$col,
+ VR512:$src, GR32:$idx))]>;
+
+ // TILEMOVROW internal - output is TILE, input is ZMM + index
+ // ACE TILEMOVROW internal - write vector to tile row
+ def PTILEMOVROWV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (int_x86_tilesetrow_internal GR16:$row, GR16:$col,
+ VR512:$src, GR32:$idx))]>;
+
+ // TOP2BF16PS internal - output TILE, inputs: dimensions, TILE acc, two ZMMs
+ let Constraints = "$src1 = $dst" in
+ def PTOP2BF16PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top2bf16ps_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BUUD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BUUDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4buud_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BUSD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BUSDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4busd_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BSSD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BSSDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4bssd_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BSUD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BSUDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4bsud_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXHFBPS internal - imm is BSR index (u8imm matches timm in pattern)
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXHF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXBHFBPS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXBHF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxbhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXHBFBPS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXHBF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxhbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXBF8PS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXBF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXBSSPS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXBSSPSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxbssps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+}
+
+} // end HasACEV1, In64BitMode
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
index 418046479a8a09..e0aba511d6fb1d 100644
--- a/llvm/lib/Target/X86/X86InstrAMX.td
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -14,8 +14,9 @@
//===----------------------------------------------------------------------===//
// AMX instructions
-multiclass AMX_TILE_COMMON<string Suffix, Predicate HasEGPR> {
-let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
+// Tile configuration instructions - shared between AMX-TILE and ACE v1
+multiclass AMX_TILE_CONFIG<string Suffix, Predicate HasEGPR> {
+let Predicates = [HasAMXTILE_Or_ACEV1, HasEGPR, In64BitMode] in {
let hasSideEffects = 1,
Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
def LDTILECFG#Suffix : I<0x49, MRM0m, (outs), (ins opaquemem:$src),
@@ -27,6 +28,12 @@ let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
"sttilecfg\t$src",
[(int_x86_sttilecfg addr:$src)]>,
T8, PD;
+}
+}
+
+// Tile load/store instructions - AMX-TILE only (not available in ACE v1)
+multiclass AMX_TILE_LOADSTORE<string Suffix, Predicate HasEGPR> {
+let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
let mayLoad = 1 in
def TILELOADD#Suffix : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst),
(ins sibmem:$src),
@@ -46,10 +53,16 @@ let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
}
let SchedRW = [WriteSystem] in {
- defm "" : AMX_TILE_COMMON<"", NoEGPR>, VEX;
- defm "" : AMX_TILE_COMMON<"_EVEX", HasEGPR>, EVEX, NoCD8;
+ // Tile configuration instructions - shared between AMX-TILE and ACE v1
+ defm "" : AMX_TILE_CONFIG<"", NoEGPR>, VEX;
+ defm "" : AMX_TILE_CONFIG<"_EVEX", HasEGPR>, EVEX, NoCD8;
- let Predicates = [HasAMXTILE, In64BitMode] in {
+ // Tile load/store instructions - AMX-TILE only
+ defm "" : AMX_TILE_LOADSTORE<"", NoEGPR>, VEX;
+ defm "" : AMX_TILE_LOADSTORE<"_EVEX", HasEGPR>, EVEX, NoCD8;
+
+ // TILERELEASE and TILEZERO - shared between AMX-TILE and ACE v1
+ let Predicates = [HasAMXTILE_Or_ACEV1, In64BitMode] in {
let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
def TILERELEASE : I<0x49, MRM_C0, (outs), (ins),
"tilerelease", [(int_x86_tilerelease)]>, VEX, T8, PS;
@@ -57,10 +70,26 @@ let SchedRW = [WriteSystem] in {
"tilezero\t$dst", []>,
VEX, T8, XD;
- // Pseduo instruction for RA.
+ // Pseudo instructions for RA - shared between AMX-TILE and ACE v1
let isPseudo = true, mayLoad = 1, hasSideEffects = 1,
Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
def PLDTILECFGV : PseudoI<(outs), (ins opaquemem:$src), []>;
+
+ let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
+ canFoldAsLoad = 1, usesCustomInserter = 1 in
+ def PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
+ [(set TILE:$dst, (int_x86_tilezero_internal
+ GR16:$src1, GR16:$src2))]>;
+
+ let usesCustomInserter = 1 in {
+ // Pseudo instruction for TILEZERO with immediate
+ def PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
+ [(int_x86_tilezero timm:$src)]>;
+ }
+ } // Predicates - shared
+
+ // Tile load/store pseudo instructions - AMX-TILE only
+ let Predicates = [HasAMXTILE, In64BitMode] in {
let isPseudo = true, mayLoad = 1 in
def PTILELOADDV : PseudoI<(outs TILE:$dst), (ins GR16:$src1,
GR16:$src2,
@@ -73,15 +102,9 @@ let SchedRW = [WriteSystem] in {
def PTILESTOREDV : PseudoI<(outs), (ins GR16:$src1,
GR16:$src2, opaquemem:$src3,
TILE:$src4), []>;
- let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
- canFoldAsLoad = 1, usesCustomInserter = 1 in
- def PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
- [(set TILE:$dst, (int_x86_tilezero_internal
- GR16:$src1, GR16:$src2))]>;
let usesCustomInserter = 1 in {
- // Pseudo instructions, using immediates instead of tile registers.
- // To be translated to the actual instructions in X86ISelLowering.cpp
+ // Pseudo instructions for tile load/store with immediates
let mayLoad = 1 in
def PTILELOADD : PseudoI<(outs), (ins u8imm:$src1, sibmem:$src2), []>;
let mayLoad = 1 in
@@ -89,10 +112,8 @@ let SchedRW = [WriteSystem] in {
sibmem:$src2), []>;
let mayStore = 1 in
def PTILESTORED : PseudoI<(outs), (ins i8mem:$dst, u8imm:$src), []>;
- def PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
- [(int_x86_tilezero timm:$src)]>;
}
- } // Predicates
+ } // Predicates - AMX-TILE only
} // SchedRW
let Predicates = [HasAMXINT8, In64BitMode] in {
@@ -368,7 +389,7 @@ let Predicates = [HasAMXMOVRS, In64BitMode], SchedRW = [WriteSystem] in {
} // HasAMXMOVRS, In64BitMode
multiclass m_tcvtrowd2ps {
- let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+ let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
def rti : Ii8<0x7, MRMSrcReg, (outs VR512:$dst),
(ins TILE:$src1, i32u8imm:$src2),
@@ -379,12 +400,12 @@ multiclass m_tcvtrowd2ps {
"tcvtrowd2ps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
[]>, T8,XS, EVEX, VVVV, EVEX_V512;
}
- } // HasAMXAVX512, HasAVX10_2, In64BitMode
+ } // HasAMXAVX512_Or_ACEV1, In64BitMode
}
defm TCVTROWD2PS : m_tcvtrowd2ps;
-let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
let usesCustomInserter = 1 in {
def PTCVTROWD2PSrti : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
@@ -448,7 +469,7 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
multiclass AMXAVX512_BASE<bits<8> Opcode1, bits<8> Opcode2, string Opstr,
Prefix P1, Prefix P2> {
- let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode], SchedRW = [WriteSystem] in {
+ let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode], SchedRW = [WriteSystem] in {
let OpPrefix = P1 in
def rte : I<Opcode1, MRMSrcReg4VOp3, (outs VR512:$dst),
(ins TILE:$src1, GR32:$src2),
@@ -476,7 +497,7 @@ defm TCVTROWPS2BF16H : AMXAVX512_BASE<0x6d, 0x07, "tcvtrowps2bf16h", XD, XD>;
defm TCVTROWPS2BF16L : AMXAVX512_BASE<0x6d, 0x77, "tcvtrowps2bf16l", XS, XS>;
multiclass AMXAVX512_TILEMOVE<bits<8> Opcode1, bits<8> Opcode2, string Opstr> {
- let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+ let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
def rti : Ii8<Opcode1, MRMSrcReg, (outs VR512:$dst),
(ins TILE:$src1, u8imm:$src2),
@@ -487,12 +508,12 @@ multiclass AMXAVX512_TILEMOVE<bits<8> Opcode1, bits<8> Opcode2, string Opstr> {
!strconcat(Opstr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
[]>, T8, PD, EVEX, VVVV, EVEX_V512;
}
- } // HasAMXAVX512, HasAVX10_2, In64BitMode
+ } // HasAMXAVX512_Or_ACEV1, In64BitMode
}
defm TILEMOVROW : AMXAVX512_TILEMOVE<0x07, 0x4A, "tilemovrow">;
-let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
let usesCustomInserter = 1 in {
def PTILEMOVROWrti : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 6352fd87238229..25475ef4fd1982 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4517,7 +4517,13 @@ static unsigned getLoadStoreRegOpcode(Register Reg,
return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
case 1024:
assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
- assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
+ assert((STI.hasAMXTILE() || STI.hasACEV1()) &&
+ "Using 8*1024-bit register requires AMX-TILE or ACE");
+ // ACE v1 doesn't have TILELOADD/TILESTORED - tile spilling should be
+ // handled by X86LowerTileCopy using TILEMOVROW row-by-row.
+ // This path should only be hit for AMX targets.
+ assert(STI.hasAMXTILE() &&
+ "ACE tile spill should use TILEMOVROW, not TILELOADD/TILESTORED");
#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
: GET_EGPR_IF_ENABLED(X86::TILESTORED);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index caf8344dc664ce..dfeb83a88e6420 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -88,6 +88,9 @@ include "X86InstrAMX.td"
// AVX10 V2 AUX instructions
include "X86InstrAVX10_V2_AUX.td"
+// ACE instructions
+include "X86InstrACE.td"
+
// RAO-INT instructions
include "X86InstrRAOINT.td"
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 112118b7725f5a..6777f9bb540129 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -191,6 +191,9 @@ def HasAMXCOMPLEX : Predicate<"Subtarget->hasAMXCOMPLEX()">;
def HasAMXFP8 : Predicate<"Subtarget->hasAMXFP8()">;
def HasAMXMOVRS : Predicate<"Subtarget->hasAMXMOVRS()">;
def HasAMXAVX512 : Predicate<"Subtarget->hasAMXAVX512()">;
+def HasACEV1 : Predicate<"Subtarget->hasACEV1()">;
+def HasAMXTILE_Or_ACEV1 : Predicate<"Subtarget->hasAMXTILE() || Subtarget->hasACEV1()">;
+def HasAMXAVX512_Or_ACEV1 : Predicate<"(Subtarget->hasAMXAVX512() && Subtarget->hasAVX10_2()) || Subtarget->hasACEV1()">;
def HasUINTR : Predicate<"Subtarget->hasUINTR()">;
def HasUSERMSR : Predicate<"Subtarget->hasUSERMSR()">;
def HasCRC32 : Predicate<"Subtarget->hasCRC32()">;
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index 9d01f572b3125d..1b7f894a79fc74 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -39,6 +39,7 @@
//===----------------------------------------------------------------------===//
//
#include "X86.h"
+#include "X86Subtarget.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
@@ -100,6 +101,47 @@ static bool containsAMXCode(Function &F) {
return false;
}
+// Check if the instruction is an ACE-only intrinsic.
+// ACE (Palette 2) doesn't have TILELOADD/TILESTORED instructions,
+// so volatile tile data transformation should skip ACE intrinsics.
+static bool isACEOnlyIntrinsic(Instruction *I) {
+ auto *II = dyn_cast<IntrinsicInst>(I);
+ if (!II)
+ return false;
+ switch (II->getIntrinsicID()) {
+ // ACE outer product intrinsics
+ case Intrinsic::x86_top2bf16ps_internal:
+ case Intrinsic::x86_top4buud_internal:
+ case Intrinsic::x86_top4busd_internal:
+ case Intrinsic::x86_top4bssd_internal:
+ case Intrinsic::x86_top4bsud_internal:
+ // ACE mixed precision intrinsics
+ case Intrinsic::x86_top4mxhf8ps_internal:
+ case Intrinsic::x86_top4mxbhf8ps_internal:
+ case Intrinsic::x86_top4mxhbf8ps_internal:
+ case Intrinsic::x86_top4mxbf8ps_internal:
+ case Intrinsic::x86_top4mxbssps_internal:
+ // ACE tile movement intrinsics
+ case Intrinsic::x86_tilesetcol_internal:
+ case Intrinsic::x86_tilesetrow_internal:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// Check if PHI node has any ACE-only intrinsic as incoming value.
+static bool hasACEIncomingValue(PHINode *PHI) {
+ for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) {
+ Value *Op = PHI->getIncomingValue(I);
+ if (auto *Inst = dyn_cast<Instruction>(Op)) {
+ if (isACEOnlyIntrinsic(Inst))
+ return true;
+ }
+ }
+ return false;
+}
+
static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB,
Type *Ty) {
Function &F = *BB->getParent();
@@ -210,6 +252,59 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
Col = II->getArgOperand(1);
break;
}
+ // ACE internal intrinsics - outer products with ZMM sources
+ // Pattern: (m, n, k, acc_tile, zmm1, zmm2) -> acc_tile += zmm1 * zmm2
+ case Intrinsic::x86_top2bf16ps_internal:
+ case Intrinsic::x86_top4buud_internal:
+ case Intrinsic::x86_top4busd_internal:
+ case Intrinsic::x86_top4bssd_internal:
+ case Intrinsic::x86_top4bsud_internal: {
+ switch (OpNo) {
+ case 3: // Accumulator tile
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(1);
+ break;
+ case 4: // ZMM source 1 - doesn't have tile shape, use full dimensions
+ LLVM_FALLTHROUGH;
+ case 5: // ZMM source 2
+ // ZMM sources don't need shape calculation for tile purposes
+ // but if needed: Row = m, Col = k for src1; Row = k/4, Col = n for src2
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(2);
+ break;
+ }
+ break;
+ }
+ // ACE TOP4MX intrinsics - mixed precision with BSR index
+ // Pattern: (m, n, k, bsr_idx, acc_tile, zmm1, zmm2)
+ case Intrinsic::x86_top4mxhf8ps_internal:
+ case Intrinsic::x86_top4mxbhf8ps_internal:
+ case Intrinsic::x86_top4mxhbf8ps_internal:
+ case Intrinsic::x86_top4mxbf8ps_internal:
+ case Intrinsic::x86_top4mxbssps_internal: {
+ switch (OpNo) {
+ case 4: // Accumulator tile
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(1);
+ break;
+ case 5: // ZMM source 1
+ LLVM_FALLTHROUGH;
+ case 6: // ZMM source 2
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(2);
+ break;
+ }
+ break;
+ }
+ // ACE TILEMOV intrinsics - move ZMM to tile row/column
+ // Pattern: (m, n, zmm_src, idx) -> tile
+ case Intrinsic::x86_tilesetcol_internal:
+ case Intrinsic::x86_tilesetrow_internal: {
+ // Output tile shape
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(1);
+ break;
+ }
}
return std::make_pair(Row, Col);
@@ -714,11 +809,18 @@ bool X86VolatileTileData::volatileTileData() {
for (Instruction *I : AMXDefInsts) {
if (isIncomingOfPHI(I))
continue;
+ // Skip ACE-only intrinsics - ACE (Palette 2) doesn't have
+ // TILELOADD/TILESTORED instructions needed for volatile model.
+ if (isACEOnlyIntrinsic(I))
+ continue;
volatileTileNonPHI(I);
Changed = true;
}
for (Instruction *I : PHIInsts) {
+ // Skip PHI nodes with ACE intrinsic incoming values.
+ if (hasACEIncomingValue(dyn_cast<PHINode>(I)))
+ continue;
volatileTilePHI(dyn_cast<PHINode>(I));
Changed = true;
}
@@ -733,9 +835,18 @@ namespace {
class X86LowerAMXCast {
Function &Func;
std::unique_ptr<DominatorTree> DT;
+ bool IsACEOnly; // ACE v1 without AMX TILELOADD/TILESTORED
public:
- X86LowerAMXCast(Function &F) : Func(F), DT(nullptr) {}
+ X86LowerAMXCast(Function &F, const TargetMachine *TM = nullptr)
+ : Func(F), DT(nullptr), IsACEOnly(false) {
+ if (TM) {
+ // Check if we're ACE-only (no AMX TILELOADD/TILESTORED available)
+ const X86Subtarget *ST =
+ static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
+ IsACEOnly = ST && ST->hasACEV1() && !ST->hasAMXTILE();
+ }
+ }
bool combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
bool combineLoadCast(IntrinsicInst *Cast, LoadInst *LD);
bool combineTilezero(IntrinsicInst *Cast);
@@ -961,6 +1072,10 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
return false;
auto *II = cast<IntrinsicInst>(Tile);
+ // ACE v1 doesn't have TILESTORED instruction - skip combining for ACE targets
+ // or ACE intrinsics. ACE uses struct-based API with vector load/store.
+ if (IsACEOnly || isACEOnlyIntrinsic(II))
+ return false;
// Tile is output from AMX intrinsic. The first operand of the
// intrinsic is row, the second operand of the intrinsic is column.
Value *Row = II->getOperand(0);
@@ -991,6 +1106,10 @@ bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
// shape information through def-use chain.
if (!isAMXIntrinsic(II))
return false;
+ // ACE v1 doesn't have TILELOADD instruction - skip combining for ACE targets
+ // or ACE intrinsics. ACE uses struct-based API with vector load/store.
+ if (IsACEOnly || isACEOnlyIntrinsic(II))
+ return false;
std::tie(Row, Col) = getShape(II, OpNo);
IRBuilder<> Builder(LD);
Value *I8Ptr;
@@ -1224,6 +1343,18 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
auto *II = dyn_cast<IntrinsicInst>(U.getUser());
if (!II)
return false; // May be bitcast from x86amx to <256 x i32>.
+ // ACE v1 doesn't have TILELOADD - for ACE targets or ACE intrinsics,
+ // use tilezero since ACE tiles are typically zero-initialized in struct
+ // API.
+ if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ Value *Row = nullptr, *Col = nullptr;
+ std::tie(Row, Col) = getShape(II, OpNo);
+ Value *NewInst = Builder.CreateIntrinsic(Intrinsic::x86_tilezero_internal,
+ {}, {Row, Col});
+ AMXCast->replaceAllUsesWith(NewInst);
+ AMXCast->eraseFromParent();
+ return true;
+ }
Prepare(AMXCast->getOperand(0)->getType());
Builder.CreateStore(Src, AllocaAddr);
// TODO we can pick an constant operand for the shape.
@@ -1246,6 +1377,15 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
auto *II = dyn_cast<IntrinsicInst>(Src);
if (!II)
return false; // May be bitcast from <256 x i32> to x86amx.
+ // ACE v1 doesn't have TILESTORED - for ACE targets or ACE intrinsics,
+ // just create a poison value since the tile result is typically not read
+ // back.
+ if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ Value *Poison = PoisonValue::get(AMXCast->getType());
+ AMXCast->replaceAllUsesWith(Poison);
+ AMXCast->eraseFromParent();
+ return true;
+ }
Prepare(AMXCast->getType());
Value *Row = II->getOperand(0);
Value *Col = II->getOperand(1);
@@ -1289,7 +1429,7 @@ bool lowerAmxType(Function &F, const TargetMachine *TM,
return false;
bool C = false;
- X86LowerAMXCast LAC(F);
+ X86LowerAMXCast LAC(F, TM);
C |= LAC.combineAMXcast(TLI);
// There might be remaining AMXcast after combineAMXcast and they should be
// handled elegantly.
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 628f6912796005..09082ba585572c 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -36,6 +36,62 @@ using namespace llvm;
#define DEBUG_TYPE "x86-lower-tile-copy"
+/// Generate ACE-specific tile copy using TILEMOVROW row-by-row.
+/// ACE v1 doesn't have TILELOADD/TILESTORED, so we need to copy
+/// tiles through ZMM registers, one row at a time.
+///
+/// Spill sequence (tile -> stack):
+/// for row = 0 to 15:
+/// TILEMOVROW zmm_temp, tmm_src, row ; read row from tile to ZMM
+/// VMOVUPS [stack + row*64], zmm_temp ; store ZMM to stack
+///
+/// Reload sequence (stack -> tile):
+/// for row = 0 to 15:
+/// VMOVUPS zmm_temp, [stack + row*64] ; load ZMM from stack
+/// TILEMOVROW tmm_dst, zmm_temp, row ; write row from ZMM to tile
+///
+static void emitACETileCopy(MachineBasicBlock &MBB, MachineInstr &MI,
+ const X86Subtarget &ST, const X86InstrInfo *TII,
+ const TargetRegisterInfo *TRI, Register SrcReg,
+ Register DstReg, bool SrcKill, int TileSS,
+ Register ScratchZMM) {
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ // Each tile has 16 rows of 64 bytes each (1KB total)
+ const unsigned NumRows = 16;
+ const unsigned RowSize = 64;
+
+ // Spill: Read each row from source tile to ZMM, then store to stack
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // TILEMOVROW zmm, tmm, imm8 (read direction: tile -> ZMM)
+ // Uses TILEMOVROWrti instruction
+ // Only kill src on the last row read
+ bool KillSrcNow = (Row == NumRows - 1) && SrcKill;
+ BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
+ .addReg(SrcReg, getKillRegState(KillSrcNow))
+ .addImm(Row);
+
+ // VMOVUPS [stack + row*64], zmm
+ MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
+ addFrameReference(MIB, TileSS, Row * RowSize);
+ MIB.addReg(ScratchZMM, RegState::Kill);
+ }
+
+ // Reload: Load each row from stack to ZMM, then write to dest tile
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // VMOVUPS zmm, [stack + row*64]
+ MachineInstrBuilder MIB =
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), ScratchZMM);
+ addFrameReference(MIB, TileSS, Row * RowSize);
+
+ // TILEMOVROW tmm, zmm, imm8 (write direction: ZMM -> tile)
+ // Uses TILEMOVROWri instruction (ACE-specific)
+ BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWri), DstReg)
+ .addReg(ScratchZMM, RegState::Kill)
+ .addImm(Row);
+ }
+}
+
namespace {
class X86LowerTileCopyLegacy : public MachineFunctionPass {
@@ -71,11 +127,13 @@ FunctionPass *llvm::createX86LowerTileCopyLegacyPass() {
static bool lowerTileCopy(MachineFunction &MF) {
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
- if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ FuncInfo->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
- assert(ST.hasAMXTILE() && "Only supported on AMXTILE targets");
+ assert((ST.hasAMXTILE() || ST.hasACEV1()) &&
+ "Only supported on AMX-TILE or ACE v1 targets");
const X86InstrInfo *TII = ST.getInstrInfo();
const TargetRegisterInfo *TRI = ST.getRegisterInfo();
@@ -99,63 +157,109 @@ static bool lowerTileCopy(MachineFunction &MF) {
if (!X86::TILERegClass.contains(DstReg, SrcReg))
continue;
- // Allocate stack slot for tile register
+ // Allocate stack slot for tile register (1KB for ACE, same for AMX)
unsigned Size = TRI->getSpillSize(X86::TILERegClass);
Align Alignment = TRI->getSpillAlign(X86::TILERegClass);
int TileSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
- int StrideSS = 0;
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ // Check if this is ACE-only (no AMX TILELOADD/TILESTORED available)
+ if (ST.hasACEV1() && !ST.hasAMXTILE()) {
+ // ACE v1: Use TILEMOVROW row-by-row copy
+ // Need a scratch ZMM register for the transfer
- // Pick a killed register to avoid a save/reload.
- Register GR64Cand = X86::NoRegister;
- for (auto RegT : GR64Regs.set_bits()) {
- if (UsedRegs.available(RegT)) {
- GR64Cand = RegT;
- break;
+ // Find an available ZMM register
+ BitVector VR512Regs =
+ TRI->getAllocatableSet(MF, TRI->getRegClass(X86::VR512RegClassID));
+ Register ScratchZMM = X86::NoRegister;
+ for (auto RegT : VR512Regs.set_bits()) {
+ if (UsedRegs.available(RegT)) {
+ ScratchZMM = RegT;
+ break;
+ }
}
- }
- const DebugLoc &DL = MI.getDebugLoc();
- if (GR64Cand) {
- // mov 64 %reg
- BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), GR64Cand).addImm(64);
+ if (!ScratchZMM) {
+ // If no ZMM available, use ZMM0 and save/restore it
+ ScratchZMM = X86::ZMM0;
+
+ // Allocate stack slot for scratch ZMM
+ int ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
+
+ // Save ZMM0
+ MachineInstrBuilder MIB =
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
+ addFrameReference(MIB, ZmmSS);
+ MIB.addReg(X86::ZMM0);
+
+ // Emit ACE tile copy
+ emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
+ TileSS, ScratchZMM);
+
+ // Restore ZMM0
+ MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0);
+ addFrameReference(MIB, ZmmSS);
+ } else {
+ // Use available scratch ZMM
+ emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
+ TileSS, ScratchZMM);
+ }
} else {
- // No available register? Save RAX and reload it after use.
-
- // Allocate stack slot for stride register
- Size = TRI->getSpillSize(X86::GR64RegClass);
- Alignment = TRI->getSpillAlign(X86::GR64RegClass);
- StrideSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
-
- // mov %reg (%sp)
- addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV64mr)),
- StrideSS)
- .addReg(X86::RAX);
- // mov 64 %reg
- BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), X86::RAX).addImm(64);
- }
- // tilestored %tmm, (%sp, %idx)
+ // AMX: Use TILESTORED/TILELOADD (original code)
+ int StrideSS = 0;
+
+ // Pick a killed register to avoid a save/reload.
+ Register GR64Cand = X86::NoRegister;
+ for (auto RegT : GR64Regs.set_bits()) {
+ if (UsedRegs.available(RegT)) {
+ GR64Cand = RegT;
+ break;
+ }
+ }
+
+ if (GR64Cand) {
+ // mov 64 %reg
+ BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), GR64Cand).addImm(64);
+ } else {
+ // No available register? Save RAX and reload it after use.
+
+ // Allocate stack slot for stride register
+ Size = TRI->getSpillSize(X86::GR64RegClass);
+ Alignment = TRI->getSpillAlign(X86::GR64RegClass);
+ StrideSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
+
+ // mov %reg (%sp)
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV64mr)),
+ StrideSS)
+ .addReg(X86::RAX);
+ // mov 64 %reg
+ BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), X86::RAX).addImm(64);
+ }
+ // tilestored %tmm, (%sp, %idx)
#define GET_EGPR_IF_ENABLED(OPC) (ST.hasEGPR() ? OPC##_EVEX : OPC)
- unsigned Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
- MachineInstr *NewMI =
- addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS)
- .addReg(SrcReg, getKillRegState(SrcMO.isKill()));
- MachineOperand *MO = &NewMI->getOperand(X86::AddrIndexReg);
- MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
- // tileloadd (%sp, %idx), %tmm
- Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
+ unsigned Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
+ MachineInstr *NewMI =
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS)
+ .addReg(SrcReg, getKillRegState(SrcMO.isKill()));
+ MachineOperand *MO = &NewMI->getOperand(X86::AddrIndexReg);
+ MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
+ // tileloadd (%sp, %idx), %tmm
+ Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
#undef GET_EGPR_IF_ENABLED
- NewMI = addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc), DstReg),
- TileSS);
- MO = &NewMI->getOperand(1 + X86::AddrIndexReg);
- MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
- MO->setIsKill(true);
- if (!GR64Cand) {
- // restore %rax
- // mov (%sp) %rax
- addFrameReference(
- BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), X86::RAX), StrideSS);
+ NewMI = addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc), DstReg),
+ TileSS);
+ MO = &NewMI->getOperand(1 + X86::AddrIndexReg);
+ MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
+ MO->setIsKill(true);
+ if (!GR64Cand) {
+ // restore %rax
+ // mov (%sp) %rax
+ addFrameReference(
+ BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), X86::RAX), StrideSS);
+ }
}
+
MI.eraseFromParent();
Changed = true;
}
diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp b/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp
index 7b57f7c23bf4da..3197219f9e9822 100644
--- a/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp
+++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp
@@ -15,7 +15,8 @@ using namespace llvm;
yaml::X86MachineFunctionInfo::X86MachineFunctionInfo(
const llvm::X86MachineFunctionInfo &MFI)
- : AMXProgModel(MFI.getAMXProgModel()) {}
+ : AMXProgModel(MFI.getAMXProgModel()), ACEProgModel(MFI.getACEProgModel()) {
+}
void yaml::X86MachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
MappingTraits<X86MachineFunctionInfo>::mapping(YamlIO, *this);
@@ -31,6 +32,7 @@ MachineFunctionInfo *X86MachineFunctionInfo::clone(
void X86MachineFunctionInfo::initializeBaseYamlFields(
const yaml::X86MachineFunctionInfo &YamlMFI) {
AMXProgModel = YamlMFI.AMXProgModel;
+ ACEProgModel = YamlMFI.ACEProgModel;
}
void X86MachineFunctionInfo::anchor() { }
diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/llvm/lib/Target/X86/X86MachineFunctionInfo.h
index 1cfe86d57071f3..f5691aaf4df036 100644
--- a/llvm/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.h
@@ -24,6 +24,7 @@
namespace llvm {
enum AMXProgModelEnum { None = 0, DirectReg = 1, ManagedRA = 2 };
+enum ACEProgModelEnum { ACE_None = 0, ACE_DirectReg = 1, ACE_ManagedRA = 2 };
class X86MachineFunctionInfo;
@@ -36,8 +37,17 @@ template <> struct ScalarEnumerationTraits<AMXProgModelEnum> {
}
};
+template <> struct ScalarEnumerationTraits<ACEProgModelEnum> {
+ static void enumeration(IO &YamlIO, ACEProgModelEnum &Value) {
+ YamlIO.enumCase(Value, "None", ACEProgModelEnum::ACE_None);
+ YamlIO.enumCase(Value, "DirectReg", ACEProgModelEnum::ACE_DirectReg);
+ YamlIO.enumCase(Value, "ManagedRA", ACEProgModelEnum::ACE_ManagedRA);
+ }
+};
+
struct X86MachineFunctionInfo final : public yaml::MachineFunctionInfo {
AMXProgModelEnum AMXProgModel;
+ ACEProgModelEnum ACEProgModel;
X86MachineFunctionInfo() = default;
X86MachineFunctionInfo(const llvm::X86MachineFunctionInfo &MFI);
@@ -49,6 +59,7 @@ struct X86MachineFunctionInfo final : public yaml::MachineFunctionInfo {
template <> struct MappingTraits<X86MachineFunctionInfo> {
static void mapping(IO &YamlIO, X86MachineFunctionInfo &MFI) {
YamlIO.mapOptional("amxProgModel", MFI.AMXProgModel);
+ YamlIO.mapOptional("aceProgModel", MFI.ACEProgModel);
}
};
} // end namespace yaml
@@ -131,6 +142,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
/// The AMX programing model used in the function.
AMXProgModelEnum AMXProgModel = AMXProgModelEnum::None;
+ /// The ACE programming model used in the function.
+ ACEProgModelEnum ACEProgModel = ACEProgModelEnum::ACE_None;
+
/// True if this function has a subset of CSRs that is handled explicitly via
/// copies.
bool IsSplitCSR = false;
@@ -260,6 +274,14 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
AMXProgModel = Model;
}
+ ACEProgModelEnum getACEProgModel() const { return ACEProgModel; }
+ void setACEProgModel(ACEProgModelEnum Model) {
+ assert(
+ (ACEProgModel == ACEProgModelEnum::ACE_None || ACEProgModel == Model) &&
+ "mixed ACE model is not supported");
+ ACEProgModel = Model;
+ }
+
SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
return ForwardedMustTailRegParms;
}
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 5180b8692db5a2..765b6cb5f56d25 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -270,8 +270,9 @@ bool X86PreTileConfigImpl::runOnMachineFunction(MachineFunction &MF) {
scope_exit ClearStateOnExit([this] { releaseMemory(); });
X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
@@ -452,7 +453,10 @@ bool X86PreTileConfigImpl::runOnMachineFunction(MachineFunction &MF) {
.addReg(Xmm);
}
// Fill in the palette first.
- addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), SS).addImm(1);
+ // Use Palette 2 for ACE v1, Palette 1 for AMX.
+ int PaletteId = ST.hasACEV1() ? 2 : 1;
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), SS)
+ .addImm(PaletteId);
return true;
}
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index a940a8ffadcbd6..50e68e39d68369 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -693,7 +693,7 @@ unsigned X86RegisterInfo::getNumSupportedRegs(const MachineFunction &MF) const {
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
if (ST.hasEGPR())
return X86::NUM_TARGET_REGS;
- if (ST.hasAMXTILE())
+ if (ST.hasAMXTILE() || ST.hasACEV1())
return X86::TMM7 + 1;
if (ST.hasAVX512())
return X86::K6_K7 + 1;
@@ -1143,7 +1143,20 @@ static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
case X86::PTDPBF8PSV:
case X86::PTDPBHF8PSV:
case X86::PTDPHBF8PSV:
- case X86::PTDPHF8PSV: {
+ case X86::PTDPHF8PSV:
+ // ACE internal pseudos - same pattern: operands 1,2 are row,col
+ case X86::PTOP2BF16PSV:
+ case X86::PTOP4BUUDV:
+ case X86::PTOP4BUSDV:
+ case X86::PTOP4BSSDV:
+ case X86::PTOP4BSUDV:
+ case X86::PTOP4MXHF8PSV:
+ case X86::PTOP4MXBHF8PSV:
+ case X86::PTOP4MXHBF8PSV:
+ case X86::PTOP4MXBF8PSV:
+ case X86::PTOP4MXBSSPSV:
+ case X86::PTILEMOVCOLV:
+ case X86::PTILEMOVROWV: {
MachineOperand &MO1 = MI->getOperand(1);
MachineOperand &MO2 = MI->getOperand(2);
ShapeT Shape(&MO1, &MO2, MRI);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index 397aeb47472903..364979f3ce8cc9 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -421,6 +421,8 @@ def KPAIRS : RegisterTuples<[sub_mask_0, sub_mask_1],
let PositionOrder = 3 in {
// Tile config registers.
def TMMCFG: X86Reg<"tmmcfg", 0>;
+ // Block Scale Register (ACE)
+ def BSR0 : X86Reg<"bsr0", 0>;
// Tile "registers".
foreach Index = 0-7 in {
def TMM#Index : X86Reg<"tmm"#Index, Index>;
@@ -852,6 +854,13 @@ let CopyCost = -1 in // Don't allow copying of tile registers
def TILE : RegisterClass<"X86", [x86amx], 8192,
(sequence "TMM%u", 0, 7)> {let Size = 8192;}
+// Block Scale Registers (ACE)
+let CopyCost = -1 in // Don't allow copying of BSR register
+def BSR : RegisterClass<"X86", [untyped], 0, (add BSR0)> {
+ let Size = 1024;
+ let isAllocatable = 0;
+}
+
//===----------------------------------------------------------------------===//
// Register categories.
//
diff --git a/llvm/lib/Target/X86/X86TileConfig.cpp b/llvm/lib/Target/X86/X86TileConfig.cpp
index 660bd0a77ab795..6667c8d1a66875 100644
--- a/llvm/lib/Target/X86/X86TileConfig.cpp
+++ b/llvm/lib/Target/X86/X86TileConfig.cpp
@@ -78,8 +78,9 @@ static bool tileConfig(MachineFunction &MF,
llvm::function_ref<LiveIntervals *()> GetLIs,
llvm::function_ref<VirtRegMap *()> GetVRM) {
X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
@@ -137,6 +138,11 @@ static bool tileConfig(MachineFunction &MF,
}
// Fill in the shape of each tile physical register.
+ // ACE Palette 2 uses fixed tile dimensions (16x64), so skip writing
+ // shapes - bytes 1-63 must remain zero for ACE.
+ if (ST.hasACEV1())
+ return true;
+
for (unsigned I = 0; I < AMXRegNum; ++I) {
if (!Phys2Virt[I])
continue;
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 384687702c7c25..e8ed9a9f572f92 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -639,6 +639,7 @@ constexpr FeatureBitset ImpliedFeaturesAMX_FP8 = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_MOVRS = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_AVX512 =
FeatureAMX_TILE | FeatureAVX10_2;
+constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAMX_AVX512;
constexpr FeatureBitset ImpliedFeaturesHRESET = {};
constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {};
diff --git a/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll b/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
new file mode 100644
index 00000000000000..9afca28b9c9acd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
@@ -0,0 +1,244 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f | FileCheck %s
+
+;
+; ACE Intrinsic Tests
+;
+
+; Test BSRINIT intrinsic
+define void @test_bsrinit() {
+; CHECK-LABEL: test_bsrinit:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrinit %bsr0
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrinit()
+ ret void
+}
+
+; Test BSRMOVF intrinsic
+define void @test_bsrmovf(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_bsrmovf:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrmovf(<16 x i32> %a, <16 x i32> %b)
+ ret void
+}
+
+; Test BSRMOVH set intrinsic
+define void @test_bsrmovh_set(<16 x i32> %a) {
+; CHECK-LABEL: test_bsrmovh_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %zmm0, %bsr0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrmovh.set(<16 x i32> %a)
+ ret void
+}
+
+; Test BSRMOVH get intrinsic
+define <16 x i32> @test_bsrmovh_get() {
+; CHECK-LABEL: test_bsrmovh_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, %zmm0
+; CHECK-NEXT: retq
+ %result = call <16 x i32> @llvm.x86.bsrmovh.get()
+ ret <16 x i32> %result
+}
+
+; Test BSRMOVL set intrinsic
+define void @test_bsrmovl_set(<16 x i32> %a) {
+; CHECK-LABEL: test_bsrmovl_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %zmm0, %bsr0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrmovl.set(<16 x i32> %a)
+ ret void
+}
+
+; Test BSRMOVL get intrinsic
+define <16 x i32> @test_bsrmovl_get() {
+; CHECK-LABEL: test_bsrmovl_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %bsr0, %zmm0
+; CHECK-NEXT: retq
+ %result = call <16 x i32> @llvm.x86.bsrmovl.get()
+ ret <16 x i32> %result
+}
+
+; Test TOP2BF16PS intrinsic
+define void @test_top2bf16ps() {
+; CHECK-LABEL: test_top2bf16ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top2bf16ps %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BUUD intrinsic
+define void @test_top4buud() {
+; CHECK-LABEL: test_top4buud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4buud %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BUSD intrinsic
+define void @test_top4busd() {
+; CHECK-LABEL: test_top4busd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4busd %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BSSD intrinsic
+define void @test_top4bssd() {
+; CHECK-LABEL: test_top4bssd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bssd %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BSUD intrinsic
+define void @test_top4bsud() {
+; CHECK-LABEL: test_top4bsud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bsud %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4MXHF8PS intrinsic
+define void @test_top4mxhf8ps() {
+; CHECK-LABEL: test_top4mxhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXBHF8PS intrinsic
+define void @test_top4mxbhf8ps() {
+; CHECK-LABEL: test_top4mxbhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbhf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXHBF8PS intrinsic
+define void @test_top4mxhbf8ps() {
+; CHECK-LABEL: test_top4mxhbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhbf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXBF8PS intrinsic
+define void @test_top4mxbf8ps() {
+; CHECK-LABEL: test_top4mxbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXBSSPS intrinsic (MX INT8 S×S with BSR scaling)
+define void @test_top4mxbssps() {
+; CHECK-LABEL: test_top4mxbssps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbssps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
+define void @test_tilemovcol(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovcol:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %src, i32 3)
+ ret void
+}
+
+; Test TILEMOVROW intrinsic - write to tile (ACE extension)
+; Uses constant index - generates immediate form
+define void @test_tilemovrow_to_tile(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovrow_to_tile:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetrow(i8 1, <16 x i32> %src, i32 5)
+ ret void
+}
+
+; Test TILEMOVCOL with register index
+define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovcol_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetcol(i8 2, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Test TILEMOVROW with register index
+define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovrow_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetrow(i8 3, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Intrinsic declarations
+declare void @llvm.x86.bsrinit()
+declare void @llvm.x86.bsrmovf(<16 x i32>, <16 x i32>)
+declare void @llvm.x86.bsrmovh.set(<16 x i32>)
+declare <16 x i32> @llvm.x86.bsrmovh.get()
+declare void @llvm.x86.bsrmovl.set(<16 x i32>)
+declare <16 x i32> @llvm.x86.bsrmovl.get()
+declare void @llvm.x86.tilesetcol(i8, <16 x i32>, i32)
+declare void @llvm.x86.tilesetrow(i8, <16 x i32>, i32)
+declare void @llvm.x86.top2bf16ps(i8, i8, i8)
+declare void @llvm.x86.top4buud(i8, i8, i8)
+declare void @llvm.x86.top4busd(i8, i8, i8)
+declare void @llvm.x86.top4bssd(i8, i8, i8)
+declare void @llvm.x86.top4bsud(i8, i8, i8)
+declare void @llvm.x86.top4mxhf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxbhf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxhbf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxbf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxbssps(i8, i8, i8, i8)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll b/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
new file mode 100644
index 00000000000000..4b114d5bc9c080
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
@@ -0,0 +1,99 @@
+; ACE v1 Greedy Register Allocation Test
+; Tests that tile registers are allocated correctly in a separate pass
+; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
+;
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+
+; Test basic tile register allocation for ACE
+define void @test_acev1_ra(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra
+ ; CHECK: PLDTILECFGV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTOP2BF16PSV
+ ; CHECK: PTILEMOVROWrtiV
+entry:
+ %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract using TILEMOVROW (ACE v1 pattern)
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test multiple tile operations requiring allocation
+define void @test_acev1_ra_multi(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra_multi
+ ; CHECK: PLDTILECFGV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTOP2BF16PSV
+entry:
+ %c1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c2, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test integer outer products register allocation
+define void @test_acev1_ra_int(ptr %out, <64 x i8> %a_i8) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra_int
+ ; CHECK: PLDTILECFGV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTOP4BSSDV
+ ; CHECK: PTOP4BUUDV
+entry:
+ %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d1 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d2 = tail call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test register allocation with control flow
+define void @test_acev1_ra_branch(ptr %out, i32 %cond, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra_branch
+ ; CHECK: PLDTILECFGV
+entry:
+ %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %icmp = icmp eq i32 %cond, 0
+ br i1 %icmp, label %then, label %else
+
+then:
+ %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ br label %merge
+
+else:
+ %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_b, <32 x bfloat> %zmm_a)
+ br label %merge
+
+merge:
+ %result = phi x86_amx [ %d1, %then ], [ %d2, %else ]
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %result, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll b/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
new file mode 100644
index 00000000000000..47228dfb8e614d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
@@ -0,0 +1,204 @@
+; ACE v1 Outer Product Operations Test
+; Tests all ACE outer product variants: BF16, INT8 (signed/unsigned combinations)
+; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
+;
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+
+; Test BF16 outer product (TOP2BF16PS)
+define void @test_top2bf16ps(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
+; CHECK-LABEL: test_top2bf16ps:
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: top2bf16ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract using TILEMOVROW (ACE v1 pattern)
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test unsigned x unsigned -> dword (TOP4BUUD)
+define void @test_top4buud(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4buud:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4buud
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test signed x signed -> dword (TOP4BSSD)
+define void @test_top4bssd(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4bssd:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4bssd
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test unsigned x signed -> dword (TOP4BUSD)
+define void @test_top4busd(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4busd:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4busd
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test signed x unsigned -> dword (TOP4BSUD)
+define void @test_top4bsud(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4bsud:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4bsud
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 HF8xHF8 -> FP32 with BSR scaling (TOP4MXHF8PS)
+define void @test_top4mxhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxhf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxhf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 BF8xHF8 -> FP32 with BSR scaling (TOP4MXBHF8PS)
+define void @test_top4mxbhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxbhf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxbhf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 HF8xBF8 -> FP32 with BSR scaling (TOP4MXHBF8PS)
+define void @test_top4mxhbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxhbf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxhbf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 BF8xBF8 -> FP32 with BSR scaling (TOP4MXBF8PS)
+define void @test_top4mxbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxbf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxbf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX INT8 SxS -> FP32 with BSR scaling (TOP4MXBSSPS)
+define void @test_top4mxbssps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxbssps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxbssps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test chained outer products (accumulation) - simplified to avoid spills
+define void @test_chained_outer_products(ptr %out, <32 x bfloat> %zmm_bf16, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_chained_outer_products:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top2bf16ps
+; CHECK: top4bssd
+; CHECK: top4buud
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; BF16 outer product
+ %d1 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_bf16, <32 x bfloat> %zmm_bf16)
+
+ ; Integer outer products
+ %d2 = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d3 = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d2, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4busd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4bsud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4mxhf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxbhf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxhbf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxbf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxbssps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll b/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
new file mode 100644
index 00000000000000..2d94260701e404
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
@@ -0,0 +1,94 @@
+; ACE v1 Basic Tile Operations Test
+; Tests basic ACE tile operations: tilezero, outer products
+; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
+;
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+
+define void @test_acev1_basic(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
+; CHECK-LABEL: test_acev1_basic:
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: top2bf16ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; ACE BF16 outer product
+ %d = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract result using TILEMOVROW (ACE v1 pattern - no TILESTORED)
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test ACE integer outer products
+define void @test_acev1_int_outer_products(ptr %out, <64 x i8> %a_i8, <64 x i8> %b_i8) {
+; CHECK-LABEL: test_acev1_int_outer_products:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK-DAG: top4buud
+; CHECK-DAG: top4bssd
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; Unsigned x Unsigned -> DWORD
+ %d0 = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %b_i8)
+
+ ; Signed x Signed -> DWORD
+ %d1 = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d0, <64 x i8> %a_i8, <64 x i8> %b_i8)
+
+ ; Extract result using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test multiple outer product accumulations
+define void @test_acev1_accumulation(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
+; CHECK-LABEL: test_acev1_accumulation:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top2bf16ps
+; CHECK: top2bf16ps
+; CHECK: top2bf16ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; Multiple accumulations
+ %d0 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d1 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d0, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d2 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract result using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test basic tilezero operation
+define void @test_acev1_tilezero(ptr %out) {
+; CHECK-LABEL: test_acev1_tilezero:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %c, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ipra-reg-usage.ll b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
index c008d691ee8419..9d84ffdd481633 100644
--- a/llvm/test/CodeGen/X86/ipra-reg-usage.ll
+++ b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
@@ -7,7 +7,7 @@
target triple = "x86_64-unknown-unknown"
declare void @bar1()
define preserve_allcc void @foo()#0 {
-; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hip $hsp $ip $mxcsr $rflags $rip $riz $rsp $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $r11b $r11bh $r11d $r11w $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
+; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hip $hsp $ip $mxcsr $rflags $rip $riz $rsp $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $r11b $r11bh $r11d $r11w $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $bsr0 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
call void @bar1()
call void @bar2()
ret void
@@ -15,7 +15,7 @@ define preserve_allcc void @foo()#0 {
declare void @bar2()
define preserve_nonecc void @foo2()#0 {
-; CHECK: foo2 Clobbered Registers: $ah $al $ax $ch $cl $cs $cx $df $dh $di $dih $dil $dl $ds $dx $eax $ecx $edi $edx $eflags $eip $eiz $es $esi $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hax $hcx $hdi $hdx $hip $hsi $hsp $ip $mxcsr $rax $rcx $rdi $rdx $rflags $rip $riz $rsi $rsp $si $sih $sil $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r8 $r9 $r10 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm0 $xmm1 $xmm2 $xmm3 $xmm4 $xmm5 $xmm6 $xmm7 $xmm8 $xmm9 $xmm10 $xmm11 $xmm12 $xmm13 $xmm14 $xmm15 $r8b $r9b $r10b $r11b $r8bh $r9bh $r10bh $r11bh $r8d $r9d $r10d $r11d $r8w $r9w $r10w $r11w $r8wh $r9wh $r10wh $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
+; CHECK: foo2 Clobbered Registers: $ah $al $ax $ch $cl $cs $cx $df $dh $di $dih $dil $dl $ds $dx $eax $ecx $edi $edx $eflags $eip $eiz $es $esi $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hax $hcx $hdi $hdx $hip $hsi $hsp $ip $mxcsr $rax $rcx $rdi $rdx $rflags $rip $riz $rsi $rsp $si $sih $sil $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r8 $r9 $r10 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm0 $xmm1 $xmm2 $xmm3 $xmm4 $xmm5 $xmm6 $xmm7 $xmm8 $xmm9 $xmm10 $xmm11 $xmm12 $xmm13 $xmm14 $xmm15 $r8b $r9b $r10b $r11b $r8bh $r9bh $r10bh $r11bh $r8d $r9d $r10d $r11d $r8w $r9w $r10w $r11w $r8wh $r9wh $r10wh $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $bsr0 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
call void @bar1()
call void @bar2()
ret void
diff --git a/llvm/test/MC/X86/ACE/ace-att.s b/llvm/test/MC/X86/ACE/ace-att.s
new file mode 100644
index 00000000000000..cf43cb899bb2c8
--- /dev/null
+++ b/llvm/test/MC/X86/ACE/ace-att.s
@@ -0,0 +1,109 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1 -show-encoding %s | FileCheck %s
+
+// CHECK: bsrinit %bsr0
+// CHECK: encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
+ bsrinit %bsr0
+
+// CHECK: bsrmovf %zmm2, %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc2]
+ bsrmovf %zmm2, %zmm1, %bsr0
+
+// CHECK: bsrmovf (%rax), %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
+ bsrmovf (%rax), %zmm1, %bsr0
+
+// CHECK: bsrmovh %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0xc1]
+ bsrmovh %zmm1, %bsr0
+
+// CHECK: bsrmovh (%rbx), %bsr0
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0x03]
+ bsrmovh (%rbx), %bsr0
+
+// CHECK: bsrmovh %bsr0, %zmm1
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc1]
+ bsrmovh %bsr0, %zmm1
+
+// CHECK: bsrmovh %bsr0, (%rcx)
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0x01]
+ bsrmovh %bsr0, (%rcx)
+
+// CHECK: bsrmovl %zmm2, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc2]
+ bsrmovl %zmm2, %bsr0
+
+// CHECK: bsrmovl (%rdx), %bsr0
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0x02]
+ bsrmovl (%rdx), %bsr0
+
+// CHECK: bsrmovl %bsr0, %zmm3
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc3]
+ bsrmovl %bsr0, %zmm3
+
+// CHECK: bsrmovl %bsr0, (%rsi)
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0x06]
+ bsrmovl %bsr0, (%rsi)
+
+// CHECK: tilemovcol $5, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xca,0x05]
+ tilemovcol $5, %zmm2, %tmm1
+
+// CHECK: tilemovcol %ecx, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x48,0x4b,0xca]
+ tilemovcol %ecx, %zmm2, %tmm1
+
+// CHECK: tilemovrow $3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x07,0xca,0x03]
+ tilemovrow $3, %zmm2, %tmm1
+
+// CHECK: tilemovrow %edx, %zmm3, %tmm2
+// CHECK: encoding: [0x62,0xf2,0xed,0x48,0x4a,0xd3]
+ tilemovrow %edx, %zmm3, %tmm2
+
+// CHECK: top2bf16ps %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5c,0xca]
+ top2bf16ps %zmm3, %zmm2, %tmm1
+
+// CHECK: top4buud %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x64,0x48,0x5e,0xca]
+ top4buud %zmm3, %zmm2, %tmm1
+
+// CHECK: top4busd %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x65,0x48,0x5e,0xca]
+ top4busd %zmm3, %zmm2, %tmm1
+
+// CHECK: top4bssd %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x5e,0xca]
+ top4bssd %zmm3, %zmm2, %tmm1
+
+// CHECK: top4bsud %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5e,0xca]
+ top4bsud %zmm3, %zmm2, %tmm1
+
+// CHECK: top4mxbf8ps $7, %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0x64,0x48,0x8d,0xca,0x07]
+ top4mxbf8ps $7, %zmm3, %zmm2, %tmm1
+
+// CHECK: top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0x65,0x48,0x8d,0xca,0x07]
+ top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
+
+// CHECK: top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
+// CHECK: encoding: [0x62,0xf3,0x5f,0x48,0x8d,0xd5,0x03]
+ top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
+
+// CHECK: top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
+// CHECK: encoding: [0x62,0xf3,0x4e,0x48,0x8d,0xdf,0x01]
+ top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
+
+// CHECK: top4mxbssps $15, %zmm8, %zmm9, %tmm4
+// CHECK: encoding: [0x62,0xd3,0x3f,0x48,0x8f,0xe1,0x0f]
+ top4mxbssps $15, %zmm8, %zmm9, %tmm4
+
+// CHECK: top4buud %zmm31, %zmm30, %tmm7
+// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+ top4buud %zmm31, %zmm30, %tmm7
+
+// CHECK: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
+ bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
diff --git a/llvm/test/MC/X86/ACE/ace-error.s b/llvm/test/MC/X86/ACE/ace-error.s
new file mode 100644
index 00000000000000..e939b843564033
--- /dev/null
+++ b/llvm/test/MC/X86/ACE/ace-error.s
@@ -0,0 +1,39 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1 %s -o /dev/null 2>&1 | FileCheck %s
+
+// Test missing explicit BSR operand
+// CHECK: error: BSR instruction requires explicit %bsr0 operand
+bsrmovf %zmm1, %zmm2
+
+// CHECK: error:
+tilemovcol %tmm1, %xmm2, $5
+
+// CHECK: error:
+top4busd %tmm1, %ymm2, %zmm3
+
+// CHECK: error:
+tilemovrow $256, %zmm2, %tmm1
+
+// Test feature gating - ACEV1 instructions require ACEV1 feature
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=-acev1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOACEV1
+
+// NOACEV1: error:
+bsrinit %bsr0
+
+// NOACEV1: error:
+top2bf16ps %zmm3, %zmm2, %tmm1
+
+// Test that ACEV1 requires AMX-TILE (dependency check)
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1,-amx-tile %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOAMX
+
+// NOAMX: error:
+tilemovcol $5, %zmm2, %tmm1
+
+// Test invalid BSR register (only bsr0 exists)
+// CHECK: error:
+bsrmovf %zmm1, %zmm2, %bsr1
+
+// Test 32-bit mode rejection (ACEV1 is 64-bit only)
+// RUN: not llvm-mc -triple i386-unknown-unknown -mattr=+acev1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO32BIT
+
+// NO32BIT: error:
+bsrinit %bsr0
diff --git a/llvm/test/MC/X86/ACE/ace-intel.s b/llvm/test/MC/X86/ACE/ace-intel.s
new file mode 100644
index 00000000000000..81b5d8e0f7e249
--- /dev/null
+++ b/llvm/test/MC/X86/ACE/ace-intel.s
@@ -0,0 +1,109 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 -mattr=+acev1 --show-encoding %s | FileCheck %s
+
+// CHECK: bsrinit bsr0
+// CHECK: encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
+ bsrinit bsr0
+
+// CHECK: bsrmovf bsr0, zmm1, zmm2
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc2]
+ bsrmovf bsr0, zmm1, zmm2
+
+// CHECK: bsrmovf bsr0, zmm1, zmmword ptr [rax]
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
+ bsrmovf bsr0, zmm1, zmmword ptr [rax]
+
+// CHECK: bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10]
+ bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: bsrmovh bsr0, zmm1
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0xc1]
+ bsrmovh bsr0, zmm1
+
+// CHECK: bsrmovh bsr0, zmmword ptr [rbx]
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0x03]
+ bsrmovh bsr0, zmmword ptr [rbx]
+
+// CHECK: bsrmovh zmm1, bsr0
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc1]
+ bsrmovh zmm1, bsr0
+
+// CHECK: bsrmovh zmmword ptr [rcx], bsr0
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0x01]
+ bsrmovh zmmword ptr [rcx], bsr0
+
+// CHECK: bsrmovl bsr0, zmm2
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc2]
+ bsrmovl bsr0, zmm2
+
+// CHECK: bsrmovl bsr0, zmmword ptr [rdx]
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0x02]
+ bsrmovl bsr0, zmmword ptr [rdx]
+
+// CHECK: bsrmovl zmm3, bsr0
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc3]
+ bsrmovl zmm3, bsr0
+
+// CHECK: bsrmovl zmmword ptr [rsi], bsr0
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0x06]
+ bsrmovl zmmword ptr [rsi], bsr0
+
+// CHECK: tilemovcol tmm1, zmm2, 5
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xca,0x05]
+ tilemovcol tmm1, zmm2, 5
+
+// CHECK: tilemovcol tmm1, zmm2, ecx
+// CHECK: encoding: [0x62,0xf2,0xf5,0x48,0x4b,0xca]
+ tilemovcol tmm1, zmm2, ecx
+
+// CHECK: tilemovrow tmm1, zmm2, 3
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x07,0xca,0x03]
+ tilemovrow tmm1, zmm2, 3
+
+// CHECK: tilemovrow tmm2, zmm3, edx
+// CHECK: encoding: [0x62,0xf2,0xed,0x48,0x4a,0xd3]
+ tilemovrow tmm2, zmm3, edx
+
+// CHECK: top2bf16ps tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5c,0xca]
+ top2bf16ps tmm1, zmm2, zmm3
+
+// CHECK: top4buud tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x64,0x48,0x5e,0xca]
+ top4buud tmm1, zmm2, zmm3
+
+// CHECK: top4busd tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x65,0x48,0x5e,0xca]
+ top4busd tmm1, zmm2, zmm3
+
+// CHECK: top4bssd tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x5e,0xca]
+ top4bssd tmm1, zmm2, zmm3
+
+// CHECK: top4bsud tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5e,0xca]
+ top4bsud tmm1, zmm2, zmm3
+
+// CHECK: top4mxbf8ps tmm1, zmm2, zmm3, 7
+// CHECK: encoding: [0x62,0xf3,0x64,0x48,0x8d,0xca,0x07]
+ top4mxbf8ps tmm1, zmm2, zmm3, 7
+
+// CHECK: top4mxhf8ps tmm1, zmm2, zmm3, 7
+// CHECK: encoding: [0x62,0xf3,0x65,0x48,0x8d,0xca,0x07]
+ top4mxhf8ps tmm1, zmm2, zmm3, 7
+
+// CHECK: top4mxbhf8ps tmm2, zmm4, zmm5, 3
+// CHECK: encoding: [0x62,0xf3,0x57,0x48,0x8d,0xd4,0x03]
+ top4mxbhf8ps tmm2, zmm4, zmm5, 3
+
+// CHECK: top4mxhbf8ps tmm3, zmm6, zmm7, 1
+// CHECK: encoding: [0x62,0xf3,0x46,0x48,0x8d,0xde,0x01]
+ top4mxhbf8ps tmm3, zmm6, zmm7, 1
+
+// CHECK: top4mxbssps tmm4, zmm8, zmm9, 15
+// CHECK: encoding: [0x62,0xd3,0x37,0x48,0x8f,0xe0,0x0f]
+ top4mxbssps tmm4, zmm8, zmm9, 15
+
+// CHECK: top4buud tmm7, zmm30, zmm31
+// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+ top4buud tmm7, zmm30, zmm31
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index 0e261b10a09b2a..b929b7e035f00c 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -354,6 +354,10 @@ static const X86FoldTableEntry Table2Addr[] = {
};
static const X86FoldTableEntry Table0[] = {
+ {X86::BSRMOVHrr_get, X86::BSRMOVHmr_get, TB_FOLDED_STORE},
+ {X86::BSRMOVHrr_set, X86::BSRMOVHrm_set, TB_FOLDED_LOAD},
+ {X86::BSRMOVLrr_get, X86::BSRMOVLmr_get, TB_FOLDED_STORE},
+ {X86::BSRMOVLrr_set, X86::BSRMOVLrm_set, TB_FOLDED_LOAD},
{X86::BT16ri8, X86::BT16mi8, TB_FOLDED_LOAD},
{X86::BT32ri8, X86::BT32mi8, TB_FOLDED_LOAD},
{X86::BT64ri8, X86::BT64mi8, TB_FOLDED_LOAD},
@@ -689,6 +693,7 @@ static const X86FoldTableEntry Table1[] = {
{X86::BLSR64rr, X86::BLSR64rm, 0},
{X86::BLSR64rr_EVEX, X86::BLSR64rm_EVEX, 0},
{X86::BLSR64rr_NF, X86::BLSR64rm_NF, 0},
+ {X86::BSRMOVFrr, X86::BSRMOVFrm, 0},
{X86::BZHI32rr, X86::BZHI32rm, 0},
{X86::BZHI32rr_EVEX, X86::BZHI32rm_EVEX, 0},
{X86::BZHI32rr_NF, X86::BZHI32rm_NF, 0},
@@ -11634,6 +11639,11 @@ static const X86FoldTableEntry MemUnfoldTable[] = {
{X86::BSR16rm, X86::BSR16rr, TB_INDEX_2|TB_FOLDED_LOAD},
{X86::BSR32rm, X86::BSR32rr, TB_INDEX_2|TB_FOLDED_LOAD},
{X86::BSR64rm, X86::BSR64rr, TB_INDEX_2|TB_FOLDED_LOAD},
+ {X86::BSRMOVFrm, X86::BSRMOVFrr, TB_INDEX_1|TB_FOLDED_LOAD},
+ {X86::BSRMOVHmr_get, X86::BSRMOVHrr_get, TB_INDEX_0|TB_FOLDED_STORE},
+ {X86::BSRMOVHrm_set, X86::BSRMOVHrr_set, TB_INDEX_0|TB_FOLDED_LOAD},
+ {X86::BSRMOVLmr_get, X86::BSRMOVLrr_get, TB_INDEX_0|TB_FOLDED_STORE},
+ {X86::BSRMOVLrm_set, X86::BSRMOVLrr_set, TB_INDEX_0|TB_FOLDED_LOAD},
{X86::BT16mi8, X86::BT16ri8, TB_INDEX_0|TB_FOLDED_LOAD},
{X86::BT32mi8, X86::BT32ri8, TB_INDEX_0|TB_FOLDED_LOAD},
{X86::BT64mi8, X86::BT64ri8, TB_INDEX_0|TB_FOLDED_LOAD},
>From da270314b35fb590971120292141bd230591510c Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Tue, 1 Sep 2026 18:07:40 +0530
Subject: [PATCH 2/6] Address review comments
- Rename the ACE tests and directories to acev1 for consistency
- Build acev1 on top of avx10.1 rather than avx10.2
- Fix test encodings and the BSR0 implicit defs
- Make the scale register type consistent with the spec
- Add memory operand tests for bsrmov
- Confirm the ACEV1 ISA implication chain
- Take the outer product ZMM sources by value instead of by register ID,
so they are allocated conventionally, and mark the 512-bit ACE builtins
with RequiredVectorWidth<512>
- Use pseudos for SPILL and RELOAD. Spills use scratch zmm. Don't use
non-ace ISA.
- Fix missing vector r/m in MRMSrcReg4VOp3.
---
clang/include/clang/Basic/BuiltinsX86_64.td | 80 ++--
clang/lib/Headers/acev1intrin.h | 148 ++++----
clang/lib/Sema/SemaX86.cpp | 17 +-
clang/test/CodeGen/X86/ace-api.c | 145 -------
clang/test/CodeGen/X86/ace-builtins.c | 119 ------
clang/test/CodeGen/X86/ace-inline-asm.c | 85 -----
clang/test/CodeGen/X86/acev1.c | 37 +-
clang/test/CodeGen/X86/acev1_api.c | 55 ++-
clang/test/CodeGen/X86/acev1_bsr.c | 18 +-
clang/test/CodeGen/X86/acev1_errors.c | 10 +-
clang/test/CodeGen/X86/acev1_inline_asm.c | 67 +++-
clang/test/CodeGen/X86/acev1_mxfp8.c | 44 +--
clang/test/CodeGen/X86/acev1_tile_movement.c | 31 +-
clang/test/CodeGen/attr-target-x86.c | 2 +-
clang/test/Preprocessor/x86_target_features.c | 24 +-
llvm/include/llvm/IR/IntrinsicsX86.td | 99 +++--
.../lib/Target/X86/AsmParser/X86AsmParser.cpp | 2 +-
.../X86/MCTargetDesc/X86MCCodeEmitter.cpp | 4 +-
llvm/lib/Target/X86/X86.td | 13 +-
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 22 +-
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 25 +-
llvm/lib/Target/X86/X86InstrACE.td | 133 ++++---
llvm/lib/Target/X86/X86InstrInfo.cpp | 11 +-
llvm/lib/Target/X86/X86LowerAMXType.cpp | 32 +-
llvm/lib/Target/X86/X86LowerTileCopy.cpp | 176 +++++----
llvm/lib/Target/X86/X86RegisterInfo.cpp | 3 +
llvm/lib/TargetParser/X86TargetParser.cpp | 7 +-
llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll | 244 ------------
.../X86/{ACE => ACEV1}/acev1-greedy-ra.ll | 13 +-
.../CodeGen/X86/ACEV1/acev1-intrinsics.ll | 357 ++++++++++++++++++
.../X86/{ACE => ACEV1}/acev1-outer-product.ll | 12 +-
llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll | 67 ++++
.../X86/{ACE => ACEV1}/acev1-tile-basic.ll | 12 +-
.../CodeGen/X86/ACEV1/acev1-tile-spill.ll | 124 ++++++
llvm/test/MC/Disassembler/X86/acev1.txt | 262 +++++++++++++
.../X86/{ACE/ace-att.s => ACEV1/acev1-att.s} | 62 ++-
.../{ACE/ace-error.s => ACEV1/acev1-error.s} | 0
.../{ACE/ace-intel.s => ACEV1/acev1-intel.s} | 38 +-
39 files changed, 1548 insertions(+), 1054 deletions(-)
delete mode 100644 clang/test/CodeGen/X86/ace-api.c
delete mode 100644 clang/test/CodeGen/X86/ace-builtins.c
delete mode 100644 clang/test/CodeGen/X86/ace-inline-asm.c
delete mode 100644 llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
rename llvm/test/CodeGen/X86/{ACE => ACEV1}/acev1-greedy-ra.ll (87%)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
rename llvm/test/CodeGen/X86/{ACE => ACEV1}/acev1-outer-product.ll (94%)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
rename llvm/test/CodeGen/X86/{ACE => ACEV1}/acev1-tile-basic.ll (88%)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
create mode 100644 llvm/test/MC/Disassembler/X86/acev1.txt
rename llvm/test/MC/X86/{ACE/ace-att.s => ACEV1/acev1-att.s} (63%)
rename llvm/test/MC/X86/{ACE/ace-error.s => ACEV1/acev1-error.s} (100%)
rename llvm/test/MC/X86/{ACE/ace-intel.s => ACEV1/acev1-intel.s} (74%)
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index 19bf9e01189920..053146d64fc855 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -223,7 +223,7 @@ let Features = "amx-tile", Attributes = [NoThrow] in {
def tilestored64_internal : X86Builtin<"void(unsigned short, unsigned short, void *, size_t, _Vector<256, int>)">;
}
-let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
+let Features = "amx-tile", Attributes = [NoThrow] in {
def tilezero_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short)">;
}
@@ -256,7 +256,7 @@ let Features = "amx-fp8", Attributes = [NoThrow] in {
def tdphf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
+let Features = "amx-tile", Attributes = [NoThrow] in {
def tile_loadconfig : X86Builtin<"void(void const *)">;
def tile_storeconfig : X86Builtin<"void(void const *)">;
def tilerelease : X86Builtin<"void()">;
@@ -391,52 +391,56 @@ let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>]
def vmovrsw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short const *>)">;
}
-// ACE (AI Compute Extensions) Builtins
-// Tile movement - single builtin handles both immediate and register index forms
-// The lowering code checks if index is constant and selects appropriate instruction
-let Features = "acev1", Attributes = [NoThrow] in {
- def tilesetcol : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
- // ACE tilemovrow writes TO tile (AMX tilemovrow reads FROM tile)
- def tilesetrow : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+// ACE (AI Compute Extensions) builtins.
+// Tile movement - a single builtin handles both the immediate and register index
+// forms; the lowering code checks whether the index is constant and selects the
+// appropriate instruction.
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def tilemovcol_set : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
+ def tilemovrow_set : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
}
-// Tile movement internal (for IR-level operations)
-let Features = "acev1", Attributes = [NoThrow] in {
- def tilesetcol_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
- // ACE tilemovrow_to_tile writes TO tile (AMX tilemovrow reads FROM tile)
- def tilesetrow_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+// Tile movement internal (for IR-level operations).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def tilemovcol_set_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
+ def tilemovrow_set_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
}
-// BSR management
+// BSR management.
let Features = "acev1", Attributes = [NoThrow] in {
def bsrinit : X86Builtin<"void()">;
- def bsrmovf : X86Builtin<"void(_Vector<16, int>, _Vector<16, int>)">;
- def bsrmovh_set : X86Builtin<"void(_Vector<16, int>)">;
- def bsrmovh_get : X86Builtin<"_Vector<16, int>()">;
- def bsrmovl_set : X86Builtin<"void(_Vector<16, int>)">;
- def bsrmovl_get : X86Builtin<"_Vector<16, int>()">;
}
-// Outer products - user-facing (immediate tile IDs)
-let Features = "acev1", Attributes = [NoThrow] in {
- def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4buud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4busd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4bssd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4bsud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def bsrmovf : X86Builtin<"void(_Vector<64, char>, _Vector<64, char>)">;
+ def bsrmovh_set : X86Builtin<"void(_Vector<64, char>)">;
+ def bsrmovh_get : X86Builtin<"_Vector<64, char>()">;
+ def bsrmovl_set : X86Builtin<"void(_Vector<64, char>)">;
+ def bsrmovl_get : X86Builtin<"_Vector<64, char>()">;
}
-// Mixed precision outer products - user-facing
-let Features = "acev1", Attributes = [NoThrow] in {
- def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+// Outer products - user-facing (immediate tile ID, ZMM source values).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
+ def top4buud : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4busd : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bssd : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bsud : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
}
-// Outer products - internal (for IR-level operations with tile values)
-let Features = "acev1", Attributes = [NoThrow] in {
+// Mixed precision outer products - user-facing.
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+}
+
+// Outer products - internal (for IR-level operations with tile values).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def top2bf16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
def top4buud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
def top4busd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
@@ -444,8 +448,8 @@ let Features = "acev1", Attributes = [NoThrow] in {
def top4bsud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
}
-// Mixed precision internal
-let Features = "acev1", Attributes = [NoThrow] in {
+// Mixed precision internal.
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def top4mxhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
def top4mxbhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
def top4mxhbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
diff --git a/clang/lib/Headers/acev1intrin.h b/clang/lib/Headers/acev1intrin.h
index 0491c7f0b383d7..3cbab91759d2f1 100644
--- a/clang/lib/Headers/acev1intrin.h
+++ b/clang/lib/Headers/acev1intrin.h
@@ -21,7 +21,7 @@
/// Load tile configuration from a 64-byte memory location. For ACE
/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1),
-/// ACE tiles have fixed dimensions of 16 rows × 64 bytes, so per-tile
+/// ACE tiles have fixed dimensions of 16 rows x 64 bytes, so per-tile
/// row/column configuration bytes are ignored. If palette_id is zero,
/// tiles return to init state and are zeroed. Invalid configurations
/// result in #GP fault.
@@ -39,7 +39,7 @@ _tile_ace_loadconfig(const void *__config) {
/// Store the current tile configuration to a 64-byte memory location.
/// For ACE (Palette 2), the stored palette_id will be 2 and per-tile
-/// configuration bytes reflect the fixed 16×64 dimensions. If tiles
+/// configuration bytes reflect the fixed 16x64 dimensions. If tiles
/// are not configured, all zeroes are stored.
///
/// \headerfile <immintrin.h>
@@ -65,7 +65,7 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
}
/// Zero the ACE tile specified by "tile". Sets all 1024 bytes
-/// (16 rows × 64 bytes) of the tile register to zero.
+/// (16 rows x 64 bytes) of the tile register to zero.
///
/// \headerfile <immintrin.h>
///
@@ -86,11 +86,11 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// \param dst
/// Destination tile register ID (0-7).
/// \param src
-/// Source ZMM register ID (0-31).
+/// Source ZMM vector (__m512i) containing 16 doublewords.
/// \param idx
/// Column index (0-15). Immediate or register form selected automatically.
#define _tile_setcol(dst, src, idx) \
- __builtin_ia32_tilesetcol((dst), (src), (idx))
+ __builtin_ia32_tilemovcol_set((dst), (src), (idx))
/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written
/// as a horizontal row in the tile at the specified index.
@@ -102,15 +102,16 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// \param dst
/// Destination tile register ID (0-7).
/// \param src
-/// Source ZMM register ID (0-31).
+/// Source ZMM vector (__m512i) containing 16 doublewords.
/// \param idx
/// Row index (0-15). Immediate or register form selected automatically.
#define _tile_setrow(dst, src, idx) \
- __builtin_ia32_tilesetrow((dst), (src), (idx))
+ __builtin_ia32_tilemovrow_set((dst), (src), (idx))
-/// Initialize the Block Scale Register (BSR) to zero. The BSR holds
-/// 32 scaling factors used by mixed-precision outer product instructions
-/// (TOP4MX* variants). Must be called before using BSR-scaled operations.
+/// Initialize the Block Scale Register (BSR), setting all 128 scale bytes to
+/// 0x7F (the E8M0 encoding of 1.0), the same state LDTILECFG leaves it in.
+/// The BSR holds the scale factors used by the mixed-precision outer product
+/// instructions (TOP4MX* variants).
///
/// \headerfile <immintrin.h>
///
@@ -119,67 +120,66 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) {
__builtin_ia32_bsrinit();
}
-/// Load the full BSR (32 scale factors) from two ZMM registers. The low
-/// half (16 factors) comes from __src1, high half from __src2. Each
-/// doubleword contains one scale factor.
+/// Load the full BSR (128 scale bytes) from two ZMM registers. The high half
+/// (A scales) comes from __src1 and the low half (B scales) from __src2.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVF </c> instruction.
///
/// \param __src1
-/// ZMM with scale factors for BSR low half (factors 0-15).
+/// ZMM with the 64 A-operand scale bytes for the BSR high half.
/// \param __src2
-/// ZMM with scale factors for BSR high half (factors 16-31).
+/// ZMM with the 64 B-operand scale bytes for the BSR low half.
static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1,
__m512i __src2) {
- __builtin_ia32_bsrmovf((__v16si)__src1, (__v16si)__src2);
+ __builtin_ia32_bsrmovf((__v64qi)__src1, (__v64qi)__src2);
}
-/// Load the high half of BSR (scale factors 16-31) from a ZMM register.
+/// Load the high half of BSR (64 A-operand scale bytes) from a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
///
/// \param __src
-/// ZMM with 16 scale factors to write to BSR high half.
+/// ZMM with 64 scale bytes to write to the BSR high half.
static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) {
- __builtin_ia32_bsrmovh_set((__v16si)__src);
+ __builtin_ia32_bsrmovh_set((__v64qi)__src);
}
-/// Read the high half of BSR (scale factors 16-31) to a ZMM register.
+/// Read the high half of BSR (64 A-operand scale bytes) to a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
///
/// \returns
-/// ZMM containing 16 scale factors from BSR high half.
+/// ZMM containing the 64 scale bytes from the BSR high half.
static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) {
return (__m512i)__builtin_ia32_bsrmovh_get();
}
-/// Load the low half of BSR (scale factors 0-15) from a ZMM register.
+/// Load the low half of BSR (64 B-operand scale bytes) from a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
///
/// \param __src
-/// ZMM with 16 scale factors to write to BSR low half.
+/// ZMM with 64 scale bytes to write to the BSR low half.
static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) {
- __builtin_ia32_bsrmovl_set((__v16si)__src);
+ __builtin_ia32_bsrmovl_set((__v64qi)__src);
}
-/// Read the low half of BSR (scale factors 0-15) to a ZMM register.
+/// Read the low half of BSR (64 B-operand scale bytes) to a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
///
/// \returns
-/// ZMM containing 16 scale factors from BSR low half.
+/// ZMM containing the 64 scale bytes from the BSR low half.
static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
return (__m512i)__builtin_ia32_bsrmovl_get();
}
@@ -195,13 +195,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 32 BF16 values.
+/// First source ZMM vector (__m512bh) containing 32 BF16 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 32 BF16 values.
+/// Second source ZMM vector (__m512bh) containing 32 BF16 values.
#define _tile_top2bf16ps(dst, src1, src2) \
- __builtin_ia32_top2bf16ps((dst), (src1), (src2))
+ __builtin_ia32_top2bf16ps((dst), (__v32bf)(src1), (__v32bf)(src2))
-/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Compute 4-way outer product of unsigned x unsigned bytes to INT32.
/// Each group of 4 unsigned byte pairs from src1 and src2 produces
/// 4 products accumulated into the destination tile as 32-bit integers.
///
@@ -212,13 +212,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// First source ZMM vector (__m512i) containing 64 unsigned bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// Second source ZMM vector (__m512i) containing 64 unsigned bytes.
#define _tile_top4buud(dst, src1, src2) \
- __builtin_ia32_top4buud((dst), (src1), (src2))
+ __builtin_ia32_top4buud((dst), (__v64qs)(src1), (__v64qs)(src2))
-/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Compute 4-way outer product of unsigned x signed bytes to INT32.
/// Each group of 4 byte pairs (unsigned from src1, signed from src2)
/// produces 4 products accumulated into the destination tile.
///
@@ -229,13 +229,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// First source ZMM vector (__m512i) containing 64 unsigned bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+/// Second source ZMM vector (__m512i) containing 64 signed bytes.
#define _tile_top4busd(dst, src1, src2) \
- __builtin_ia32_top4busd((dst), (src1), (src2))
+ __builtin_ia32_top4busd((dst), (__v64qs)(src1), (__v64qs)(src2))
-/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Compute 4-way outer product of signed x signed bytes to INT32.
/// Each group of 4 signed byte pairs produces 4 products accumulated
/// into the destination tile as 32-bit integers.
///
@@ -246,13 +246,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// First source ZMM vector (__m512i) containing 64 signed bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+/// Second source ZMM vector (__m512i) containing 64 signed bytes.
#define _tile_top4bssd(dst, src1, src2) \
- __builtin_ia32_top4bssd((dst), (src1), (src2))
+ __builtin_ia32_top4bssd((dst), (__v64qs)(src1), (__v64qs)(src2))
-/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Compute 4-way outer product of signed x unsigned bytes to INT32.
/// Each group of 4 byte pairs (signed from src1, unsigned from src2)
/// produces 4 products accumulated into the destination tile.
///
@@ -263,15 +263,15 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// First source ZMM vector (__m512i) containing 64 signed bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// Second source ZMM vector (__m512i) containing 64 unsigned bytes.
#define _tile_top4bsud(dst, src1, src2) \
- __builtin_ia32_top4bsud((dst), (src1), (src2))
+ __builtin_ia32_top4bsud((dst), (__v64qs)(src1), (__v64qs)(src2))
/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
-/// Multiplies HF8 values from src1 with BF8 values from src2, converting
-/// to FP32 and accumulating into the destination tile.
+/// Multiplies HF8 values from both sources, converting to FP32 and
+/// accumulating into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -280,13 +280,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing HF8 values.
+/// First source ZMM vector (__m512i) containing HF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing BF8 values.
+/// Second source ZMM vector (__m512i) containing HF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxhf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxhf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxhf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product with BF8/HF8 format.
/// Multiplies BF8 values from src1 with HF8 (E4M3) values from src2,
@@ -299,13 +299,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing BF8 values.
+/// First source ZMM vector (__m512i) containing BF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing HF8 values.
+/// Second source ZMM vector (__m512i) containing HF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxbhf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbhf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxbhf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product with HF8/BF8 format.
/// Multiplies HF8 (E4M3) values from src1 with BF8 values from src2,
@@ -318,13 +318,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing HF8 values.
+/// First source ZMM vector (__m512i) containing HF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing BF8 values.
+/// Second source ZMM vector (__m512i) containing BF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxhbf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxhbf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxhbf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
/// Multiplies BF8 values from both sources, converting to FP32 and
@@ -337,13 +337,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing BF8 values.
+/// First source ZMM vector (__m512i) containing BF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing BF8 values.
+/// Second source ZMM vector (__m512i) containing BF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxbf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxbf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product of signed INT8 with BSR
/// scaling. Multiplies signed bytes, applies scale factors from the BSR,
@@ -356,15 +356,15 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing signed bytes.
+/// First source ZMM vector (__m512i) containing signed bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing signed bytes.
+/// Second source ZMM vector (__m512i) containing signed bytes.
/// \param imm
/// 8-bit immediate selecting BSR scale factors to apply.
#define _tile_top4mxbssps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbssps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxbssps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
-/// ACE tile type with fixed dimensions (16 rows × 64 bytes = 1024 bytes).
+/// ACE tile type with fixed dimensions (16 rows x 64 bytes = 1024 bytes).
/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1.
typedef int __acetile __attribute__((__vector_size__(1024), __aligned__(64)));
@@ -399,9 +399,9 @@ static __inline__ void __tile_ace_zero(__acetile *dst) {
*dst = __builtin_ia32_tilezero_internal(16, 64);
}
-/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Compute 4-way outer product of unsigned x unsigned bytes to INT32.
/// Multiplies 64 unsigned bytes from each ZMM source, producing a
-/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+/// 16x16 grid of 32-bit sums accumulated into the ACE tile.
///
/// \headerfile <immintrin.h>
///
@@ -420,9 +420,9 @@ static __inline__ void __tile_ace_top4buud(__acetile *dst, __m512i src1,
(__v64qs)src2);
}
-/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Compute 4-way outer product of unsigned x signed bytes to INT32.
/// Multiplies 64 bytes from each ZMM source (unsigned from src1,
-/// signed from src2), producing a 16×16 grid of 32-bit sums.
+/// signed from src2), producing a 16x16 grid of 32-bit sums.
///
/// \headerfile <immintrin.h>
///
@@ -441,9 +441,9 @@ static __inline__ void __tile_ace_top4busd(__acetile *dst, __m512i src1,
(__v64qs)src2);
}
-/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Compute 4-way outer product of signed x signed bytes to INT32.
/// Multiplies 64 signed bytes from each ZMM source, producing a
-/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+/// 16x16 grid of 32-bit sums accumulated into the ACE tile.
///
/// \headerfile <immintrin.h>
///
@@ -462,9 +462,9 @@ static __inline__ void __tile_ace_top4bssd(__acetile *dst, __m512i src1,
(__v64qs)src2);
}
-/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Compute 4-way outer product of signed x unsigned bytes to INT32.
/// Multiplies 64 bytes from each ZMM source (signed from src1,
-/// unsigned from src2), producing a 16×16 grid of 32-bit sums.
+/// unsigned from src2), producing a 16x16 grid of 32-bit sums.
///
/// \headerfile <immintrin.h>
///
@@ -484,7 +484,7 @@ static __inline__ void __tile_ace_top4bsud(__acetile *dst, __m512i src1,
}
/// Compute 2-way outer product of BF16 to FP32.
-/// Multiplies 32 BF16 values from each source, producing a 16×16 grid
+/// Multiplies 32 BF16 values from each source, producing a 16x16 grid
/// of FP32 sums accumulated into the ACE tile.
///
/// \headerfile <immintrin.h>
@@ -620,7 +620,7 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
unsigned int idx) {
- *dst = __builtin_ia32_tilesetcol_internal(16, 64, (__v16si)src, idx);
+ *dst = __builtin_ia32_tilemovcol_set_internal(16, 64, (__v16si)src, idx);
}
/// Write a ZMM vector as a row in an ACE tile.
@@ -639,7 +639,7 @@ static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_setrow(__acetile *dst, __m512i src,
unsigned int idx) {
- *dst = __builtin_ia32_tilesetrow_internal(16, 64, (__v16si)src, idx);
+ *dst = __builtin_ia32_tilemovrow_set_internal(16, 64, (__v16si)src, idx);
}
/// Read a row from an ACE tile to a ZMM vector.
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index 1599672ec5404b..3b598225240c3a 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -535,6 +535,18 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tcvtrowps2phl:
case X86::BI__builtin_ia32_tcvtrowd2ps:
case X86::BI__builtin_ia32_tilemovrow:
+ // The ACE outer products name only the accumulator tile; their two sources
+ // are ZMM values, so neither the range nor the duplicate rule applies there.
+ case X86::BI__builtin_ia32_top2bf16ps:
+ case X86::BI__builtin_ia32_top4buud:
+ case X86::BI__builtin_ia32_top4busd:
+ case X86::BI__builtin_ia32_top4bssd:
+ case X86::BI__builtin_ia32_top4bsud:
+ case X86::BI__builtin_ia32_top4mxhf8ps:
+ case X86::BI__builtin_ia32_top4mxbhf8ps:
+ case X86::BI__builtin_ia32_top4mxhbf8ps:
+ case X86::BI__builtin_ia32_top4mxbf8ps:
+ case X86::BI__builtin_ia32_top4mxbssps:
return CheckBuiltinTileArgumentsRange(TheCall, 0);
case X86::BI__builtin_ia32_tdpbssd:
case X86::BI__builtin_ia32_tdpbsud:
@@ -548,11 +560,6 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tdpbhf8ps:
case X86::BI__builtin_ia32_tdphbf8ps:
case X86::BI__builtin_ia32_tdphf8ps:
- case X86::BI__builtin_ia32_top2bf16ps:
- case X86::BI__builtin_ia32_top4buud:
- case X86::BI__builtin_ia32_top4busd:
- case X86::BI__builtin_ia32_top4bssd:
- case X86::BI__builtin_ia32_top4bsud:
return CheckBuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
case X86::BI__builtin_ia32_tcvtrowps2bf16hi:
case X86::BI__builtin_ia32_tcvtrowps2bf16li:
diff --git a/clang/test/CodeGen/X86/ace-api.c b/clang/test/CodeGen/X86/ace-api.c
deleted file mode 100644
index 8f82fbf66e39ad..00000000000000
--- a/clang/test/CodeGen/X86/ace-api.c
+++ /dev/null
@@ -1,145 +0,0 @@
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
-// RUN: -emit-llvm -o - -Werror | FileCheck %s
-
-// Test ACE struct-based API functions using __acetile type
-// ACE v1 uses fixed 16x64 tile dimensions (Palette 2)
-
-#include <immintrin.h>
-
-// CHECK-LABEL: @test_ace_top4buud
-// CHECK: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %{{.*}})
-// CHECK: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-// CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %{{.*}})
-void test_ace_top4buud(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4buud(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4busd
-// CHECK: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-void test_ace_top4busd(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4busd(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4bssd
-// CHECK: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-void test_ace_top4bssd(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4bssd(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4bsud
-// CHECK: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-void test_ace_top4bsud(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4bsud(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top2bf16ps
-// CHECK: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
-void test_ace_top2bf16ps(__acetile *dst, __m512bh a, __m512bh b) {
- __tile_ace_top2bf16ps(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4mxhf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxhf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxhf8ps(dst, a, b, 5);
-}
-
-// CHECK-LABEL: @test_ace_top4mxbhf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxbhf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxbhf8ps(dst, a, b, 3);
-}
-
-// CHECK-LABEL: @test_ace_top4mxhbf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxhbf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxhbf8ps(dst, a, b, 7);
-}
-
-// CHECK-LABEL: @test_ace_top4mxbf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxbf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxbf8ps(dst, a, b, 1);
-}
-
-// CHECK-LABEL: @test_ace_top4mxbssps
-// CHECK: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxbssps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxbssps(dst, a, b, 9);
-}
-
-// CHECK-LABEL: @test_ace_setcol
-// CHECK: call x86_amx @llvm.x86.tilesetcol.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
-void test_ace_setcol(__acetile *dst, __m512i src) {
- __tile_ace_setcol(dst, src, 3);
-}
-
-// CHECK-LABEL: @test_ace_setrow
-// CHECK: call x86_amx @llvm.x86.tilesetrow.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
-void test_ace_setrow(__acetile *dst, __m512i src) {
- __tile_ace_setrow(dst, src, 5);
-}
-
-// CHECK-LABEL: @test_ace_getrow
-// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512i test_ace_getrow(__acetile *src) {
- return __tile_ace_getrow(src, 7);
-}
-
-// CHECK-LABEL: @test_ace_zero
-// CHECK: call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
-void test_ace_zero(__acetile *dst) {
- __tile_ace_zero(dst);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowd2ps
-// CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512 test_ace_cvtrowd2ps(__acetile *src) {
- return __tile_ace_cvtrowd2ps(src, 2);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2bf16h
-// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512bh test_ace_cvtrowps2bf16h(__acetile *src) {
- return __tile_ace_cvtrowps2bf16h(src, 4);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2bf16l
-// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512bh test_ace_cvtrowps2bf16l(__acetile *src) {
- return __tile_ace_cvtrowps2bf16l(src, 6);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2phh
-// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512h test_ace_cvtrowps2phh(__acetile *src) {
- return __tile_ace_cvtrowps2phh(src, 8);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2phl
-// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512h test_ace_cvtrowps2phl(__acetile *src) {
- return __tile_ace_cvtrowps2phl(src, 10);
-}
-
-// CHECK-LABEL: @test_ace_workflow
-// CHECK: call x86_amx @llvm.x86.tilezero.internal
-// CHECK: call x86_amx @llvm.x86.tilesetrow.internal
-// CHECK: call x86_amx @llvm.x86.top4buud.internal
-// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
-void test_ace_workflow(__m512i *input, __m512i *output) {
- __acetile acc;
-
- // Zero the tile
- __tile_ace_zero(&acc);
-
- // Initialize with data via setrow
- __tile_ace_setrow(&acc, input[0], 0);
-
- // Perform outer product computation
- __tile_ace_top4buud(&acc, input[1], input[2]);
-
- // Extract result via getrow
- output[0] = __tile_ace_getrow(&acc, 0);
-}
diff --git a/clang/test/CodeGen/X86/ace-builtins.c b/clang/test/CodeGen/X86/ace-builtins.c
deleted file mode 100644
index edaa80c78486f0..00000000000000
--- a/clang/test/CodeGen/X86/ace-builtins.c
+++ /dev/null
@@ -1,119 +0,0 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +acev1 -target-feature +avx512f -emit-llvm -o - | FileCheck %s
-
-#include <immintrin.h>
-
-// BSR Management Tests
-
-// CHECK-LABEL: @test_bsrinit
-// CHECK: call void @llvm.x86.bsrinit()
-void test_bsrinit(void) {
- _bsr0_init();
-}
-
-// CHECK-LABEL: @test_bsrmovf
-// CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_bsrmovf(__m512i a, __m512i b) {
- _bsr0_movf(a, b);
-}
-
-// CHECK-LABEL: @test_bsrmovh_set
-// CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
-void test_bsrmovh_set(__m512i a) {
- _bsr0_movh_set(a);
-}
-
-// CHECK-LABEL: @test_bsrmovh_get
-// CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
-__m512i test_bsrmovh_get(void) {
- return _bsr0_movh_get();
-}
-
-// CHECK-LABEL: @test_bsrmovl_set
-// CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
-void test_bsrmovl_set(__m512i a) {
- _bsr0_movl_set(a);
-}
-
-// CHECK-LABEL: @test_bsrmovl_get
-// CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
-__m512i test_bsrmovl_get(void) {
- return _bsr0_movl_get();
-}
-
-// Tile Movement Tests
-
-// CHECK-LABEL: @test_tilemovcol
-// CHECK: call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %{{.*}}, i32 5)
-void test_tilemovcol(__m512i src) {
- _tile_setcol(0, src, 5);
-}
-
-// CHECK-LABEL: @test_tilemovrow_to_tile
-// CHECK: call void @llvm.x86.tilesetrow(i8 0, <16 x i32> %{{.*}}, i32 5)
-void test_tilemovrow_to_tile(__m512i src) {
- _tile_setrow(0, src, 5);
-}
-
-// Outer Product Tests
-
-// CHECK-LABEL: @test_top2bf16ps
-// CHECK: call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
-void test_top2bf16ps(void) {
- _tile_top2bf16ps(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4buud
-// CHECK: call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
-void test_top4buud(void) {
- _tile_top4buud(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4busd
-// CHECK: call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
-void test_top4busd(void) {
- _tile_top4busd(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4bssd
-// CHECK: call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
-void test_top4bssd(void) {
- _tile_top4bssd(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4bsud
-// CHECK: call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
-void test_top4bsud(void) {
- _tile_top4bsud(0, 1, 2);
-}
-
-// Mixed Precision Outer Product Tests
-
-// CHECK-LABEL: @test_top4mxhf8ps
-// CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxhf8ps(void) {
- _tile_top4mxhf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxbhf8ps
-// CHECK: call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxbhf8ps(void) {
- _tile_top4mxbhf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxhbf8ps
-// CHECK: call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxhbf8ps(void) {
- _tile_top4mxhbf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxbf8ps
-// CHECK: call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxbf8ps(void) {
- _tile_top4mxbf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxbssps
-// CHECK: call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxbssps(void) {
- _tile_top4mxbssps(0, 1, 2, 5);
-}
diff --git a/clang/test/CodeGen/X86/ace-inline-asm.c b/clang/test/CodeGen/X86/ace-inline-asm.c
deleted file mode 100644
index 27e36d84149fcc..00000000000000
--- a/clang/test/CodeGen/X86/ace-inline-asm.c
+++ /dev/null
@@ -1,85 +0,0 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f \
-// RUN: -emit-llvm -o - -Wall -Werror | FileCheck %s
-
-// Test ACE instructions in inline assembly
-
-// BSR Management Instructions
-
-// CHECK-LABEL: @test_bsrinit_asm
-// CHECK: call void asm sideeffect "bsrinit", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrinit_asm(void) {
- __asm__ volatile("bsrinit");
-}
-
-// CHECK-LABEL: @test_bsrmovf_asm
-// CHECK: call void asm sideeffect "bsrmovf %zmm1, %zmm0", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrmovf_asm(void) {
- __asm__ volatile("bsrmovf %%zmm1, %%zmm0" :::);
-}
-
-// CHECK-LABEL: @test_bsrmovh_asm
-// CHECK: call void asm sideeffect "bsrmovh %zmm2", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrmovh_asm(void) {
- __asm__ volatile("bsrmovh %%zmm2" :::);
-}
-
-// CHECK-LABEL: @test_bsrmovl_asm
-// CHECK: call void asm sideeffect "bsrmovl %zmm3", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrmovl_asm(void) {
- __asm__ volatile("bsrmovl %%zmm3" :::);
-}
-
-// Tile Movement Instructions
-
-// CHECK-LABEL: @test_tilemovcol_asm
-// CHECK: call void asm sideeffect "tilemovcol $$$$5, %zmm0, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
-void test_tilemovcol_asm(void) {
- __asm__ volatile("tilemovcol $$5, %%zmm0, %%tmm1" ::: "tmm1");
-}
-
-// CHECK-LABEL: @test_tilemovrow_asm
-// CHECK: call void asm sideeffect "tilemovrow $$$$3, %zmm0, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
-void test_tilemovrow_asm(void) {
- __asm__ volatile("tilemovrow $$3, %%zmm0, %%tmm2" ::: "tmm2");
-}
-
-// Outer Product Instructions
-
-// CHECK-LABEL: @test_top4buud_asm
-// CHECK: call void asm sideeffect "top4buud %zmm1, %zmm0, %tmm0", "~{tmm0},~{dirflag},~{fpsr},~{flags}"()
-void test_top4buud_asm(void) {
- __asm__ volatile("top4buud %%zmm1, %%zmm0, %%tmm0" ::: "tmm0");
-}
-
-// CHECK-LABEL: @test_top4bssd_asm
-// CHECK: call void asm sideeffect "top4bssd %zmm2, %zmm1, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
-void test_top4bssd_asm(void) {
- __asm__ volatile("top4bssd %%zmm2, %%zmm1, %%tmm1" ::: "tmm1");
-}
-
-// CHECK-LABEL: @test_top2bf16ps_asm
-// CHECK: call void asm sideeffect "top2bf16ps %zmm3, %zmm2, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
-void test_top2bf16ps_asm(void) {
- __asm__ volatile("top2bf16ps %%zmm3, %%zmm2, %%tmm2" ::: "tmm2");
-}
-
-// Mixed Precision Outer Product Instructions
-
-// CHECK-LABEL: @test_top4mxhfbps_asm
-// CHECK: call void asm sideeffect "top4mxhfbps $$$$5, %zmm1, %zmm0, %tmm3", "~{tmm3},~{dirflag},~{fpsr},~{flags}"()
-void test_top4mxhfbps_asm(void) {
- __asm__ volatile("top4mxhfbps $$5, %%zmm1, %%zmm0, %%tmm3" ::: "tmm3");
-}
-
-// Full ACE operation sequence in inline assembly
-// CHECK-LABEL: @test_ace_sequence_asm
-// CHECK: call void asm sideeffect
-void test_ace_sequence_asm(void) {
- __asm__ volatile(
- "bsrinit \n\t"
- "top4buud %%zmm1, %%zmm0, %%tmm0 \n\t"
- "top4bssd %%zmm3, %%zmm2, %%tmm1 \n\t"
- ::: "memory", "tmm0", "tmm1"
- );
-}
diff --git a/clang/test/CodeGen/X86/acev1.c b/clang/test/CodeGen/X86/acev1.c
index 8f487b8ea2f51a..50efae140ab569 100644
--- a/clang/test/CodeGen/X86/acev1.c
+++ b/clang/test/CodeGen/X86/acev1.c
@@ -1,22 +1,25 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
// Tests macro-based ACE v1 intrinsics emit correct LLVM IR
#include <immintrin.h>
-void test_acev1_outer_products(void) {
+// The outer products take the accumulator tile by ID, but their two sources are
+// ordinary ZMM values left to the register allocator.
+void test_acev1_outer_products(__m512bh bf1, __m512bh bf2, __m512i i1,
+ __m512i i2) {
// CHECK-LABEL: @test_acev1_outer_products
- // CHECK: call void @llvm.x86.top2bf16ps(i8 0,
- // CHECK: call void @llvm.x86.top4buud(i8 1,
- // CHECK: call void @llvm.x86.top4busd(i8 2,
- // CHECK: call void @llvm.x86.top4bssd(i8 3,
- // CHECK: call void @llvm.x86.top4bsud(i8 4,
- _tile_top2bf16ps(0, 1, 2);
- _tile_top4buud(1, 2, 3);
- _tile_top4busd(2, 3, 4);
- _tile_top4bssd(3, 4, 5);
- _tile_top4bsud(4, 5, 6);
+ // CHECK: call void @llvm.x86.top2bf16ps(i8 0, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
+ // CHECK: call void @llvm.x86.top4buud(i8 1, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.top4busd(i8 2, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.top4bssd(i8 3, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.top4bsud(i8 4, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ _tile_top2bf16ps(0, bf1, bf2);
+ _tile_top4buud(1, i1, i2);
+ _tile_top4busd(2, i1, i2);
+ _tile_top4bssd(3, i1, i2);
+ _tile_top4bsud(4, i1, i2);
}
void test_acev1_tile_config(void *data) {
@@ -31,6 +34,16 @@ void test_acev1_tile_config(void *data) {
_tile_ace_zero(0);
}
+// ACE v1 has no TILELOADD/TILESTORED, so the macro API moves data through
+// TILEMOVROW/TILEMOVCOL with an explicit tile register ID.
+void test_acev1_tile_movement(__m512i src) {
+ // CHECK-LABEL: @test_acev1_tile_movement
+ // CHECK: call void @llvm.x86.tilemovrow.set(i8 0, <16 x i32> %{{.*}}, i32 5)
+ // CHECK: call void @llvm.x86.tilemovcol.set(i8 1, <16 x i32> %{{.*}}, i32 3)
+ _tile_setrow(0, src, 5);
+ _tile_setcol(1, src, 3);
+}
+
void test_acev1_bsr(void) {
// CHECK-LABEL: @test_acev1_bsr
// CHECK: call void @llvm.x86.bsrinit()
diff --git a/clang/test/CodeGen/X86/acev1_api.c b/clang/test/CodeGen/X86/acev1_api.c
index acb12dae30cc4e..55127fc23fa15b 100644
--- a/clang/test/CodeGen/X86/acev1_api.c
+++ b/clang/test/CodeGen/X86/acev1_api.c
@@ -1,9 +1,9 @@
// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
+// RUN: -target-feature +acev1 -target-feature +avx10.1 \
// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
-// Tests ACE v1 APIs using __acetile type (fixed 16x64 dimensions)
-// Note: ACE v1 does not have TILELOADD/TILESTORED - uses ACE-specific tile operations
+// Tests ACE v1 APIs using __acetile type (fixed 16x64 dimensions).
+// ACE v1 has no TILELOADD/TILESTORED, so it uses ACE-specific tile operations.
#include <immintrin.h>
@@ -20,7 +20,7 @@ void test_tile_ace_zero(void) {
void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4buud
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4buud(&dst, src1, src2);
}
@@ -29,7 +29,7 @@ void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4busd
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4busd(&dst, src1, src2);
}
@@ -38,7 +38,7 @@ void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4bssd
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4bssd(&dst, src1, src2);
}
@@ -47,7 +47,7 @@ void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4bsud
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4bsud(&dst, src1, src2);
}
@@ -56,7 +56,7 @@ void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top2bf16ps(__acetile dst, __m512bh src1, __m512bh src2) {
// CHECK-LABEL: @test_tile_ace_top2bf16ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top2bf16ps(&dst, src1, src2);
}
@@ -109,14 +109,14 @@ void test_tile_ace_top4mxbssps(__acetile dst, __m512i src1, __m512i src2) {
// Test tile setrow (ACE-specific: ZMM to tile row)
void test_tile_ace_setrow(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__tile_ace_setrow(&dst, src, 0);
}
// Test tile setcol (ACE-specific: ZMM to tile column)
void test_tile_ace_setcol(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__tile_ace_setcol(&dst, src, 0);
}
@@ -127,6 +127,41 @@ __m512i test_tile_ace_getrow(__acetile src) {
return __tile_ace_getrow(&src, 0);
}
+// Test tile row read with INT32 to FP32 conversion (ACE-specific)
+__m512 test_tile_ace_cvtrowd2ps(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowd2ps
+ // CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowd2ps(&src, 2);
+}
+
+// Test tile row read with FP32 to BF16 conversion, high half of each dword
+__m512bh test_tile_ace_cvtrowps2bf16h(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2bf16h
+ // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2bf16h(&src, 4);
+}
+
+// Test tile row read with FP32 to BF16 conversion, low half of each dword
+__m512bh test_tile_ace_cvtrowps2bf16l(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2bf16l
+ // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2bf16l(&src, 6);
+}
+
+// Test tile row read with FP32 to FP16 conversion, high half of each dword
+__m512h test_tile_ace_cvtrowps2phh(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2phh
+ // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2phh(&src, 8);
+}
+
+// Test tile row read with FP32 to FP16 conversion, low half of each dword
+__m512h test_tile_ace_cvtrowps2phl(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2phl
+ // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2phl(&src, 10);
+}
+
// Integration test: ACE computation workflow
void test_ace_workflow(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_ace_workflow
diff --git a/clang/test/CodeGen/X86/acev1_bsr.c b/clang/test/CodeGen/X86/acev1_bsr.c
index d68ecdcb59f7d2..be7d1d8dfa4162 100644
--- a/clang/test/CodeGen/X86/acev1_bsr.c
+++ b/clang/test/CodeGen/X86/acev1_bsr.c
@@ -1,7 +1,7 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
-// Tests ACE v1 BSR (Block Sparse Register) operations emit correct LLVM IR
+// Tests ACE v1 BSR (Block Scale Register) operations emit correct LLVM IR
#include <immintrin.h>
#include <stdint.h>
@@ -14,31 +14,31 @@ void test_bsr_init(void) {
void test_bsr_movf(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_bsr_movf
- // CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+ // CHECK: call void @llvm.x86.bsrmovf(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
_bsr0_movf(src1, src2);
}
void test_bsr_movh_set(__m512i src) {
// CHECK-LABEL: @test_bsr_movh_set
- // CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
+ // CHECK: call void @llvm.x86.bsrmovh.set(<64 x i8> %{{.*}})
_bsr0_movh_set(src);
}
__m512i test_bsr_movh_get(void) {
// CHECK-LABEL: @test_bsr_movh_get
- // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovh.get()
return _bsr0_movh_get();
}
void test_bsr_movl_set(__m512i src) {
// CHECK-LABEL: @test_bsr_movl_set
- // CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
+ // CHECK: call void @llvm.x86.bsrmovl.set(<64 x i8> %{{.*}})
_bsr0_movl_set(src);
}
__m512i test_bsr_movl_get(void) {
// CHECK-LABEL: @test_bsr_movl_get
- // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovl.get()
return _bsr0_movl_get();
}
@@ -47,8 +47,8 @@ void test_bsr_sequence(void) {
// CHECK-LABEL: @test_bsr_sequence
// CHECK: call void @llvm.x86.bsrinit()
// CHECK: call void @llvm.x86.bsrmovf
- // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
- // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovh.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovl.get()
_bsr0_init();
__m512i zmm1 = _mm512_set1_epi32(1);
diff --git a/clang/test/CodeGen/X86/acev1_errors.c b/clang/test/CodeGen/X86/acev1_errors.c
index c928882b62f4a3..0776e3c026d3e9 100644
--- a/clang/test/CodeGen/X86/acev1_errors.c
+++ b/clang/test/CodeGen/X86/acev1_errors.c
@@ -1,17 +1,17 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -fsyntax-only -verify
+// RUN: -target-feature +avx10.1 -fsyntax-only -verify
// Tests compile-time semantic errors for ACE v1 intrinsics
#include <immintrin.h>
-void test_tile_range_errors(void) {
+void test_tile_range_errors(__m512bh bf, __m512i i) {
// Tile index must be in range [0, 7]
_tile_ace_zero(8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
_tile_ace_zero(16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
// Outer product tile index errors
- _tile_top4buud(8, 0, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_top4bssd(16, 0, 0); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
- _tile_top2bf16ps(9, 0, 0); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
+ _tile_top4buud(8, i, i); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_top4bssd(16, i, i); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ _tile_top2bf16ps(9, bf, bf); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
}
diff --git a/clang/test/CodeGen/X86/acev1_inline_asm.c b/clang/test/CodeGen/X86/acev1_inline_asm.c
index 1dbf7d76a53554..57da62d269b756 100644
--- a/clang/test/CodeGen/X86/acev1_inline_asm.c
+++ b/clang/test/CodeGen/X86/acev1_inline_asm.c
@@ -1,20 +1,24 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
// Tests inline assembly with ACE v1 tile registers
#include <immintrin.h>
+// TILESTORED is not available in palette 2, so the result leaves the tile one
+// row at a time through TILEMOVROW and is stored from the ZMM.
void test_acev1_inline_asm_outer_product(void) {
// CHECK-LABEL: @test_acev1_inline_asm_outer_product
// CHECK: call void asm sideeffect "tilezero %tmm0
// CHECK: top4buud %zmm0, %zmm1, %tmm0
- // CHECK: tilestored %tmm0
+ // CHECK: tilemovrow $$0, %tmm0, %zmm2
+ // CHECK: vmovups %zmm2, 0(%rdi)
__asm__ volatile (
- "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm0 \n\t"
"top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
- "tilestored %%tmm0, 0(%%rdi) \n\t"
- ::: "memory", "tmm0", "zmm0", "zmm1"
+ "tilemovrow $0, %%tmm0, %%zmm2 \n\t"
+ "vmovups %%zmm2, 0(%%rdi) \n\t"
+ ::: "memory", "tmm0", "zmm0", "zmm1", "zmm2"
);
}
@@ -53,6 +57,59 @@ void test_acev1_inline_asm_tile_config(void *cfg) {
);
}
+// Tile movement is the only way in or out of an ACE tile, since ACE v1 has no
+// TILELOADD/TILESTORED. Immediate index form.
+void test_acev1_inline_asm_tile_movement(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_tile_movement
+ // CHECK: call void asm sideeffect "tilemovrow $$3, %zmm0, %tmm2
+ // CHECK: tilemovcol $$5, %zmm1, %tmm3
+ __asm__ volatile (
+ "tilemovrow $3, %%zmm0, %%tmm2 \n\t"
+ "tilemovcol $5, %%zmm1, %%tmm3 \n\t"
+ ::: "memory", "tmm2", "tmm3", "zmm0", "zmm1"
+ );
+}
+
+// Tile movement with the row/column index supplied in a GPR.
+void test_acev1_inline_asm_tile_movement_reg_index(int idx) {
+ // CHECK-LABEL: @test_acev1_inline_asm_tile_movement_reg_index
+ // CHECK: call void asm sideeffect "tilemovrow $0, %zmm0, %tmm0
+ // CHECK: tilemovcol $0, %zmm0, %tmm1
+ __asm__ volatile (
+ "tilemovrow %0, %%zmm0, %%tmm0 \n\t"
+ "tilemovcol %0, %%zmm0, %%tmm1 \n\t"
+ :: "r"(idx) : "memory", "tmm0", "tmm1", "zmm0"
+ );
+}
+
+// BSR high/low halves, both set (ZMM to BSR) and get (BSR to ZMM) directions.
+void test_acev1_inline_asm_bsr_movh_movl(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_bsr_movh_movl
+ // CHECK: call void asm sideeffect "bsrmovh %zmm1, %bsr0
+ // CHECK: bsrmovl %zmm2, %bsr0
+ // CHECK: bsrmovh %bsr0, %zmm3
+ // CHECK: bsrmovl %bsr0, %zmm4
+ __asm__ volatile (
+ "bsrmovh %%zmm1, %%bsr0 \n\t"
+ "bsrmovl %%zmm2, %%bsr0 \n\t"
+ "bsrmovh %%bsr0, %%zmm3 \n\t"
+ "bsrmovl %%bsr0, %%zmm4 \n\t"
+ ::: "memory", "zmm1", "zmm2", "zmm3", "zmm4"
+ );
+}
+
+// MX outer product, which takes the BSR scaling mode as an immediate.
+void test_acev1_inline_asm_mx_outer_product(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_mx_outer_product
+ // CHECK: call void asm sideeffect "tilezero %tmm1
+ // CHECK: top4mxhf8ps $$7, %zmm3, %zmm2, %tmm1
+ __asm__ volatile (
+ "tilezero %%tmm1 \n\t"
+ "top4mxhf8ps $7, %%zmm3, %%zmm2, %%tmm1 \n\t"
+ ::: "memory", "tmm1", "zmm2", "zmm3"
+ );
+}
+
void test_acev1_inline_asm_all_outer_products(void) {
// CHECK-LABEL: @test_acev1_inline_asm_all_outer_products
// CHECK: top4buud
diff --git a/clang/test/CodeGen/X86/acev1_mxfp8.c b/clang/test/CodeGen/X86/acev1_mxfp8.c
index e99ca9dda4067f..dc0d04cdf76ab8 100644
--- a/clang/test/CodeGen/X86/acev1_mxfp8.c
+++ b/clang/test/CodeGen/X86/acev1_mxfp8.c
@@ -1,51 +1,51 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
// Tests ACE v1 MX FP8 outer product macro-based intrinsics
#include <immintrin.h>
-void test_top4mxhf8ps(void) {
+void test_top4mxhf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxhf8ps
- // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 0)
- _tile_top4mxhf8ps(0, 1, 2, 0);
+ // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
+ _tile_top4mxhf8ps(0, src1, src2, 0);
}
-void test_top4mxbhf8ps(void) {
+void test_top4mxbhf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxbhf8ps
- // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, i8 2, i8 3, i8 1)
- _tile_top4mxbhf8ps(1, 2, 3, 1);
+ // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 1)
+ _tile_top4mxbhf8ps(1, src1, src2, 1);
}
-void test_top4mxhbf8ps(void) {
+void test_top4mxhbf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxhbf8ps
- // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, i8 3, i8 4, i8 0)
- _tile_top4mxhbf8ps(2, 3, 4, 0);
+ // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
+ _tile_top4mxhbf8ps(2, src1, src2, 0);
}
-void test_top4mxbf8ps(void) {
+void test_top4mxbf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxbf8ps
- // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, i8 4, i8 5, i8 1)
- _tile_top4mxbf8ps(3, 4, 5, 1);
+ // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 1)
+ _tile_top4mxbf8ps(3, src1, src2, 1);
}
-void test_top4mxbssps(void) {
+void test_top4mxbssps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxbssps
- // CHECK: call void @llvm.x86.top4mxbssps(i8 4, i8 5, i8 6, i8 0)
- _tile_top4mxbssps(4, 5, 6, 0);
+ // CHECK: call void @llvm.x86.top4mxbssps(i8 4, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
+ _tile_top4mxbssps(4, src1, src2, 0);
}
// Test all MX FP8 variants in sequence
-void test_all_mxfp8_variants(void) {
+void test_all_mxfp8_variants(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_all_mxfp8_variants
// CHECK: call void @llvm.x86.top4mxhf8ps
// CHECK: call void @llvm.x86.top4mxbhf8ps
// CHECK: call void @llvm.x86.top4mxhbf8ps
// CHECK: call void @llvm.x86.top4mxbf8ps
// CHECK: call void @llvm.x86.top4mxbssps
- _tile_top4mxhf8ps(0, 1, 2, 0);
- _tile_top4mxbhf8ps(3, 4, 5, 0);
- _tile_top4mxhbf8ps(6, 7, 0, 0);
- _tile_top4mxbf8ps(1, 2, 3, 1);
- _tile_top4mxbssps(4, 5, 6, 1);
+ _tile_top4mxhf8ps(0, src1, src2, 0);
+ _tile_top4mxbhf8ps(3, src1, src2, 0);
+ _tile_top4mxhbf8ps(6, src1, src2, 0);
+ _tile_top4mxbf8ps(1, src1, src2, 1);
+ _tile_top4mxbssps(4, src1, src2, 1);
}
diff --git a/clang/test/CodeGen/X86/acev1_tile_movement.c b/clang/test/CodeGen/X86/acev1_tile_movement.c
index f1383104f5db23..a2648b41b1d382 100644
--- a/clang/test/CodeGen/X86/acev1_tile_movement.c
+++ b/clang/test/CodeGen/X86/acev1_tile_movement.c
@@ -1,25 +1,26 @@
// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f \
+// RUN: -target-feature +acev1 -target-feature +avx10.1 \
// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
-// Tests ACE v1 tile movement operations (ZMM <-> Tile row/col)
-// Note: ACE v1 does not have TILELOADD/TILESTORED - uses TILEMOVROW/TILEMOVCOL for data movement
+// Tests ACE v1 tile movement operations (ZMM <-> Tile row/col).
+// ACE v1 has no TILELOADD/TILESTORED, so it moves data with
+// TILEMOVROW/TILEMOVCOL.
#include <immintrin.h>
// Test tile setrow - write ZMM to tile row (ACE-specific)
void test_tile_ace_setrow(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__tile_ace_setrow(&dst, src, 0);
}
// Test tile setrow with different row indices
void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow_indices
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__tile_ace_setrow(&dst, src, 0);
__tile_ace_setrow(&dst, src, 8);
__tile_ace_setrow(&dst, src, 15);
@@ -28,15 +29,15 @@ void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
// Test tile setcol - write ZMM to tile column (ACE-specific)
void test_tile_ace_setcol(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__tile_ace_setcol(&dst, src, 0);
}
// Test tile setcol with different column indices
void test_tile_ace_setcol_indices(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol_indices
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__tile_ace_setcol(&dst, src, 0);
__tile_ace_setcol(&dst, src, 8);
}
@@ -63,7 +64,7 @@ void test_tile_ace_getrow_indices(__acetile src, __m512i *out) {
void test_tile_init_via_setrow(void) {
// CHECK-LABEL: @test_tile_init_via_setrow
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__acetile tile;
__tile_ace_zero(&tile);
@@ -75,9 +76,9 @@ void test_tile_init_via_setrow(void) {
void test_tile_movement_combined(void) {
// CHECK-LABEL: @test_tile_movement_combined
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__acetile tile;
__tile_ace_zero(&tile);
@@ -96,7 +97,7 @@ void test_tile_movement_combined(void) {
void test_ace_data_workflow(__m512i *input, __m512i *output) {
// CHECK-LABEL: @test_ace_data_workflow
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
// CHECK: call x86_amx @llvm.x86.top4buud.internal
// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
__acetile tile;
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index 9824ffe76c0a3c..aaf9d8618a2a3e 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -507,7 +507,7 @@ __attribute__((target("fxsr")))
void f_fxsr(void) {}
// CHECK: [[f_general_regs_only]] = {{.*}}"target-cpu"="i686"
-// CHECK-SAME: "target-features"="+cmov,+cx8,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-mmx,-pclmul,-sha,-sha512,-sm3,-sm4,-sse,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-x87,-xop"
+// CHECK-SAME: "target-features"="+cmov,+cx8,-acev1,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-mmx,-pclmul,-sha,-sha512,-sm3,-sm4,-sse,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-x87,-xop"
__attribute__((target("general-regs-only")))
void f_general_regs_only(void) {}
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 636e995b3f6f1a..fb7f6d68cc1774 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -547,13 +547,33 @@
// NO-AMX-AVX512-NOT: #define __AMX_AVX512__ 1
+// ACE v1 requires AVX10.1 as its vector baseline and AMX-TILE for the tile
+// registers and tile management instructions, but nothing above either.
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 -x c \
-// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 %s
-// ACEV1: #define __ACEV1__ 1
+// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 \
+// RUN: --implicit-check-not=__AVX10_2 --implicit-check-not=__AMX_AVX512__ %s
+// ACEV1-DAG: #define __ACEV1__ 1
+// ACEV1-DAG: #define __AVX10_1__ 1
+// ACEV1-DAG: #define __AVX512F__ 1
+// ACEV1-DAG: #define __AMX_TILE__ 1
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mno-acev1 -x c \
// RUN: -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
// NO-ACEV1-NOT: #define __ACEV1__ 1
+// AVX10.2 is not required: disabling it keeps ACE v1 and its baseline.
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-avx10.2 -x c -E -dM -o - %s | FileCheck \
+// RUN: -check-prefix=ACEV1-BASELINE %s
+// ACEV1-BASELINE-DAG: #define __ACEV1__ 1
+// ACEV1-BASELINE-DAG: #define __AVX10_1__ 1
+// ACEV1-BASELINE-DAG: #define __AMX_TILE__ 1
+
+// Both baselines are necessary: dropping either drops ACE v1 with it.
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-avx10.1 -x c -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-amx-tile -x c -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+
// RUN: %clang -target i386-unknown-unknown -march=atom -mavxvnni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVXVNNI %s
// AVXVNNI: #define __AVX2__ 1
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 43a0885bd8bcd8..8276e2ce7ad2c7 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5745,110 +5745,105 @@ let TargetPrefix = "x86" in {
Intrinsic<[], [], []>;
// BSRMOVF - Move Full to BSR
- // Takes two ZMM registers as input (doubleword integers), writes to BSR (implicit)
+ // Takes two ZMM registers of E8M0 scale bytes, writes to BSR (implicit)
def int_x86_bsrmovf : ClangBuiltin<"__builtin_ia32_bsrmovf">,
- Intrinsic<[], [llvm_v16i32_ty, llvm_v16i32_ty], []>;
+ Intrinsic<[], [llvm_v64i8_ty, llvm_v64i8_ty], []>;
// BSRMOVH - Move Half to/from BSR
- // Load variant: read from ZMM (doubleword integers), update BSR
+ // Load variant: read 64 scale bytes from ZMM, update BSR
def int_x86_bsrmovh_set : ClangBuiltin<"__builtin_ia32_bsrmovh_set">,
- Intrinsic<[], [llvm_v16i32_ty], []>;
- // Store variant: read from BSR, return ZMM (doubleword integers)
+ Intrinsic<[], [llvm_v64i8_ty], []>;
+ // Store variant: read from BSR, return 64 scale bytes in a ZMM
def int_x86_bsrmovh_get : ClangBuiltin<"__builtin_ia32_bsrmovh_get">,
- Intrinsic<[llvm_v16i32_ty], [], []>;
+ Intrinsic<[llvm_v64i8_ty], [], []>;
// BSRMOVL - Move Low to/from BSR
- // Load variant: read from ZMM (doubleword integers), write to BSR
+ // Load variant: read 64 scale bytes from ZMM, write to BSR
def int_x86_bsrmovl_set : ClangBuiltin<"__builtin_ia32_bsrmovl_set">,
- Intrinsic<[], [llvm_v16i32_ty], []>;
- // Store variant: read from BSR, return ZMM (doubleword integers)
+ Intrinsic<[], [llvm_v64i8_ty], []>;
+ // Store variant: read from BSR, return 64 scale bytes in a ZMM
def int_x86_bsrmovl_get : ClangBuiltin<"__builtin_ia32_bsrmovl_get">,
- Intrinsic<[llvm_v16i32_ty], [], []>;
+ Intrinsic<[llvm_v64i8_ty], [], []>;
// TILEMOVCOL - Move Column from Vector to Tile
// Single intrinsic handles both immediate and register index forms.
// Lowering checks if index is constant and selects appropriate instruction.
- def int_x86_tilesetcol : ClangBuiltin<"__builtin_ia32_tilesetcol">,
+ def int_x86_tilemovcol_set : ClangBuiltin<"__builtin_ia32_tilemovcol_set">,
Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
- // ACE TILEMOVROW - Move Row from Vector TO Tile (extends AMX TILEMOVROW)
- // Note: AMX has int_x86_tilemovrow which reads FROM tile to vector
- // ACE adds the reverse direction: writing TO tile from vector
+ // ACE TILEMOVROW - Move Row from Vector to Tile (extends AMX TILEMOVROW)
+ // AMX has int_x86_tilemovrow, which reads from the tile into a vector; ACE
+ // adds the reverse direction, writing to the tile from a vector.
// Single intrinsic handles both immediate and register index forms.
// Lowering checks if index is constant and selects appropriate instruction.
- def int_x86_tilesetrow : ClangBuiltin<"__builtin_ia32_tilesetrow">,
+ def int_x86_tilemovrow_set : ClangBuiltin<"__builtin_ia32_tilemovrow_set">,
Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
// TOP2BF16PS - Outer Product BF16 to Single Precision
def int_x86_top2bf16ps : ClangBuiltin<"__builtin_ia32_top2bf16ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v32bf16_ty, llvm_v32bf16_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4 - Tile Outer Product (4x4 block operations)
// TOP4BUUD - Unsigned/Unsigned
def int_x86_top4buud : ClangBuiltin<"__builtin_ia32_top4buud">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4BUSD - Unsigned/Signed
def int_x86_top4busd : ClangBuiltin<"__builtin_ia32_top4busd">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4BSSD - Signed/Signed
def int_x86_top4bssd : ClangBuiltin<"__builtin_ia32_top4bssd">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4BSUD - Signed/Unsigned
def int_x86_top4bsud : ClangBuiltin<"__builtin_ia32_top4bsud">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4MX - Mixed Precision Outer Products
- // TOP4MXHFBPS - FP16/BF16 to FP32
+ // TOP4MXHF8PS - MX FP8 HF8 (E4M3) to FP32
def int_x86_top4mxhf8ps : ClangBuiltin<"__builtin_ia32_top4mxhf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
- // TOP4MXBHFBPS - BF16/FP16 to FP32
+ // TOP4MXBHF8PS - MX FP8 BF8xHF8 to FP32
def int_x86_top4mxbhf8ps : ClangBuiltin<"__builtin_ia32_top4mxbhf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
- // TOP4MXHBFBPS - FP16/BF16/BF16 to FP32
+ // TOP4MXHBF8PS - MX FP8 HF8xBF8 to FP32
def int_x86_top4mxhbf8ps : ClangBuiltin<"__builtin_ia32_top4mxhbf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
// TOP4MXBF8PS - MX FP8 BF8xBF8 to FP32
def int_x86_top4mxbf8ps : ClangBuiltin<"__builtin_ia32_top4mxbf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
// TOP4MXBSSPS - MX INT8 SxS to FP32 (BSR scaled)
def int_x86_top4mxbssps : ClangBuiltin<"__builtin_ia32_top4mxbssps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
// ACE - internal intrinsics (for IR-level optimization)
// These take tile/ZMM values directly instead of immediate tile IDs
// TILEMOVCOL internal - takes dimensions and returns tile
- def int_x86_tilesetcol_internal :
- ClangBuiltin<"__builtin_ia32_tilesetcol_internal">,
+ def int_x86_tilemovcol_set_internal :
+ ClangBuiltin<"__builtin_ia32_tilemovcol_set_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
[]>;
@@ -5856,8 +5851,8 @@ let TargetPrefix = "x86" in {
// ACE TILEMOVROW internal - takes dimensions, vector, index and returns tile
// Note: AMX has int_x86_tilemovrow_internal which extracts from tile to vector
// ACE adds the reverse: write vector to tile row
- def int_x86_tilesetrow_internal :
- ClangBuiltin<"__builtin_ia32_tilesetrow_internal">,
+ def int_x86_tilemovrow_set_internal :
+ ClangBuiltin<"__builtin_ia32_tilemovrow_set_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
[]>;
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 255a2a70aea103..018c7144c75ffd 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -4015,7 +4015,7 @@ bool X86AsmParser::parseInstruction(ParseInstructionInfo &Info, StringRef Name,
// For BSRMOVH and BSRMOVL, detect set vs get by operand order
if (Name == "bsrmovh" || Name == "bsrmovl") {
- // Intel: BSR first = set (ZMM→BSR); ATT: non-BSR first = set
+ // Intel: BSR first = set (ZMM->BSR); ATT: non-BSR first = set
bool IsLoad = isParsingIntelSyntax() ? (FirstBsrPos < FirstNonBsrPos)
: (FirstNonBsrPos < FirstBsrPos);
StringRef NewMnemonic =
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index ab535c81dbf42c..230902928ef834 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -1206,7 +1206,9 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
// Instruction format for 4VOp3:
// src1(ModR/M), src2(ModR/M), src3(VEX_4V)
Prefix.setRR2(MI, CurOp++);
- Prefix.setBB2(MI, CurOp++);
+ Prefix.setBB2(MI, CurOp);
+ Prefix.setX(MI, CurOp, 4);
+ ++CurOp;
Prefix.set4VV2(MI, CurOp++);
break;
}
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 6d1133a0d5d397..4f1d0d76c8423c 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -296,9 +296,6 @@ def FeatureAMXAVX512 : SubtargetFeature<"amx-avx512",
"HasAMXAVX512", "true",
"Support AMX-AVX512 instructions",
[FeatureAMXTILE]>;
-def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
- "Support ACEV1 (AI Compute Extensions v1) instructions",
- [FeatureAMXAVX512]>;
def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true",
"Support CMPCCXADD instructions">;
def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true",
@@ -366,6 +363,16 @@ def FeatureAVX10_2_512 : SubtargetFeature<"avx10.2-512", "HasAVX10_2_512", "true
def FeatureAVX10_V2_AUX : SubtargetFeature<"avx10v2aux", "HasAVX10_V2_AUX", "true",
"Support AVX10 V2 AUX instructions",
[FeatureAVX10_1]>;
+// Defined after AVX10.1 since ACE v1 builds on it as its vector baseline.
+// ACE v1 reuses the AMX tile architecture: per spec section 15.5.5 an
+// implementation that supports only ACE still reports AMX-TILE, which is what
+// enumerates the tile registers and LDTILECFG/STTILECFG/TILEZERO/TILERELEASE.
+// The AMX compute features and palette 1 are not required, so AMX-AVX512 is not
+// implied; TILEMOVROW and the TCVTROW* family are enumerated under
+// "AMX-AVX512 || ACE_VSN >= 1".
+def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
+ "Support ACEV1 (AI Compute Extensions v1) instructions",
+ [FeatureAMXTILE, FeatureAVX10_1]>;
def FeatureEGPR : SubtargetFeature<"egpr", "HasEGPR", "true",
"Support extended general purpose register">;
def FeaturePush2Pop2 : SubtargetFeature<"push2pop2", "HasPush2Pop2", "true",
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index a924360caad7ee..41c10a84b8badd 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -226,8 +226,12 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
- // ACE v1 doesn't have TILESTORED - use TILEMOVROW row-by-row
- if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ // ACE configures palette 2, which has no TILESTORED - use TILEMOVROW
+ // row-by-row through a scratch ZMM.
+ // FIXME: This defines the tile and the scratch register once per row, which
+ // is not valid SSA. X86VolatileTileData also still emits the AMX load/store
+ // intrinsics at -O0, so the ACE -O0 path needs both fixed together.
+ if (ST->hasACEV1()) {
const DebugLoc &DL = Before->getDebugLoc();
Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
@@ -235,12 +239,12 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
const unsigned RowSize = 64;
for (unsigned Row = 0; Row < NumRows; ++Row) {
- // TILEMOVROW zmm, tmm, imm8 (read from tile)
+ // tilemovrow $row, %tmm, %zmm
BuildMI(*MBB, Before, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
.addReg(VirtReg)
.addImm(Row);
- // VMOVUPS [stack + row*64], zmm
+ // vmovups %zmm, row*64(%sp)
MachineInstrBuilder MIB =
BuildMI(*MBB, Before, DL, TII->get(X86::VMOVUPSZmr));
addFrameReference(MIB, FI, Row * RowSize);
@@ -278,8 +282,10 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
else
TileReg = MRI->createVirtualRegister(&RC);
- // ACE v1 doesn't have TILELOADD - use TILEMOVROW row-by-row
- if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ // ACE configures palette 2, which has no TILELOADD - use TILEMOVROW
+ // row-by-row through a scratch ZMM.
+ // FIXME: See the corresponding comment in spill().
+ if (ST->hasACEV1()) {
const DebugLoc &DL = UseMI->getDebugLoc();
Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
@@ -287,12 +293,12 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
const unsigned RowSize = 64;
for (unsigned Row = 0; Row < NumRows; ++Row) {
- // VMOVUPS zmm, [stack + row*64]
+ // vmovups row*64(%sp), %zmm
MachineInstrBuilder MIB = BuildMI(*UseMI->getParent(), UseMI, DL,
TII->get(X86::VMOVUPSZrm), ScratchZMM);
addFrameReference(MIB, FI, Row * RowSize);
- // TILEMOVROW tmm, zmm, imm8 (write to tile)
+ // tilemovrow $row, %zmm, %tmm
BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWri),
TileReg)
.addReg(ScratchZMM)
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index c4f747147b2fcb..d1cc15598e4044 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -5785,7 +5785,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
case Intrinsic::x86_tcvtrowps2phh_internal:
case Intrinsic::x86_tcvtrowps2phl_internal:
case Intrinsic::x86_tilemovrow_internal: {
- if (!Subtarget->hasAMXAVX512())
+ if (!Subtarget->hasAMXAVX512() && !Subtarget->hasACEV1())
break;
auto *MFI =
CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c53f9a4c60aa26..3b5bb70e3db02b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28777,8 +28777,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
// ACE Tile Movement Intrinsics - handle both immediate and register index
// forms using a single intrinsic. Check if index is constant to select
// the appropriate instruction form.
- case Intrinsic::x86_tilesetrow:
- case Intrinsic::x86_tilesetcol: {
+ case Intrinsic::x86_tilemovrow_set:
+ case Intrinsic::x86_tilemovcol_set: {
SDLoc DL(Op);
SDValue Chain = Op.getOperand(0);
unsigned TileID = Op.getConstantOperandVal(2);
@@ -28786,7 +28786,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDValue Idx = Op.getOperand(4);
unsigned PseudoImm, PseudoReg;
- if (IntNo == Intrinsic::x86_tilesetrow) {
+ if (IntNo == Intrinsic::x86_tilemovrow_set) {
PseudoImm = X86::PTILEMOVROW;
PseudoReg = X86::PTILEMOVROW_REG;
} else {
@@ -39870,17 +39870,12 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
break;
}
// These instructions: TILE dst (rw), ZMM src1, ZMM src2
- // The pseudo has: u8imm dst, u8imm src1, u8imm src2
- // src1 and src2 are ZMM register indices (0-31)
+ // The pseudo has: u8imm dst, VR512 src1, VR512 src2
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
- unsigned Src1Imm = MI.getOperand(1).getImm();
- unsigned Src2Imm = MI.getOperand(2).getImm();
- assert(Src1Imm < 32 && "Illegal ZMM register index");
- assert(Src2Imm < 32 && "Illegal ZMM register index");
BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
.addReg(DstReg, RegState::Undef)
- .addReg(X86::ZMM0 + Src1Imm)
- .addReg(X86::ZMM0 + Src2Imm);
+ .addReg(MI.getOperand(1).getReg())
+ .addReg(MI.getOperand(2).getReg());
MI.eraseFromParent();
return BB;
}
@@ -39911,15 +39906,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
}
// These instructions: TILE dst (rw), ZMM src1, ZMM src2, imm8
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
- unsigned Src1Imm = MI.getOperand(1).getImm();
- unsigned Src2Imm = MI.getOperand(2).getImm();
unsigned Imm = MI.getOperand(3).getImm();
- assert(Src1Imm < 32 && "Illegal ZMM register index");
- assert(Src2Imm < 32 && "Illegal ZMM register index");
BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
.addReg(DstReg, RegState::Undef)
- .addReg(X86::ZMM0 + Src1Imm)
- .addReg(X86::ZMM0 + Src2Imm)
+ .addReg(MI.getOperand(1).getReg())
+ .addReg(MI.getOperand(2).getReg())
.addImm(Imm);
MI.eraseFromParent();
return BB;
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
index f786f8ca345837..26f531a19ff0be 100644
--- a/llvm/lib/Target/X86/X86InstrACE.td
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -19,7 +19,8 @@ let Predicates = [HasACEV1, In64BitMode] in {
// BSRINIT - Initialize Block Scale Register
// VEX.128.F2.0F38.W1 49 11:000:000 - BSRINIT bsr0
// BSR is implicit destination, ModRM = 11:000:000 (mod=11, reg=000, r/m=000)
-// Note: BSR0 is implicit (via Defs), not in operand list. Asm syntax has no explicit operand.
+// BSR0 is implicit (via Defs) rather than an operand, so the asm syntax takes
+// no explicit operand.
let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
def BSRINIT : I<0x49, MRM_C0, (outs), (ins),
"bsrinit", []>,
@@ -28,13 +29,14 @@ let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
// BSRMOVF - Move Full to BSR
// EVEX.512.NP.MAP6.W1 95 11:000:bbb - BSRMOVF bsr0, zmm1, zmm2/m512
-// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1, ModRM.r/m encodes zmm2
+// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1 and ModRM.r/m
+// encodes zmm2
// BSR is implicit destination (via Defs)
let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
def BSRMOVFrr : I<0x95, MRM0r, (outs),
(ins VR512:$src1, VR512:$src2),
"bsrmovf\t{$src2, $src1|$src1, $src2}",
- [(int_x86_bsrmovf (v16i32 VR512:$src1), (v16i32 VR512:$src2))]>,
+ [(int_x86_bsrmovf (v64i8 VR512:$src1), (v64i8 VR512:$src2))]>,
EVEX, T_MAP6, REX_W, EVEX_V512, VVVV;
let mayLoad = 1 in
@@ -49,14 +51,15 @@ let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
// EVEX.512.F2.MAP6.W0 95 /r - BSRMOVH zmm1/m512, bsr0 (store, W0)
// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
-// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+// AsmString uses a "_set"/"_get" suffix for asm matching; the printer emits the
+// real mnemonic.
let SchedRW = [WriteVecLogic] in {
// Set BSR high half (W1): BSR is both input and output (read-modify-write)
let Defs = [BSR0], Uses = [BSR0] in {
def BSRMOVHrr_set : I<0x95, MRM0r, (outs),
(ins VR512:$src),
"bsrmovh_set\t$src",
- [(int_x86_bsrmovh_set (v16i32 VR512:$src))]>,
+ [(int_x86_bsrmovh_set (v64i8 VR512:$src))]>,
EVEX, XD, T_MAP6, REX_W, EVEX_V512;
let mayLoad = 1 in
@@ -66,18 +69,20 @@ let SchedRW = [WriteVecLogic] in {
EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
- // Get BSR high half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ // Get BSR high half (W0): BSR is the implicit source, ZMM/mem the explicit
+ // destination
let Uses = [BSR0] in {
def BSRMOVHrr_get : I<0x95, MRM0r, (outs VR512:$dst),
(ins),
"bsrmovh_get\t$dst",
- [(set (v16i32 VR512:$dst), (int_x86_bsrmovh_get))]>,
+ [(set (v64i8 VR512:$dst), (int_x86_bsrmovh_get))]>,
EVEX, XD, T_MAP6, EVEX_V512;
let mayStore = 1 in
def BSRMOVHmr_get : I<0x95, MRM0m, (outs),
(ins f512mem:$dst),
- "bsrmovh_get\t$dst", []>,
+ "bsrmovh_get\t$dst",
+ [(store (v64i8 (int_x86_bsrmovh_get)), addr:$dst)]>,
EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
}
@@ -87,14 +92,15 @@ let SchedRW = [WriteVecLogic] in {
// EVEX.512.F3.MAP6.W0 95 /r - BSRMOVL zmm1/m512, bsr0 (store, W0)
// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
-// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+// AsmString uses a "_set"/"_get" suffix for asm matching; the printer emits the
+// real mnemonic.
let SchedRW = [WriteVecLogic] in {
// Set BSR low half (W1): BSR is both input and output (read-modify-write)
let Defs = [BSR0], Uses = [BSR0] in {
def BSRMOVLrr_set : I<0x95, MRM0r, (outs),
(ins VR512:$src),
"bsrmovl_set\t$src",
- [(int_x86_bsrmovl_set (v16i32 VR512:$src))]>,
+ [(int_x86_bsrmovl_set (v64i8 VR512:$src))]>,
EVEX, XS, T_MAP6, REX_W, EVEX_V512;
let mayLoad = 1 in
@@ -104,18 +110,20 @@ let SchedRW = [WriteVecLogic] in {
EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
- // Get BSR low half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ // Get BSR low half (W0): BSR is the implicit source, ZMM/mem the explicit
+ // destination
let Uses = [BSR0] in {
def BSRMOVLrr_get : I<0x95, MRM0r, (outs VR512:$dst),
(ins),
"bsrmovl_get\t$dst",
- [(set (v16i32 VR512:$dst), (int_x86_bsrmovl_get))]>,
+ [(set (v64i8 VR512:$dst), (int_x86_bsrmovl_get))]>,
EVEX, XS, T_MAP6, EVEX_V512;
let mayStore = 1 in
def BSRMOVLmr_get : I<0x95, MRM0m, (outs),
(ins f512mem:$dst),
- "bsrmovl_get\t$dst", []>,
+ "bsrmovl_get\t$dst",
+ [(store (v64i8 (int_x86_bsrmovl_get)), addr:$dst)]>,
EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
}
@@ -159,7 +167,8 @@ let SchedRW = [WriteVecLogic] in {
// Operand 1: ModRM:reg (rw) = tmm1
// Operand 2: ModRM:r/m (r) = zmm2
// Operand 3: EVEX.vvvv (r) = zmm3
-// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+// Use MRMSrcReg4VOp3: operand 1 goes to r/m and operand 2 to vvvv, so src2
+// and src3 are listed in the opposite order from the spec.
let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
def TOP2BF16PSrrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
@@ -176,7 +185,8 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
// Operand 1: ModRM:reg (rw) = tmm1
// Operand 2: ModRM:r/m (r) = zmm2
// Operand 3: EVEX.vvvv (r) = zmm3
-// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+// Use MRMSrcReg4VOp3: operand 1 goes to r/m and operand 2 to vvvv, so src2
+// and src3 are listed in the opposite order from the spec.
let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
def TOP4BUUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
@@ -211,8 +221,9 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
// Operand 2: ModRM:r/m (r) = zmm2
// Operand 3: EVEX.vvvv (r) = zmm3
// Operand 4: imm8
-// MRMSrcReg encodes: src1→reg, src2→r/m, src3→vvvv (so swap to get spec order)
-let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+// MRMSrcReg encodes src1 in reg, src2 in r/m and src3 in vvvv, so the operands
+// are listed swapped to match the spec order.
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic], Uses = [BSR0] in {
def TOP4MXBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
@@ -244,8 +255,8 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
// These are used by intrinsics and expanded via custom inserter
//===----------------------------------------------------------------------===//
-// Pseudo instructions for user-facing intrinsics (immediate tile IDs)
-// These use usesCustomInserter to expand to real instructions in X86ISelLowering
+// Pseudo instructions for user-facing intrinsics (immediate tile IDs).
+// usesCustomInserter expands these to real instructions in X86ISelLowering.
let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
// BSRINIT pseudo - no operands, just triggers BSR initialization
def PBSRINIT : PseudoI<(outs), (ins),
@@ -256,7 +267,7 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
// TILEMOVCOL pseudo - immediate index variant
// dst: tile ID (imm), src: ZMM value (VR512), idx: column index (imm)
- // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ // Custom lowering picks the imm or reg variant, so there is no pattern here.
def PTILEMOVCOL : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
// TILEMOVCOL pseudo - register index variant
@@ -264,63 +275,63 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
// TILEMOVROW pseudo - immediate index variant (ACE: write to tile)
// dst: tile ID (imm), src: ZMM value (VR512), idx: row index (imm)
- // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ // Custom lowering picks the imm or reg variant, so there is no pattern here.
def PTILEMOVROW : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
// TILEMOVROW pseudo - register index variant (ACE: write to tile)
def PTILEMOVROW_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
- // TOP2BF16PS pseudo
- def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top2bf16ps timm:$dst, timm:$src1, timm:$src2)]>;
+ // TOP2BF16PS pseudo - tile dst by ID, ZMM sources by value
+ def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top2bf16ps timm:$dst, VR512:$src1, VR512:$src2)]>;
- // TOP4 pseudos - three tile operands (dst is also accumulator)
- def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4buud timm:$dst, timm:$src1, timm:$src2)]>;
+ // TOP4 pseudos - tile dst by ID (also the accumulator), ZMM sources by value
+ def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4buud timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4busd timm:$dst, timm:$src1, timm:$src2)]>;
+ def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4busd timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4bssd timm:$dst, timm:$src1, timm:$src2)]>;
+ def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4bssd timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4bsud timm:$dst, timm:$src1, timm:$src2)]>;
+ def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4bsud timm:$dst, VR512:$src1, VR512:$src2)]>;
- // TOP4MX pseudos - four operands (dst, src1, src2, imm)
- def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ // TOP4MX pseudos - tile dst by ID, ZMM sources by value, plus imm8
+ def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxbhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxbhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxhbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxhbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxbssps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxbssps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
}
-// Pseudo instructions for internal intrinsics (register allocation)
-// These are used for IR-level optimization and register allocation
-// usesCustomInserter is needed to set ACEProgModel in EmitInstrWithCustomInserter
+// Pseudo instructions for internal intrinsics (register allocation).
+// These are used for IR-level optimization and register allocation.
+// usesCustomInserter is needed to set ACEProgModel in
+// EmitInstrWithCustomInserter.
let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
// TILEMOVCOL internal - output is TILE, input is ZMM + index
def PTILEMOVCOLV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
[(set TILE:$dst,
- (int_x86_tilesetcol_internal GR16:$row, GR16:$col,
+ (int_x86_tilemovcol_set_internal GR16:$row, GR16:$col,
VR512:$src, GR32:$idx))]>;
// TILEMOVROW internal - output is TILE, input is ZMM + index
- // ACE TILEMOVROW internal - write vector to tile row
def PTILEMOVROWV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
[(set TILE:$dst,
- (int_x86_tilesetrow_internal GR16:$row, GR16:$col,
+ (int_x86_tilemovrow_set_internal GR16:$row, GR16:$col,
VR512:$src, GR32:$idx))]>;
// TOP2BF16PS internal - output TILE, inputs: dimensions, TILE acc, two ZMMs
@@ -369,7 +380,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXHFBPS internal - imm is BSR index (u8imm matches timm in pattern)
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXHF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -378,7 +389,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXBHFBPS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXBHF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -387,7 +398,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXHBFBPS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXHBF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -396,7 +407,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXBF8PS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXBF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -405,7 +416,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXBSSPS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXBSSPSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -414,4 +425,16 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
}
+// Pseudo instructions for tile spill and reload (register allocation)
+// ACE has no TILELOADD/TILESTORED, so the register allocator emits these and
+// X86LowerTileCopy expands them into a row-by-row TILEMOVROW sequence.
+// Operand order matches storeRegToStackSlot/loadRegFromStackSlot.
+let isPseudo = true, hasNoSchedulingInfo = 1 in {
+ let mayStore = 1 in
+ def ACE_TILESPILL : PseudoI<(outs), (ins opaquemem:$dst, TILE:$src), []>;
+
+ let mayLoad = 1 in
+ def ACE_TILERELOAD : PseudoI<(outs TILE:$dst), (ins opaquemem:$src), []>;
+}
+
} // end HasACEV1, In64BitMode
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 25475ef4fd1982..854f016ac8f902 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4519,11 +4519,12 @@ static unsigned getLoadStoreRegOpcode(Register Reg,
assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
assert((STI.hasAMXTILE() || STI.hasACEV1()) &&
"Using 8*1024-bit register requires AMX-TILE or ACE");
- // ACE v1 doesn't have TILELOADD/TILESTORED - tile spilling should be
- // handled by X86LowerTileCopy using TILEMOVROW row-by-row.
- // This path should only be hit for AMX targets.
- assert(STI.hasAMXTILE() &&
- "ACE tile spill should use TILEMOVROW, not TILELOADD/TILESTORED");
+ // ACE configures palette 2, which has no TILELOADD/TILESTORED. Spill and
+ // reload through pseudos that X86LowerTileCopy expands into TILEMOVROW.
+ // This is the only place these pseudos are created, so it is what keeps
+ // the row-by-row sequences confined to ACE targets.
+ if (STI.hasACEV1())
+ return Load ? X86::ACE_TILERELOAD : X86::ACE_TILESPILL;
#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
: GET_EGPR_IF_ENABLED(X86::TILESTORED);
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index 1b7f894a79fc74..56df999d5249f2 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -122,8 +122,8 @@ static bool isACEOnlyIntrinsic(Instruction *I) {
case Intrinsic::x86_top4mxbf8ps_internal:
case Intrinsic::x86_top4mxbssps_internal:
// ACE tile movement intrinsics
- case Intrinsic::x86_tilesetcol_internal:
- case Intrinsic::x86_tilesetrow_internal:
+ case Intrinsic::x86_tilemovcol_set_internal:
+ case Intrinsic::x86_tilemovrow_set_internal:
return true;
default:
return false;
@@ -298,8 +298,8 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
}
// ACE TILEMOV intrinsics - move ZMM to tile row/column
// Pattern: (m, n, zmm_src, idx) -> tile
- case Intrinsic::x86_tilesetcol_internal:
- case Intrinsic::x86_tilesetrow_internal: {
+ case Intrinsic::x86_tilemovcol_set_internal:
+ case Intrinsic::x86_tilemovrow_set_internal: {
// Output tile shape
Row = II->getArgOperand(0);
Col = II->getArgOperand(1);
@@ -835,16 +835,17 @@ namespace {
class X86LowerAMXCast {
Function &Func;
std::unique_ptr<DominatorTree> DT;
- bool IsACEOnly; // ACE v1 without AMX TILELOADD/TILESTORED
+ bool IsACE; // ACE configures palette 2, which has no TILELOADD/TILESTORED
public:
X86LowerAMXCast(Function &F, const TargetMachine *TM = nullptr)
- : Func(F), DT(nullptr), IsACEOnly(false) {
+ : Func(F), DT(nullptr), IsACE(false) {
if (TM) {
- // Check if we're ACE-only (no AMX TILELOADD/TILESTORED available)
+ // ACE implies AMX-TILE, but ACE code runs under palette 2 where
+ // TILELOADD/TILESTORED are unavailable, so key off ACE itself.
const X86Subtarget *ST =
static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
- IsACEOnly = ST && ST->hasACEV1() && !ST->hasAMXTILE();
+ IsACE = ST && ST->hasACEV1();
}
}
bool combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
@@ -1074,7 +1075,7 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
auto *II = cast<IntrinsicInst>(Tile);
// ACE v1 doesn't have TILESTORED instruction - skip combining for ACE targets
// or ACE intrinsics. ACE uses struct-based API with vector load/store.
- if (IsACEOnly || isACEOnlyIntrinsic(II))
+ if (IsACE || isACEOnlyIntrinsic(II))
return false;
// Tile is output from AMX intrinsic. The first operand of the
// intrinsic is row, the second operand of the intrinsic is column.
@@ -1108,7 +1109,7 @@ bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
return false;
// ACE v1 doesn't have TILELOADD instruction - skip combining for ACE targets
// or ACE intrinsics. ACE uses struct-based API with vector load/store.
- if (IsACEOnly || isACEOnlyIntrinsic(II))
+ if (IsACE || isACEOnlyIntrinsic(II))
return false;
std::tie(Row, Col) = getShape(II, OpNo);
IRBuilder<> Builder(LD);
@@ -1346,7 +1347,7 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
// ACE v1 doesn't have TILELOADD - for ACE targets or ACE intrinsics,
// use tilezero since ACE tiles are typically zero-initialized in struct
// API.
- if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ if (IsACE || isACEOnlyIntrinsic(II)) {
Value *Row = nullptr, *Col = nullptr;
std::tie(Row, Col) = getShape(II, OpNo);
Value *NewInst = Builder.CreateIntrinsic(Intrinsic::x86_tilezero_internal,
@@ -1380,7 +1381,7 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
// ACE v1 doesn't have TILESTORED - for ACE targets or ACE intrinsics,
// just create a poison value since the tile result is typically not read
// back.
- if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ if (IsACE || isACEOnlyIntrinsic(II)) {
Value *Poison = PoisonValue::get(AMXCast->getType());
AMXCast->replaceAllUsesWith(Poison);
AMXCast->eraseFromParent();
@@ -1446,7 +1447,12 @@ bool lowerAmxType(Function &F, const TargetMachine *TM,
// "Clang -O2 -S -emit-llvm t.c" + "llc t.ll") we should make
// sure the amx data is volatile, that is necessary for AMX fast
// register allocation.
- if (!F.hasFnAttribute(Attribute::OptimizeNone)) {
+ // This spills tiles with TILESTORED and reloads them with TILELOADD, which
+ // ACE cannot do under palette 2. ACE tile data is left non-volatile, so the
+ // tile registers stay live and X86FastPreTileConfig spills them instead.
+ const X86Subtarget *ST =
+ static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
+ if (!F.hasFnAttribute(Attribute::OptimizeNone) && !(ST && ST->hasACEV1())) {
X86VolatileTileData VTD(F);
C = VTD.volatileTileData() || C;
}
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 09082ba585572c..4b9bf9639c2420 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -36,62 +36,63 @@ using namespace llvm;
#define DEBUG_TYPE "x86-lower-tile-copy"
-/// Generate ACE-specific tile copy using TILEMOVROW row-by-row.
-/// ACE v1 doesn't have TILELOADD/TILESTORED, so we need to copy
-/// tiles through ZMM registers, one row at a time.
-///
-/// Spill sequence (tile -> stack):
-/// for row = 0 to 15:
-/// TILEMOVROW zmm_temp, tmm_src, row ; read row from tile to ZMM
-/// VMOVUPS [stack + row*64], zmm_temp ; store ZMM to stack
-///
-/// Reload sequence (stack -> tile):
-/// for row = 0 to 15:
-/// VMOVUPS zmm_temp, [stack + row*64] ; load ZMM from stack
-/// TILEMOVROW tmm_dst, zmm_temp, row ; write row from ZMM to tile
-///
-static void emitACETileCopy(MachineBasicBlock &MBB, MachineInstr &MI,
- const X86Subtarget &ST, const X86InstrInfo *TII,
- const TargetRegisterInfo *TRI, Register SrcReg,
- Register DstReg, bool SrcKill, int TileSS,
- Register ScratchZMM) {
- const DebugLoc &DL = MI.getDebugLoc();
-
- // Each tile has 16 rows of 64 bytes each (1KB total)
- const unsigned NumRows = 16;
- const unsigned RowSize = 64;
-
- // Spill: Read each row from source tile to ZMM, then store to stack
- for (unsigned Row = 0; Row < NumRows; ++Row) {
- // TILEMOVROW zmm, tmm, imm8 (read direction: tile -> ZMM)
- // Uses TILEMOVROWrti instruction
- // Only kill src on the last row read
- bool KillSrcNow = (Row == NumRows - 1) && SrcKill;
+// ACE has no TILELOADD/TILESTORED, so a tile moves to and from memory one row
+// at a time through a scratch ZMM. Each tile is 16 rows of 64 bytes (1KB).
+static const unsigned ACENumTileRows = 16;
+static const unsigned ACETileRowSize = 64;
+
+/// Emit the ACE tile spill sequence, storing \p SrcReg to \p TileSS.
+static void emitACETileSpill(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ const X86InstrInfo *TII, Register SrcReg,
+ bool SrcKill, int TileSS, Register ScratchZMM) {
+ const DebugLoc &DL = MI->getDebugLoc();
+ for (unsigned Row = 0; Row < ACENumTileRows; ++Row) {
+ // tilemovrow $row, %tmm, %zmm
+ // Only kill src on the last row read.
+ bool KillSrcNow = (Row == ACENumTileRows - 1) && SrcKill;
BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
.addReg(SrcReg, getKillRegState(KillSrcNow))
.addImm(Row);
- // VMOVUPS [stack + row*64], zmm
+ // vmovups %zmm, row*64(%sp)
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
- addFrameReference(MIB, TileSS, Row * RowSize);
+ addFrameReference(MIB, TileSS, Row * ACETileRowSize);
MIB.addReg(ScratchZMM, RegState::Kill);
}
+}
- // Reload: Load each row from stack to ZMM, then write to dest tile
- for (unsigned Row = 0; Row < NumRows; ++Row) {
- // VMOVUPS zmm, [stack + row*64]
+/// Emit the ACE tile reload sequence, loading \p DstReg from \p TileSS.
+static void emitACETileReload(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ const X86InstrInfo *TII, Register DstReg,
+ int TileSS, Register ScratchZMM) {
+ const DebugLoc &DL = MI->getDebugLoc();
+ for (unsigned Row = 0; Row < ACENumTileRows; ++Row) {
+ // vmovups row*64(%sp), %zmm
MachineInstrBuilder MIB =
BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), ScratchZMM);
- addFrameReference(MIB, TileSS, Row * RowSize);
+ addFrameReference(MIB, TileSS, Row * ACETileRowSize);
- // TILEMOVROW tmm, zmm, imm8 (write direction: ZMM -> tile)
- // Uses TILEMOVROWri instruction (ACE-specific)
+ // tilemovrow $row, %zmm, %tmm
BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWri), DstReg)
.addReg(ScratchZMM, RegState::Kill)
.addImm(Row);
}
}
+/// Pick a ZMM that is dead here, or NoRegister if none is available.
+static Register findScratchZMM(MachineFunction &MF,
+ const TargetRegisterInfo *TRI,
+ const LiveRegUnits &UsedRegs) {
+ BitVector VR512Regs =
+ TRI->getAllocatableSet(MF, TRI->getRegClass(X86::VR512RegClassID));
+ for (auto RegT : VR512Regs.set_bits())
+ if (UsedRegs.available(RegT))
+ return RegT;
+ return X86::NoRegister;
+}
+
namespace {
class X86LowerTileCopyLegacy : public MachineFunctionPass {
@@ -148,6 +149,48 @@ static bool lowerTileCopy(MachineFunction &MF) {
if (MI.isDebugInstr())
continue;
UsedRegs.stepBackward(MI);
+
+ // Expand the tile spill/reload pseudos inserted by the register
+ // allocator. getLoadStoreRegOpcode is the only place these are created,
+ // and it only does so for ACE targets.
+ unsigned Opcode = MI.getOpcode();
+ if (Opcode == X86::ACE_TILESPILL || Opcode == X86::ACE_TILERELOAD) {
+ assert(ST.hasACEV1() && "ACE tile spill pseudo on non-ACE target");
+ bool IsSpill = Opcode == X86::ACE_TILESPILL;
+ // ACE_TILESPILL is (mem, tile), ACE_TILERELOAD is (tile, mem).
+ MachineOperand &TileMO =
+ MI.getOperand(IsSpill ? X86::AddrNumOperands : 0);
+ int TileSS =
+ MI.getOperand((IsSpill ? 0 : 1) + X86::AddrBaseReg).getIndex();
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ Register ScratchZMM = findScratchZMM(MF, TRI, UsedRegs);
+ int ZmmSS = -1;
+ if (!ScratchZMM) {
+ // No available register? Save ZMM0 and reload it after use.
+ ScratchZMM = X86::ZMM0;
+ ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)),
+ ZmmSS)
+ .addReg(X86::ZMM0);
+ }
+
+ if (IsSpill)
+ emitACETileSpill(MBB, MI, TII, TileMO.getReg(), TileMO.isKill(),
+ TileSS, ScratchZMM);
+ else
+ emitACETileReload(MBB, MI, TII, TileMO.getReg(), TileSS, ScratchZMM);
+
+ if (ZmmSS != -1)
+ addFrameReference(
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0),
+ ZmmSS);
+
+ MI.eraseFromParent();
+ Changed = true;
+ continue;
+ }
+
if (!MI.isCopy())
continue;
MachineOperand &DstMO = MI.getOperand(0);
@@ -164,47 +207,28 @@ static bool lowerTileCopy(MachineFunction &MF) {
const DebugLoc &DL = MI.getDebugLoc();
- // Check if this is ACE-only (no AMX TILELOADD/TILESTORED available)
- if (ST.hasACEV1() && !ST.hasAMXTILE()) {
- // ACE v1: Use TILEMOVROW row-by-row copy
- // Need a scratch ZMM register for the transfer
-
- // Find an available ZMM register
- BitVector VR512Regs =
- TRI->getAllocatableSet(MF, TRI->getRegClass(X86::VR512RegClassID));
- Register ScratchZMM = X86::NoRegister;
- for (auto RegT : VR512Regs.set_bits()) {
- if (UsedRegs.available(RegT)) {
- ScratchZMM = RegT;
- break;
- }
- }
-
+ // ACE configures palette 2, which has no TILELOADD/TILESTORED, so copy
+ // the tile row-by-row through a scratch ZMM.
+ if (ST.hasACEV1()) {
+ Register ScratchZMM = findScratchZMM(MF, TRI, UsedRegs);
+ int ZmmSS = -1;
if (!ScratchZMM) {
- // If no ZMM available, use ZMM0 and save/restore it
+ // No available register? Save ZMM0 and reload it after use.
ScratchZMM = X86::ZMM0;
+ ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)),
+ ZmmSS)
+ .addReg(X86::ZMM0);
+ }
- // Allocate stack slot for scratch ZMM
- int ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
-
- // Save ZMM0
- MachineInstrBuilder MIB =
- BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
- addFrameReference(MIB, ZmmSS);
- MIB.addReg(X86::ZMM0);
-
- // Emit ACE tile copy
- emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
- TileSS, ScratchZMM);
+ emitACETileSpill(MBB, MI, TII, SrcReg, SrcMO.isKill(), TileSS,
+ ScratchZMM);
+ emitACETileReload(MBB, MI, TII, DstReg, TileSS, ScratchZMM);
- // Restore ZMM0
- MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0);
- addFrameReference(MIB, ZmmSS);
- } else {
- // Use available scratch ZMM
- emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
- TileSS, ScratchZMM);
- }
+ if (ZmmSS != -1)
+ addFrameReference(
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0),
+ ZmmSS);
} else {
// AMX: Use TILESTORED/TILELOADD (original code)
int StrideSS = 0;
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 50e68e39d68369..bcc7f463e48d8f 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -519,6 +519,9 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
// Set the SIMD floating point control register as reserved.
Reserved.set(X86::MXCSR);
+ // Set the ACE block scale register as reserved.
+ Reserved.set(X86::BSR0);
+
// Set the stack-pointer register and its aliases as reserved.
for (const MCPhysReg &SubReg : subregs_inclusive(X86::RSP))
Reserved.set(SubReg);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index e8ed9a9f572f92..a86f536fc2951e 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -639,9 +639,14 @@ constexpr FeatureBitset ImpliedFeaturesAMX_FP8 = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_MOVRS = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_AVX512 =
FeatureAMX_TILE | FeatureAVX10_2;
-constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAMX_AVX512;
constexpr FeatureBitset ImpliedFeaturesHRESET = {};
+// ACE v1 Features. AVX10.1 is the vector baseline required by the ACE v1
+// feature detection algorithm. AMX-TILE provides the tile registers and tile
+// management instructions that ACE reuses; per spec section 15.5.5 an ACE-only
+// implementation still reports it.
+constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAVX10_1 | FeatureAMX_TILE;
+
constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {};
constexpr FeatureBitset ImpliedFeaturesCMPCCXADD = {};
constexpr FeatureBitset ImpliedFeaturesRAOINT = {};
diff --git a/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll b/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
deleted file mode 100644
index 9afca28b9c9acd..00000000000000
--- a/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
+++ /dev/null
@@ -1,244 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f | FileCheck %s
-
-;
-; ACE Intrinsic Tests
-;
-
-; Test BSRINIT intrinsic
-define void @test_bsrinit() {
-; CHECK-LABEL: test_bsrinit:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrinit %bsr0
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrinit()
- ret void
-}
-
-; Test BSRMOVF intrinsic
-define void @test_bsrmovf(<16 x i32> %a, <16 x i32> %b) {
-; CHECK-LABEL: test_bsrmovf:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrmovf(<16 x i32> %a, <16 x i32> %b)
- ret void
-}
-
-; Test BSRMOVH set intrinsic
-define void @test_bsrmovh_set(<16 x i32> %a) {
-; CHECK-LABEL: test_bsrmovh_set:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovh %zmm0, %bsr0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrmovh.set(<16 x i32> %a)
- ret void
-}
-
-; Test BSRMOVH get intrinsic
-define <16 x i32> @test_bsrmovh_get() {
-; CHECK-LABEL: test_bsrmovh_get:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovh %bsr0, %zmm0
-; CHECK-NEXT: retq
- %result = call <16 x i32> @llvm.x86.bsrmovh.get()
- ret <16 x i32> %result
-}
-
-; Test BSRMOVL set intrinsic
-define void @test_bsrmovl_set(<16 x i32> %a) {
-; CHECK-LABEL: test_bsrmovl_set:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovl %zmm0, %bsr0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrmovl.set(<16 x i32> %a)
- ret void
-}
-
-; Test BSRMOVL get intrinsic
-define <16 x i32> @test_bsrmovl_get() {
-; CHECK-LABEL: test_bsrmovl_get:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovl %bsr0, %zmm0
-; CHECK-NEXT: retq
- %result = call <16 x i32> @llvm.x86.bsrmovl.get()
- ret <16 x i32> %result
-}
-
-; Test TOP2BF16PS intrinsic
-define void @test_top2bf16ps() {
-; CHECK-LABEL: test_top2bf16ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top2bf16ps %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BUUD intrinsic
-define void @test_top4buud() {
-; CHECK-LABEL: test_top4buud:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4buud %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BUSD intrinsic
-define void @test_top4busd() {
-; CHECK-LABEL: test_top4busd:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4busd %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BSSD intrinsic
-define void @test_top4bssd() {
-; CHECK-LABEL: test_top4bssd:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4bssd %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BSUD intrinsic
-define void @test_top4bsud() {
-; CHECK-LABEL: test_top4bsud:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4bsud %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4MXHF8PS intrinsic
-define void @test_top4mxhf8ps() {
-; CHECK-LABEL: test_top4mxhf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxhf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXBHF8PS intrinsic
-define void @test_top4mxbhf8ps() {
-; CHECK-LABEL: test_top4mxbhf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxbhf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXHBF8PS intrinsic
-define void @test_top4mxhbf8ps() {
-; CHECK-LABEL: test_top4mxhbf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxhbf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXBF8PS intrinsic
-define void @test_top4mxbf8ps() {
-; CHECK-LABEL: test_top4mxbf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxbf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXBSSPS intrinsic (MX INT8 S×S with BSR scaling)
-define void @test_top4mxbssps() {
-; CHECK-LABEL: test_top4mxbssps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxbssps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
-define void @test_tilemovcol(<16 x i32> %src) {
-; CHECK-LABEL: test_tilemovcol:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %src, i32 3)
- ret void
-}
-
-; Test TILEMOVROW intrinsic - write to tile (ACE extension)
-; Uses constant index - generates immediate form
-define void @test_tilemovrow_to_tile(<16 x i32> %src) {
-; CHECK-LABEL: test_tilemovrow_to_tile:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetrow(i8 1, <16 x i32> %src, i32 5)
- ret void
-}
-
-; Test TILEMOVCOL with register index
-define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
-; CHECK-LABEL: test_tilemovcol_reg:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetcol(i8 2, <16 x i32> %src, i32 %idx)
- ret void
-}
-
-; Test TILEMOVROW with register index
-define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
-; CHECK-LABEL: test_tilemovrow_reg:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetrow(i8 3, <16 x i32> %src, i32 %idx)
- ret void
-}
-
-; Intrinsic declarations
-declare void @llvm.x86.bsrinit()
-declare void @llvm.x86.bsrmovf(<16 x i32>, <16 x i32>)
-declare void @llvm.x86.bsrmovh.set(<16 x i32>)
-declare <16 x i32> @llvm.x86.bsrmovh.get()
-declare void @llvm.x86.bsrmovl.set(<16 x i32>)
-declare <16 x i32> @llvm.x86.bsrmovl.get()
-declare void @llvm.x86.tilesetcol(i8, <16 x i32>, i32)
-declare void @llvm.x86.tilesetrow(i8, <16 x i32>, i32)
-declare void @llvm.x86.top2bf16ps(i8, i8, i8)
-declare void @llvm.x86.top4buud(i8, i8, i8)
-declare void @llvm.x86.top4busd(i8, i8, i8)
-declare void @llvm.x86.top4bssd(i8, i8, i8)
-declare void @llvm.x86.top4bsud(i8, i8, i8)
-declare void @llvm.x86.top4mxhf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxbhf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxhbf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxbf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxbssps(i8, i8, i8, i8)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
similarity index 87%
rename from llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
rename to llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
index 4b114d5bc9c080..ab72e042e0b991 100644
--- a/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
@@ -1,10 +1,11 @@
-; ACE v1 Greedy Register Allocation Test
-; Tests that tile registers are allocated correctly in a separate pass
-; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
-;
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+
+; Test the tile register is allocated in a separate pass. ACE v1 alone, and
+; composed with AMX-TILE, must both work. ACE v1 extracts tile data with
+; TILEMOVROW rather than TILESTORED.
-; Test basic tile register allocation for ACE
define void @test_acev1_ra(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
; CHECK-LABEL: name: test_acev1_ra
; CHECK: PLDTILECFGV
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
new file mode 100644
index 00000000000000..d7a913f8e22900
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
@@ -0,0 +1,357 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,-avx10.2 -verify-machineinstrs --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs --show-mc-encoding | FileCheck %s
+
+; Test the ACE v1 intrinsics. acev1 implies its AVX10.1 and AMX-TILE baselines,
+; so it is sufficient alone. Nothing here depends on AVX10.2, which is above the
+; baseline, and requesting AMX-TILE explicitly is redundant but must not change
+; anything.
+
+; Test BSRINIT intrinsic
+define void @test_bsrinit() {
+; CHECK-LABEL: test_bsrinit:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrinit %bsr0 # encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrinit()
+ ret void
+}
+
+; Test BSRMOVF intrinsic
+define void @test_bsrmovf(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_bsrmovf:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0xc1]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with memory operand
+define void @test_bsrmovf_mem(<64 x i8> %a, ptr %p) {
+; CHECK-LABEL: test_bsrmovf_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf (%rdi), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x07]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %b = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with memory operand (base+index*scale+disp)
+define void @test_bsrmovf_mem_amode(<64 x i8> %a, ptr %base, i64 %idx) {
+; CHECK-LABEL: test_bsrmovf_mem_amode:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf 64(%rdi,%rsi,4), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x44,0xb7,0x01]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %elt = getelementptr i32, ptr %base, i64 %idx
+ %addr = getelementptr i8, ptr %elt, i64 64
+ %b = load <64 x i8>, ptr %addr
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with unaligned memory operand
+define void @test_bsrmovf_mem_unaligned(<64 x i8> %a, ptr %p) {
+; CHECK-LABEL: test_bsrmovf_mem_unaligned:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf (%rdi), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x07]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %b = load <64 x i8>, ptr %p, align 1
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with memory operand in first source (not foldable)
+define void @test_bsrmovf_mem_src1(ptr %p, <64 x i8> %b) {
+; CHECK-LABEL: test_bsrmovf_mem_src1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps (%rdi), %zmm1 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0x0f]
+; CHECK-NEXT: bsrmovf %zmm0, %zmm1, %bsr0 # encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %a = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVH set intrinsic
+define void @test_bsrmovh_set(<64 x i8> %a) {
+; CHECK-LABEL: test_bsrmovh_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %zmm0, %bsr0 # encoding: [0x62,0xf6,0xff,0x48,0x95,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrmovh.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVH set intrinsic with memory operand
+define void @test_bsrmovh_set_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovh_set_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh (%rdi), %bsr0 # encoding: [0x62,0xf6,0xff,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %a = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovh.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVH get intrinsic
+define <64 x i8> @test_bsrmovh_get() {
+; CHECK-LABEL: test_bsrmovh_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, %zmm0 # encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ ret <64 x i8> %result
+}
+
+; Test BSRMOVH get intrinsic with memory operand
+define void @test_bsrmovh_get_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovh_get_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7f,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ store <64 x i8> %result, ptr %p
+ ret void
+}
+
+; Test BSRMOVH get intrinsic with unaligned memory operand
+define void @test_bsrmovh_get_mem_unaligned(ptr %p) {
+; CHECK-LABEL: test_bsrmovh_get_mem_unaligned:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7f,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ store <64 x i8> %result, ptr %p, align 1
+ ret void
+}
+
+; Test BSRMOVL set intrinsic
+define void @test_bsrmovl_set(<64 x i8> %a) {
+; CHECK-LABEL: test_bsrmovl_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrmovl.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVL set intrinsic with memory operand
+define void @test_bsrmovl_set_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovl_set_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl (%rdi), %bsr0 # encoding: [0x62,0xf6,0xfe,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %a = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovl.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVL get intrinsic
+define <64 x i8> @test_bsrmovl_get() {
+; CHECK-LABEL: test_bsrmovl_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %bsr0, %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovl.get()
+ ret <64 x i8> %result
+}
+
+; Test BSRMOVL get intrinsic with memory operand
+define void @test_bsrmovl_get_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovl_get_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7e,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovl.get()
+ store <64 x i8> %result, ptr %p
+ ret void
+}
+
+; The outer products name only the accumulator tile by ID; both ZMM sources are
+; ordinary values assigned by the register allocator.
+
+; Test TOP2BF16PS intrinsic
+define void @test_top2bf16ps(<32 x bfloat> %a, <32 x bfloat> %b) {
+; CHECK-LABEL: test_top2bf16ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top2bf16ps %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x76,0x48,0x5c,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top2bf16ps(i8 0, <32 x bfloat> %a, <32 x bfloat> %b)
+ ret void
+}
+
+; Test TOP4BUUD intrinsic
+define void @test_top4buud(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4buud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4buud %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x74,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4buud(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4BUSD intrinsic
+define void @test_top4busd(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4busd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4busd %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x75,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4busd(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4BSSD intrinsic
+define void @test_top4bssd(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4bssd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bssd %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x77,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4bssd(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4BSUD intrinsic
+define void @test_top4bsud(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4bsud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bsud %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x76,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4bsud(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4MXHF8PS intrinsic
+define void @test_top4mxhf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxhf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXBHF8PS intrinsic
+define void @test_top4mxbhf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxbhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbhf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxbhf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXHBF8PS intrinsic
+define void @test_top4mxhbf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxhbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhbf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxhbf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXBF8PS intrinsic
+define void @test_top4mxbf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7c,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxbf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXBSSPS intrinsic (MX INT8 SxS with BSR scaling)
+define void @test_top4mxbssps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxbssps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbssps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x8f,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxbssps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
+define void @test_tilemovcol(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovcol:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xc0,0x03]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovcol.set(i8 0, <16 x i32> %src, i32 3)
+ ret void
+}
+
+; Test TILEMOVROW intrinsic - write to tile (ACE extension)
+; Uses constant index - generates immediate form
+define void @test_tilemovrow_to_tile(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovrow_to_tile:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1 # encoding: [0x62,0xf3,0xfd,0x48,0x07,0xc8,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovrow.set(i8 1, <16 x i32> %src, i32 5)
+ ret void
+}
+
+; Test TILEMOVCOL with register index
+define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovcol_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2 # encoding: [0x62,0xf2,0xc5,0x48,0x4b,0xd0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovcol.set(i8 2, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Test TILEMOVROW with register index
+define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovrow_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3 # encoding: [0x62,0xf2,0xc5,0x48,0x4a,0xd8]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovrow.set(i8 3, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Intrinsic declarations
+declare void @llvm.x86.bsrinit()
+declare void @llvm.x86.bsrmovf(<64 x i8>, <64 x i8>)
+declare void @llvm.x86.bsrmovh.set(<64 x i8>)
+declare <64 x i8> @llvm.x86.bsrmovh.get()
+declare void @llvm.x86.bsrmovl.set(<64 x i8>)
+declare <64 x i8> @llvm.x86.bsrmovl.get()
+declare void @llvm.x86.tilemovcol.set(i8, <16 x i32>, i32)
+declare void @llvm.x86.tilemovrow.set(i8, <16 x i32>, i32)
+declare void @llvm.x86.top2bf16ps(i8, <32 x bfloat>, <32 x bfloat>)
+declare void @llvm.x86.top4buud(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4busd(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4bssd(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4bsud(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4mxhf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxbhf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxhbf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxbf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxbssps(i8, <16 x i32>, <16 x i32>, i8)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
similarity index 94%
rename from llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
rename to llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
index 47228dfb8e614d..7586363819a695 100644
--- a/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
@@ -1,8 +1,10 @@
-; ACE v1 Outer Product Operations Test
-; Tests all ACE outer product variants: BF16, INT8 (signed/unsigned combinations)
-; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
-;
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs | FileCheck %s
+
+; Test all ACE outer product variants: BF16 and INT8 signed/unsigned
+; combinations. ACE v1 alone, and composed with AMX-TILE, must both work. ACE v1
+; extracts tile data with TILEMOVROW rather than TILESTORED.
; Test BF16 outer product (TOP2BF16PS)
define void @test_top2bf16ps(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
new file mode 100644
index 00000000000000..a97adffe41db8e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
@@ -0,0 +1,67 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -O0 \
+; RUN: -verify-machineinstrs | FileCheck %s \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 \
+; RUN: -verify-machineinstrs | FileCheck %s \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+
+; Test that tile data is not made volatile at -O0. X86VolatileTileData shortens
+; tile live ranges before fast register allocation by round-tripping every tile
+; through memory with TILESTORED/TILELOADD. ACE configures palette 2, where
+; those are unavailable, so the transform must be skipped and no tile load/store
+; may appear. -O2 is included as a control: the same IR must be clean at both
+; levels.
+;
+; Only one tile is live per function here: with the tile data left non-volatile,
+; anything that keeps a second tile live at -O0 reaches the row-by-row spill in
+; X86FastPreTileConfig, which does not yet produce valid SSA. Extend this test
+; once that is fixed.
+
+; A tile is produced and read back a row at a time, never stored as a tile.
+define void @extract_one_row(ptr %out) nounwind {
+; CHECK-LABEL: extract_one_row:
+; CHECK: movb $2, {{-?[0-9]+}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+entry:
+ %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %r = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x i32> %r, ptr %out, align 64
+ ret void
+}
+
+; Several rows of the same tile, so the tile stays live across extractions.
+define void @extract_several_rows(ptr %out) nounwind {
+; CHECK-LABEL: extract_several_rows:
+; CHECK: movb $2, {{-?[0-9]+}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+entry:
+ %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x i32> %r0, ptr %out, align 64
+ %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 7)
+ store volatile <16 x i32> %r7, ptr %out, align 64
+ ret void
+}
+
+; The converting row reads are gated the same way and must also stay clean.
+define void @extract_row_convert(ptr %out) nounwind {
+; CHECK-LABEL: extract_row_convert:
+; CHECK: movb $2, {{-?[0-9]+}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tcvtrowd2ps ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+entry:
+ %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %r = call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x float> %r, ptr %out, align 64
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
+declare <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
similarity index 88%
rename from llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
rename to llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
index 2d94260701e404..711de8d3c98473 100644
--- a/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
@@ -1,8 +1,10 @@
-; ACE v1 Basic Tile Operations Test
-; Tests basic ACE tile operations: tilezero, outer products
-; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
-;
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs | FileCheck %s
+
+; Test basic ACE tile operations: tilezero and outer products. ACE v1 alone, and
+; composed with AMX-TILE, must both work. ACE v1 extracts tile data with
+; TILEMOVROW rather than TILESTORED.
define void @test_acev1_basic(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
; CHECK-LABEL: test_acev1_basic:
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
new file mode 100644
index 00000000000000..52098739b8d8a4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
@@ -0,0 +1,124 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefix=ACE \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefix=ACE \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+
+; Test tile spilling. Nine simultaneously live tiles exceed the eight physical
+; TMM registers, so the tile register allocator must spill. ACE v1 configures
+; palette 2, which has no TILELOADD/TILESTORED, so a spill moves one row at a
+; time through a scratch ZMM. ACE v1 alone, and composed with AMX-TILE, must
+; both avoid TILELOADD/TILESTORED: the palette is 2 whenever ACE is enabled, and
+; those forms are unavailable there.
+;
+; tilemovrow prints the same mnemonic in both directions; the operand order
+; distinguishes them ($idx, %tmm, %zmm reads a row out, $idx, %zmm, %tmm writes
+; one in).
+
+; Nine live tiles accumulated with bf16 outer products.
+define void @spill_nine_tiles_bf16(ptr %out, <32 x bfloat> %a, <32 x bfloat> %b) nounwind {
+; ACE-LABEL: spill_nine_tiles_bf16:
+; ACE: ldtilecfg
+; Spill reads each of the 16 rows into a ZMM and stores it to the stack slot.
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; Reload loads each row back and writes it into the tile.
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
+entry:
+ %t0 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t3 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t4 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t5 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d0 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t0, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t1, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t2, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d3 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t3, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d4 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t4, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d5 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t5, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d6 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t6, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d7 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t7, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d8 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t8, <32 x bfloat> %a, <32 x bfloat> %b)
+
+ ; Keep all nine results live to the end so the allocator has to spill.
+ %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ store volatile <16 x i32> %r0, ptr %out, align 64
+ %r1 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ store volatile <16 x i32> %r1, ptr %out, align 64
+ %r2 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %r2, ptr %out, align 64
+ %r3 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ store volatile <16 x i32> %r3, ptr %out, align 64
+ %r4 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ store volatile <16 x i32> %r4, ptr %out, align 64
+ %r5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ store volatile <16 x i32> %r5, ptr %out, align 64
+ %r6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ store volatile <16 x i32> %r6, ptr %out, align 64
+ %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ store volatile <16 x i32> %r7, ptr %out, align 64
+ %r8 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+ store volatile <16 x i32> %r8, ptr %out, align 64
+
+ ret void
+}
+
+; Same pressure, integer outer products, to check the spill path is independent
+; of the producing instruction.
+define void @spill_nine_tiles_int(ptr %out, <64 x i8> %a) nounwind {
+; ACE-LABEL: spill_nine_tiles_int:
+; ACE: ldtilecfg
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
+entry:
+ %t0 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t3 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t4 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t5 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d0 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t0, <64 x i8> %a, <64 x i8> %a)
+ %d1 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t1, <64 x i8> %a, <64 x i8> %a)
+ %d2 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t2, <64 x i8> %a, <64 x i8> %a)
+ %d3 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t3, <64 x i8> %a, <64 x i8> %a)
+ %d4 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t4, <64 x i8> %a, <64 x i8> %a)
+ %d5 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t5, <64 x i8> %a, <64 x i8> %a)
+ %d6 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t6, <64 x i8> %a, <64 x i8> %a)
+ %d7 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t7, <64 x i8> %a, <64 x i8> %a)
+ %d8 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t8, <64 x i8> %a, <64 x i8> %a)
+
+ %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ store volatile <16 x i32> %r0, ptr %out, align 64
+ %r1 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ store volatile <16 x i32> %r1, ptr %out, align 64
+ %r2 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %r2, ptr %out, align 64
+ %r3 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ store volatile <16 x i32> %r3, ptr %out, align 64
+ %r4 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ store volatile <16 x i32> %r4, ptr %out, align 64
+ %r5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ store volatile <16 x i32> %r5, ptr %out, align 64
+ %r6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ store volatile <16 x i32> %r6, ptr %out, align 64
+ %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ store volatile <16 x i32> %r7, ptr %out, align 64
+ %r8 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+ store volatile <16 x i32> %r8, ptr %out, align 64
+
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/MC/Disassembler/X86/acev1.txt b/llvm/test/MC/Disassembler/X86/acev1.txt
new file mode 100644
index 00000000000000..08f3b4f34bad78
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/acev1.txt
@@ -0,0 +1,262 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+## Check that every byte sequence below is also reachable from the assembler.
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | llvm-mc -triple=x86_64 -mattr=+acev1 -filetype=obj -o %t.o
+# RUN: llvm-objdump -d --no-print-imm-hex %t.o | FileCheck %s --check-prefixes=ATT
+
+## BSR management
+
+# ATT: bsrinit %bsr0
+# INTEL: bsrinit bsr0
+0xc4,0xe2,0xfb,0x49,0xc0
+
+## BSRMOVF - fill both BSR halves from two ZMMs
+
+# ATT: bsrmovf %zmm2, %zmm1, %bsr0
+# INTEL: bsrmovf bsr0, zmm1, zmm2
+0x62,0xf6,0xf4,0x48,0x95,0xc2
+
+# ATT: bsrmovf %zmm31, %zmm30, %bsr0
+# INTEL: bsrmovf bsr0, zmm30, zmm31
+0x62,0x96,0x8c,0x40,0x95,0xc7
+
+# ATT: bsrmovf (%rax), %zmm1, %bsr0
+# INTEL: bsrmovf bsr0, zmm1, zmmword ptr [rax]
+0x62,0xf6,0xf4,0x48,0x95,0x00
+
+# ATT: bsrmovf -64(%rsp), %zmm0, %bsr0
+# INTEL: bsrmovf bsr0, zmm0, zmmword ptr [rsp - 64]
+0x62,0xf6,0xfc,0x48,0x95,0x44,0x24,0xff
+
+# ATT: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+# INTEL: bsrmovf bsr0, zmm10, zmmword ptr [rdx + 4*rax + 64]
+0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01
+
+# ATT: bsrmovf 268435456(%rbp,%r14,8), %zmm1, %bsr0
+# INTEL: bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10
+
+## BSRMOVH - high half, set (ZMM to BSR) and get (BSR to ZMM)
+
+# ATT: bsrmovh %zmm1, %bsr0
+# INTEL: bsrmovh bsr0, zmm1
+0x62,0xf6,0xff,0x48,0x95,0xc1
+
+# ATT: bsrmovh %zmm31, %bsr0
+# INTEL: bsrmovh bsr0, zmm31
+0x62,0x96,0xff,0x48,0x95,0xc7
+
+# ATT: bsrmovh (%rbx), %bsr0
+# INTEL: bsrmovh bsr0, zmmword ptr [rbx]
+0x62,0xf6,0xff,0x48,0x95,0x03
+
+# ATT: bsrmovh 64(%rdx,%rax,4), %bsr0
+# INTEL: bsrmovh bsr0, zmmword ptr [rdx + 4*rax + 64]
+0x62,0xf6,0xff,0x48,0x95,0x44,0x82,0x01
+
+# ATT: bsrmovh %bsr0, %zmm1
+# INTEL: bsrmovh zmm1, bsr0
+0x62,0xf6,0x7f,0x48,0x95,0xc1
+
+# ATT: bsrmovh %bsr0, %zmm31
+# INTEL: bsrmovh zmm31, bsr0
+0x62,0x96,0x7f,0x48,0x95,0xc7
+
+# ATT: bsrmovh %bsr0, (%rcx)
+# INTEL: bsrmovh zmmword ptr [rcx], bsr0
+0x62,0xf6,0x7f,0x48,0x95,0x01
+
+# ATT: bsrmovh %bsr0, 268435456(%rbp,%r14,8)
+# INTEL: bsrmovh zmmword ptr [rbp + 8*r14 + 268435456], bsr0
+0x62,0xb6,0x7f,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10
+
+## BSRMOVL - low half, set (ZMM to BSR) and get (BSR to ZMM)
+
+# ATT: bsrmovl %zmm2, %bsr0
+# INTEL: bsrmovl bsr0, zmm2
+0x62,0xf6,0xfe,0x48,0x95,0xc2
+
+# ATT: bsrmovl %zmm30, %bsr0
+# INTEL: bsrmovl bsr0, zmm30
+0x62,0x96,0xfe,0x48,0x95,0xc6
+
+# ATT: bsrmovl (%rdx), %bsr0
+# INTEL: bsrmovl bsr0, zmmword ptr [rdx]
+0x62,0xf6,0xfe,0x48,0x95,0x02
+
+# ATT: bsrmovl 64(%rdx,%rax,4), %bsr0
+# INTEL: bsrmovl bsr0, zmmword ptr [rdx + 4*rax + 64]
+0x62,0xf6,0xfe,0x48,0x95,0x44,0x82,0x01
+
+# ATT: bsrmovl %bsr0, %zmm3
+# INTEL: bsrmovl zmm3, bsr0
+0x62,0xf6,0x7e,0x48,0x95,0xc3
+
+# ATT: bsrmovl %bsr0, %zmm30
+# INTEL: bsrmovl zmm30, bsr0
+0x62,0x96,0x7e,0x48,0x95,0xc6
+
+# ATT: bsrmovl %bsr0, (%rsi)
+# INTEL: bsrmovl zmmword ptr [rsi], bsr0
+0x62,0xf6,0x7e,0x48,0x95,0x06
+
+# ATT: bsrmovl %bsr0, 268435456(%rbp,%r14,8)
+# INTEL: bsrmovl zmmword ptr [rbp + 8*r14 + 268435456], bsr0
+0x62,0xb6,0x7e,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10
+
+## TILEMOVCOL - write a ZMM into a tile column (ACE only, EVEX.W1)
+
+# ATT: tilemovcol $0, %zmm0, %tmm0
+# INTEL: tilemovcol tmm0, zmm0, 0
+0x62,0xf3,0xfd,0x48,0x2f,0xc0,0x00
+
+# ATT: tilemovcol $5, %zmm2, %tmm1
+# INTEL: tilemovcol tmm1, zmm2, 5
+0x62,0xf3,0xfd,0x48,0x2f,0xca,0x05
+
+# ATT: tilemovcol $15, %zmm31, %tmm7
+# INTEL: tilemovcol tmm7, zmm31, 15
+0x62,0x93,0xfd,0x48,0x2f,0xff,0x0f
+
+# ATT: tilemovcol %ecx, %zmm2, %tmm1
+# INTEL: tilemovcol tmm1, zmm2, ecx
+0x62,0xf2,0xf5,0x48,0x4b,0xca
+
+# ATT: tilemovcol %r14d, %zmm15, %tmm7
+# INTEL: tilemovcol tmm7, zmm15, r14d
+0x62,0xd2,0x8d,0x48,0x4b,0xff
+
+# ATT: tilemovcol %edx, %zmm16, %tmm1
+# INTEL: tilemovcol tmm1, zmm16, edx
+0x62,0xb2,0xed,0x48,0x4b,0xc8
+
+# ATT: tilemovcol %edx, %zmm31, %tmm1
+# INTEL: tilemovcol tmm1, zmm31, edx
+0x62,0x92,0xed,0x48,0x4b,0xcf
+
+## TILEMOVROW write - ZMM into a tile row (ACE only, EVEX.W1)
+
+# ATT: tilemovrow $0, %zmm0, %tmm0
+# INTEL: tilemovrow tmm0, zmm0, 0
+0x62,0xf3,0xfd,0x48,0x07,0xc0,0x00
+
+# ATT: tilemovrow $3, %zmm2, %tmm1
+# INTEL: tilemovrow tmm1, zmm2, 3
+0x62,0xf3,0xfd,0x48,0x07,0xca,0x03
+
+# ATT: tilemovrow $15, %zmm31, %tmm7
+# INTEL: tilemovrow tmm7, zmm31, 15
+0x62,0x93,0xfd,0x48,0x07,0xff,0x0f
+
+# ATT: tilemovrow %edx, %zmm3, %tmm2
+# INTEL: tilemovrow tmm2, zmm3, edx
+0x62,0xf2,0xed,0x48,0x4a,0xd3
+
+# ATT: tilemovrow %r14d, %zmm15, %tmm7
+# INTEL: tilemovrow tmm7, zmm15, r14d
+0x62,0xd2,0x8d,0x48,0x4a,0xff
+
+# ATT: tilemovrow %edx, %zmm16, %tmm1
+# INTEL: tilemovrow tmm1, zmm16, edx
+0x62,0xb2,0xed,0x48,0x4a,0xc8
+
+# ATT: tilemovrow %edx, %zmm31, %tmm1
+# INTEL: tilemovrow tmm1, zmm31, edx
+0x62,0x92,0xed,0x48,0x4a,0xcf
+
+## TILEMOVROW read - tile row into a ZMM (EVEX.W0, same opcode as write)
+
+# ATT: tilemovrow $0, %tmm0, %zmm0
+# INTEL: tilemovrow zmm0, tmm0, 0
+0x62,0xf3,0x7d,0x48,0x07,0xc0,0x00
+
+# ATT: tilemovrow $3, %tmm1, %zmm2
+# INTEL: tilemovrow zmm2, tmm1, 3
+0x62,0xf3,0x7d,0x48,0x07,0xd1,0x03
+
+# ATT: tilemovrow $15, %tmm7, %zmm31
+# INTEL: tilemovrow zmm31, tmm7, 15
+0x62,0x63,0x7d,0x48,0x07,0xff,0x0f
+
+# ATT: tilemovrow %edx, %tmm2, %zmm3
+# INTEL: tilemovrow zmm3, tmm2, edx
+0x62,0xf2,0x6d,0x48,0x4a,0xda
+
+# ATT: tilemovrow %r14d, %tmm7, %zmm31
+# INTEL: tilemovrow zmm31, tmm7, r14d
+0x62,0x62,0x0d,0x48,0x4a,0xff
+
+## TOP2BF16PS - rank-2 BF16 outer product
+
+# ATT: top2bf16ps %zmm3, %zmm2, %tmm1
+# INTEL: top2bf16ps tmm1, zmm2, zmm3
+0x62,0xf2,0x66,0x48,0x5c,0xca
+
+# ATT: top2bf16ps %zmm31, %zmm30, %tmm7
+# INTEL: top2bf16ps tmm7, zmm30, zmm31
+0x62,0x92,0x06,0x40,0x5c,0xfe
+
+## TOP4 - rank-4 INT8 outer products
+
+# ATT: top4buud %zmm3, %zmm2, %tmm1
+# INTEL: top4buud tmm1, zmm2, zmm3
+0x62,0xf2,0x64,0x48,0x5e,0xca
+
+# ATT: top4buud %zmm31, %zmm30, %tmm7
+# INTEL: top4buud tmm7, zmm30, zmm31
+0x62,0x92,0x04,0x40,0x5e,0xfe
+
+# ATT: top4busd %zmm3, %zmm2, %tmm1
+# INTEL: top4busd tmm1, zmm2, zmm3
+0x62,0xf2,0x65,0x48,0x5e,0xca
+
+# ATT: top4busd %zmm31, %zmm30, %tmm7
+# INTEL: top4busd tmm7, zmm30, zmm31
+0x62,0x92,0x05,0x40,0x5e,0xfe
+
+# ATT: top4bssd %zmm3, %zmm2, %tmm1
+# INTEL: top4bssd tmm1, zmm2, zmm3
+0x62,0xf2,0x67,0x48,0x5e,0xca
+
+# ATT: top4bssd %zmm31, %zmm30, %tmm7
+# INTEL: top4bssd tmm7, zmm30, zmm31
+0x62,0x92,0x07,0x40,0x5e,0xfe
+
+# ATT: top4bsud %zmm3, %zmm2, %tmm1
+# INTEL: top4bsud tmm1, zmm2, zmm3
+0x62,0xf2,0x66,0x48,0x5e,0xca
+
+# ATT: top4bsud %zmm31, %zmm30, %tmm7
+# INTEL: top4bsud tmm7, zmm30, zmm31
+0x62,0x92,0x06,0x40,0x5e,0xfe
+
+## TOP4MX - rank-4 MX outer products, BSR scaling mode in imm8
+
+# ATT: top4mxbf8ps $0, %zmm0, %zmm0, %tmm0
+# INTEL: top4mxbf8ps tmm0, zmm0, zmm0, 0
+0x62,0xf3,0x7c,0x48,0x8d,0xc0,0x00
+
+# ATT: top4mxbf8ps $7, %zmm3, %zmm2, %tmm1
+# INTEL: top4mxbf8ps tmm1, zmm2, zmm3, 7
+0x62,0xf3,0x64,0x48,0x8d,0xca,0x07
+
+# ATT: top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
+# INTEL: top4mxhf8ps tmm1, zmm2, zmm3, 7
+0x62,0xf3,0x65,0x48,0x8d,0xca,0x07
+
+# ATT: top4mxbhf8ps $3, %zmm5, %zmm4, %tmm2
+# INTEL: top4mxbhf8ps tmm2, zmm4, zmm5, 3
+0x62,0xf3,0x57,0x48,0x8d,0xd4,0x03
+
+# ATT: top4mxhbf8ps $1, %zmm7, %zmm6, %tmm3
+# INTEL: top4mxhbf8ps tmm3, zmm6, zmm7, 1
+0x62,0xf3,0x46,0x48,0x8d,0xde,0x01
+
+# ATT: top4mxbssps $15, %zmm9, %zmm8, %tmm4
+# INTEL: top4mxbssps tmm4, zmm8, zmm9, 15
+0x62,0xd3,0x37,0x48,0x8f,0xe0,0x0f
+
+# ATT: top4mxbssps $255, %zmm31, %zmm30, %tmm7
+# INTEL: top4mxbssps tmm7, zmm30, zmm31, 255
+0x62,0x93,0x07,0x40,0x8f,0xfe,0xff
diff --git a/llvm/test/MC/X86/ACE/ace-att.s b/llvm/test/MC/X86/ACEV1/acev1-att.s
similarity index 63%
rename from llvm/test/MC/X86/ACE/ace-att.s
rename to llvm/test/MC/X86/ACEV1/acev1-att.s
index cf43cb899bb2c8..a6f13e2c79ab32 100644
--- a/llvm/test/MC/X86/ACE/ace-att.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-att.s
@@ -12,6 +12,14 @@
// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
bsrmovf (%rax), %zmm1, %bsr0
+// CHECK: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
+ bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+
+// CHECK: bsrmovf 268435456(%rbp,%r14,8), %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10]
+ bsrmovf 268435456(%rbp,%r14,8), %zmm1, %bsr0
+
// CHECK: bsrmovh %zmm1, %bsr0
// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0xc1]
bsrmovh %zmm1, %bsr0
@@ -88,22 +96,50 @@
// CHECK: encoding: [0x62,0xf3,0x65,0x48,0x8d,0xca,0x07]
top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
-// CHECK: top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
-// CHECK: encoding: [0x62,0xf3,0x5f,0x48,0x8d,0xd5,0x03]
- top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
+// CHECK: top4mxbhf8ps $3, %zmm5, %zmm4, %tmm2
+// CHECK: encoding: [0x62,0xf3,0x57,0x48,0x8d,0xd4,0x03]
+ top4mxbhf8ps $3, %zmm5, %zmm4, %tmm2
-// CHECK: top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
-// CHECK: encoding: [0x62,0xf3,0x4e,0x48,0x8d,0xdf,0x01]
- top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
+// CHECK: top4mxhbf8ps $1, %zmm7, %zmm6, %tmm3
+// CHECK: encoding: [0x62,0xf3,0x46,0x48,0x8d,0xde,0x01]
+ top4mxhbf8ps $1, %zmm7, %zmm6, %tmm3
-// CHECK: top4mxbssps $15, %zmm8, %zmm9, %tmm4
-// CHECK: encoding: [0x62,0xd3,0x3f,0x48,0x8f,0xe1,0x0f]
- top4mxbssps $15, %zmm8, %zmm9, %tmm4
+// CHECK: top4mxbssps $15, %zmm9, %zmm8, %tmm4
+// CHECK: encoding: [0x62,0xd3,0x37,0x48,0x8f,0xe0,0x0f]
+ top4mxbssps $15, %zmm9, %zmm8, %tmm4
// CHECK: top4buud %zmm31, %zmm30, %tmm7
-// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+// CHECK: encoding: [0x62,0x92,0x04,0x40,0x5e,0xfe]
top4buud %zmm31, %zmm30, %tmm7
-// CHECK: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
-// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
- bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+// CHECK: tilemovrow %edx, %zmm31, %tmm1
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4a,0xcf]
+ tilemovrow %edx, %zmm31, %tmm1
+
+// CHECK: tilemovrow %edx, %zmm16, %tmm1
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4a,0xc8]
+ tilemovrow %edx, %zmm16, %tmm1
+
+// CHECK: tilemovcol %edx, %zmm31, %tmm1
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4b,0xcf]
+ tilemovcol %edx, %zmm31, %tmm1
+
+// CHECK: tilemovcol %edx, %zmm16, %tmm1
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4b,0xc8]
+ tilemovcol %edx, %zmm16, %tmm1
+
+// CHECK: top2bf16ps %zmm31, %zmm30, %tmm7
+// CHECK: encoding: [0x62,0x92,0x06,0x40,0x5c,0xfe]
+ top2bf16ps %zmm31, %zmm30, %tmm7
+
+// CHECK: top4busd %zmm15, %zmm31, %tmm1
+// CHECK: encoding: [0x62,0x92,0x05,0x48,0x5e,0xcf]
+ top4busd %zmm15, %zmm31, %tmm1
+
+// CHECK: top4bssd %zmm31, %zmm15, %tmm1
+// CHECK: encoding: [0x62,0xd2,0x07,0x40,0x5e,0xcf]
+ top4bssd %zmm31, %zmm15, %tmm1
+
+// CHECK: top4bsud %zmm0, %zmm16, %tmm1
+// CHECK: encoding: [0x62,0xb2,0x7e,0x48,0x5e,0xc8]
+ top4bsud %zmm0, %zmm16, %tmm1
diff --git a/llvm/test/MC/X86/ACE/ace-error.s b/llvm/test/MC/X86/ACEV1/acev1-error.s
similarity index 100%
rename from llvm/test/MC/X86/ACE/ace-error.s
rename to llvm/test/MC/X86/ACEV1/acev1-error.s
diff --git a/llvm/test/MC/X86/ACE/ace-intel.s b/llvm/test/MC/X86/ACEV1/acev1-intel.s
similarity index 74%
rename from llvm/test/MC/X86/ACE/ace-intel.s
rename to llvm/test/MC/X86/ACEV1/acev1-intel.s
index 81b5d8e0f7e249..daab4536709ece 100644
--- a/llvm/test/MC/X86/ACE/ace-intel.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-intel.s
@@ -12,6 +12,10 @@
// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
bsrmovf bsr0, zmm1, zmmword ptr [rax]
+// CHECK: bsrmovf bsr0, zmm10, zmmword ptr [rdx + 4*rax + 64]
+// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
+ bsrmovf bsr0, zmm10, zmmword ptr [rdx + 4*rax + 64]
+
// CHECK: bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
// CHECK: encoding: [0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10]
bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
@@ -105,5 +109,37 @@
top4mxbssps tmm4, zmm8, zmm9, 15
// CHECK: top4buud tmm7, zmm30, zmm31
-// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+// CHECK: encoding: [0x62,0x92,0x04,0x40,0x5e,0xfe]
top4buud tmm7, zmm30, zmm31
+
+// CHECK: tilemovrow tmm1, zmm31, edx
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4a,0xcf]
+ tilemovrow tmm1, zmm31, edx
+
+// CHECK: tilemovrow tmm1, zmm16, edx
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4a,0xc8]
+ tilemovrow tmm1, zmm16, edx
+
+// CHECK: tilemovcol tmm1, zmm31, edx
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4b,0xcf]
+ tilemovcol tmm1, zmm31, edx
+
+// CHECK: tilemovcol tmm1, zmm16, edx
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4b,0xc8]
+ tilemovcol tmm1, zmm16, edx
+
+// CHECK: top2bf16ps tmm7, zmm30, zmm31
+// CHECK: encoding: [0x62,0x92,0x06,0x40,0x5c,0xfe]
+ top2bf16ps tmm7, zmm30, zmm31
+
+// CHECK: top4busd tmm1, zmm31, zmm15
+// CHECK: encoding: [0x62,0x92,0x05,0x48,0x5e,0xcf]
+ top4busd tmm1, zmm31, zmm15
+
+// CHECK: top4bssd tmm1, zmm15, zmm31
+// CHECK: encoding: [0x62,0xd2,0x07,0x40,0x5e,0xcf]
+ top4bssd tmm1, zmm15, zmm31
+
+// CHECK: top4bsud tmm1, zmm16, zmm0
+// CHECK: encoding: [0x62,0xb2,0x7e,0x48,0x5e,0xc8]
+ top4bsud tmm1, zmm16, zmm0
>From 69ea6c56b0f0100533422648b12d4c0300981bda Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Wed, 9 Sep 2026 16:49:19 +0530
Subject: [PATCH 3/6] [X86][ACEV1] Address review comments
- Drop the AMX_TILE dependency; under acev1 enable only the AMX
instructions that ACE v1 actually shares.
- Namespace the ACE-invented intrinsics under llvm.x86.acev1.* and leave
the shared AMX-AVX512 ones in the flat llvm.x86.* namespace.
- Sync the public intrinsic and builtin names with GCC, and add the
helper macros from the ACE v1 spec.
- Tighten Sema checking of the outer-product group bits, and add driver,
preprocessor and feature-error test coverage.
- Spell BSR0 literally in the AsmStrings with Defs/Uses instead of
carrying it as a custom operand, which removes the bespoke handling
from X86AsmParser and both instruction printers.
- Drop the unused BSR register class; like TMMCFG, BSR0 needs only its
X86Reg definition and the getReservedRegs entry.
- Adopt the operand-type record naming convention, 't' for a TILE and
'e' for a GR32 index, on both the instructions and their pseudos,
retiring the ad-hoc _REG suffix.
- Use WriteSystem rather than WriteVecLogic for the tile outer products
and the BSR moves.
- Pass tiles by pointer in the unit tests, and drop comments that only
restate the code.
- Update clang/test/CIR/CodeGen/attr-target-x86.c for the new acev1
feature; its negated target-feature lists are exhaustive, so adding a
subtarget feature changes them.
---
clang/include/clang/Basic/BuiltinsX86_64.td | 60 +--
.../clang/Basic/DiagnosticSemaKinds.td | 2 +
clang/include/clang/Options/Options.td | 6 +-
clang/include/clang/Sema/SemaX86.h | 1 +
clang/lib/Basic/Targets/X86.cpp | 13 +-
clang/lib/Headers/acev1intrin.h | 355 +++++++++-----
clang/lib/Sema/SemaX86.cpp | 32 +-
clang/test/CIR/CodeGen/attr-target-x86.c | 8 +-
clang/test/CodeGen/X86/acev1.c | 45 +-
clang/test/CodeGen/X86/acev1_api.c | 103 ++--
clang/test/CodeGen/X86/acev1_bsr.c | 60 +--
clang/test/CodeGen/X86/acev1_errors.c | 109 ++++-
clang/test/CodeGen/X86/acev1_feature_errors.c | 30 ++
clang/test/CodeGen/X86/acev1_inline_asm.c | 160 ++-----
clang/test/CodeGen/X86/acev1_mxfp8.c | 51 --
clang/test/CodeGen/X86/acev1_tile_movement.c | 80 +---
clang/test/Driver/x86-target-features.c | 7 +
clang/test/Preprocessor/x86_target_features.c | 17 +-
llvm/include/llvm/IR/IntrinsicsX86.td | 164 ++-----
.../lib/Target/X86/AsmParser/X86AsmParser.cpp | 50 --
.../X86/MCTargetDesc/X86ATTInstPrinter.cpp | 92 +---
.../X86/MCTargetDesc/X86ATTInstPrinter.h | 1 -
.../X86/MCTargetDesc/X86IntelInstPrinter.cpp | 88 +---
.../X86/MCTargetDesc/X86IntelInstPrinter.h | 1 -
llvm/lib/Target/X86/X86.td | 12 +-
llvm/lib/Target/X86/X86ExpandPseudo.cpp | 81 ++--
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 3 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 166 +++----
llvm/lib/Target/X86/X86InstrACE.td | 451 +++++++-----------
llvm/lib/Target/X86/X86LowerAMXType.cpp | 49 +-
llvm/lib/Target/X86/X86LowerTileCopy.cpp | 2 +-
llvm/lib/Target/X86/X86RegisterInfo.cpp | 25 +-
llvm/lib/Target/X86/X86RegisterInfo.td | 7 -
llvm/lib/TargetParser/X86TargetParser.cpp | 7 +-
.../test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll | 28 +-
.../CodeGen/X86/ACEV1/acev1-intrinsics.ll | 180 +++----
.../CodeGen/X86/ACEV1/acev1-outer-product.ll | 56 +--
.../CodeGen/X86/ACEV1/acev1-tile-basic.ll | 18 +-
.../CodeGen/X86/ACEV1/acev1-tile-spill.ll | 40 +-
llvm/test/MC/X86/ACEV1/acev1-att.s | 20 +
llvm/test/MC/X86/ACEV1/acev1-error.s | 27 +-
llvm/test/MC/X86/ACEV1/acev1-intel.s | 20 +
42 files changed, 1245 insertions(+), 1482 deletions(-)
create mode 100644 clang/test/CodeGen/X86/acev1_feature_errors.c
delete mode 100644 clang/test/CodeGen/X86/acev1_mxfp8.c
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index 053146d64fc855..8d1d19367e0c6e 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -223,7 +223,7 @@ let Features = "amx-tile", Attributes = [NoThrow] in {
def tilestored64_internal : X86Builtin<"void(unsigned short, unsigned short, void *, size_t, _Vector<256, int>)">;
}
-let Features = "amx-tile", Attributes = [NoThrow] in {
+let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
def tilezero_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short)">;
}
@@ -256,7 +256,7 @@ let Features = "amx-fp8", Attributes = [NoThrow] in {
def tdphf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-tile", Attributes = [NoThrow] in {
+let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
def tile_loadconfig : X86Builtin<"void(void const *)">;
def tile_storeconfig : X86Builtin<"void(void const *)">;
def tilerelease : X86Builtin<"void()">;
@@ -396,63 +396,63 @@ let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>]
// forms; the lowering code checks whether the index is constant and selects the
// appropriate instruction.
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
- def tilemovcol_set : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ def tilemovcolinsert : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
// ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
- def tilemovrow_set : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ def tilemovrowinsert : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
}
// Tile movement internal (for IR-level operations).
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
- def tilemovcol_set_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+ def tilemovcolinsert_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
// ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
- def tilemovrow_set_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+ def tilemovrowinsert_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
}
// BSR management.
let Features = "acev1", Attributes = [NoThrow] in {
- def bsrinit : X86Builtin<"void()">;
+ def bsr0init : X86Builtin<"void()">;
}
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
- def bsrmovf : X86Builtin<"void(_Vector<64, char>, _Vector<64, char>)">;
- def bsrmovh_set : X86Builtin<"void(_Vector<64, char>)">;
- def bsrmovh_get : X86Builtin<"_Vector<64, char>()">;
- def bsrmovl_set : X86Builtin<"void(_Vector<64, char>)">;
- def bsrmovl_get : X86Builtin<"_Vector<64, char>()">;
+ def bsr0movf : X86Builtin<"void(_Vector<64, char>, _Vector<64, char>)">;
+ def bsr0movhinsert : X86Builtin<"void(_Vector<64, char>)">;
+ def bsr0movhextract : X86Builtin<"_Vector<64, char>()">;
+ def bsr0movlinsert : X86Builtin<"void(_Vector<64, char>)">;
+ def bsr0movlextract : X86Builtin<"_Vector<64, char>()">;
}
// Outer products - user-facing (immediate tile ID, ZMM source values).
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
- def top4buud : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
- def top4busd : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
- def top4bssd : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
- def top4bsud : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4buud : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>)">;
+ def top4busd : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>)">;
+ def top4bssd : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>)">;
+ def top4bsud : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>)">;
}
// Mixed precision outer products - user-facing.
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
- def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
- def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
- def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
- def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
- def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>, _Constant unsigned char)">;
+ def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>, _Constant unsigned char)">;
+ def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>, _Constant unsigned char)">;
+ def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>, _Constant unsigned char)">;
+ def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Vector<64, char>, _Vector<64, char>, _Constant unsigned char)">;
}
// Outer products - internal (for IR-level operations with tile values).
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def top2bf16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
- def top4buud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
- def top4busd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
- def top4bssd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
- def top4bsud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4buud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4busd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4bssd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4bsud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
}
// Mixed precision internal.
let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
- def top4mxhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
- def top4mxbhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
- def top4mxhbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
- def top4mxbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
- def top4mxbssps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4mxbhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4mxhbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4mxbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
+ def top4mxbssps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<64, char>, _Vector<64, char>)">;
}
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 3e56980e22902b..78a2316ea07ccf 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11681,6 +11681,8 @@ def err_x86_builtin_invalid_scale : Error<
"scale argument must be 1, 2, 4, or 8">;
def err_x86_builtin_tile_arg_duplicate : Error<
"tile arguments must refer to different tiles">;
+def err_x86_builtin_invalid_scale_group : Error<
+ "scale group argument must have bits 2, 5, 6, and 7 clear">;
def err_builtin_target_unsupported : Error<
"builtin is not supported on this target">;
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index c7cc6ca5a145c6..fcb79029701c76 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -7462,8 +7462,10 @@ def mamx_tile : Flag<["-"], "mamx-tile">, Group<m_x86_Features_Group>;
def mno_amx_tile : Flag<["-"], "mno-amx-tile">, Group<m_x86_Features_Group>;
def mamx_movrs: Flag<["-"], "mamx-movrs">, Group<m_x86_Features_Group>;
def mno_amx_movrs: Flag<["-"], "mno-amx-movrs">, Group<m_x86_Features_Group>;
-def macev1 : Flag<["-"], "macev1">, Group<m_x86_Features_Group>;
-def mno_acev1 : Flag<["-"], "mno-acev1">, Group<m_x86_Features_Group>;
+def macev1 : Flag<["-"], "macev1">, Group<m_x86_Features_Group>,
+ HelpText<"Enable ACE V1 instructions">;
+def mno_acev1 : Flag<["-"], "mno-acev1">, Group<m_x86_Features_Group>,
+ HelpText<"Disable ACE V1 instructions">;
def mcmpccxadd : Flag<["-"], "mcmpccxadd">, Group<m_x86_Features_Group>;
def mno_cmpccxadd : Flag<["-"], "mno-cmpccxadd">, Group<m_x86_Features_Group>;
def msse : Flag<["-"], "msse">, Group<m_x86_Features_Group>;
diff --git a/clang/include/clang/Sema/SemaX86.h b/clang/include/clang/Sema/SemaX86.h
index 00003142a50f39..a087e7db83f6f4 100644
--- a/clang/include/clang/Sema/SemaX86.h
+++ b/clang/include/clang/Sema/SemaX86.h
@@ -30,6 +30,7 @@ class SemaX86 : public SemaBase {
bool CheckBuiltinGatherScatterScale(unsigned BuiltinID, CallExpr *TheCall);
bool CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall);
bool CheckBuiltinTileArgumentsRange(CallExpr *TheCall, ArrayRef<int> ArgNums);
+ bool CheckBuiltinScaleGroup(CallExpr *TheCall, int ArgNum);
bool CheckBuiltinTileDuplicate(CallExpr *TheCall, ArrayRef<int> ArgNums);
bool CheckBuiltinTileRangeAndDuplicate(CallExpr *TheCall,
ArrayRef<int> ArgNums);
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 120324c31d550d..3f6640a094bda4 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -88,13 +88,12 @@ static const char *const GCCRegNames[] = {
"zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", "zmm16", "zmm17",
"zmm18", "zmm19", "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25",
"zmm26", "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", "k0", "k1",
- "k2", "k3", "k4", "k5", "k6", "k7",
- "cr0", "cr2", "cr3", "cr4", "cr8",
- "dr0", "dr1", "dr2", "dr3", "dr6", "dr7",
- "bnd0", "bnd1", "bnd2", "bnd3",
- "tmm0", "tmm1", "tmm2", "tmm3", "tmm4", "tmm5", "tmm6", "tmm7",
- "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
- "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+ "k2", "k3", "k4", "k5", "k6", "k7", "cr0", "cr2",
+ "cr3", "cr4", "cr8", "dr0", "dr1", "dr2", "dr3", "dr6",
+ "dr7", "bnd0", "bnd1", "bnd2", "bnd3", "tmm0", "tmm1", "tmm2",
+ "tmm3", "tmm4", "tmm5", "tmm6", "tmm7", "r16", "r17", "r18",
+ "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26",
+ "r27", "r28", "r29", "r30", "r31", "bsr0",
};
const TargetInfo::AddlRegName AddlRegNames[] = {
diff --git a/clang/lib/Headers/acev1intrin.h b/clang/lib/Headers/acev1intrin.h
index 3cbab91759d2f1..cd5087b7d009d1 100644
--- a/clang/lib/Headers/acev1intrin.h
+++ b/clang/lib/Headers/acev1intrin.h
@@ -13,12 +13,14 @@
#ifndef __ACEV1INTRIN_H
#define __ACEV1INTRIN_H
-#ifdef __x86_64__
+#if defined(__x86_64__) && defined(__SSE2__)
/* Define the default attributes for the functions in this file. */
#define __DEFAULT_FN_ATTRS_ACE \
__attribute__((__always_inline__, __nodebug__, __target__("acev1")))
+// clang-format off
+
/// Load tile configuration from a 64-byte memory location. For ACE
/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1),
/// ACE tiles have fixed dimensions of 16 rows x 64 bytes, so per-tile
@@ -60,7 +62,8 @@ _tile_ace_storeconfig(void *__config) {
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> TILERELEASE </c> instruction.
-static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_tile_ace_release(void) {
__builtin_ia32_tilerelease();
}
@@ -89,8 +92,8 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// Source ZMM vector (__m512i) containing 16 doublewords.
/// \param idx
/// Column index (0-15). Immediate or register form selected automatically.
-#define _tile_setcol(dst, src, idx) \
- __builtin_ia32_tilemovcol_set((dst), (src), (idx))
+#define _tile_insertcol(dst, src, idx) \
+ __builtin_ia32_tilemovcolinsert((dst), (__v16si)(src), (idx))
/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written
/// as a horizontal row in the tile at the specified index.
@@ -105,8 +108,8 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// Source ZMM vector (__m512i) containing 16 doublewords.
/// \param idx
/// Row index (0-15). Immediate or register form selected automatically.
-#define _tile_setrow(dst, src, idx) \
- __builtin_ia32_tilemovrow_set((dst), (src), (idx))
+#define _tile_insertrow(dst, src, idx) \
+ __builtin_ia32_tilemovrowinsert((dst), (__v16si)(src), (idx))
/// Initialize the Block Scale Register (BSR), setting all 128 scale bytes to
/// 0x7F (the E8M0 encoding of 1.0), the same state LDTILECFG leaves it in.
@@ -116,8 +119,9 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRINIT </c> instruction.
-static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) {
- __builtin_ia32_bsrinit();
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_bsr0_init(void) {
+ __builtin_ia32_bsr0init();
}
/// Load the full BSR (128 scale bytes) from two ZMM registers. The high half
@@ -131,9 +135,9 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) {
/// ZMM with the 64 A-operand scale bytes for the BSR high half.
/// \param __src2
/// ZMM with the 64 B-operand scale bytes for the BSR low half.
-static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1,
- __m512i __src2) {
- __builtin_ia32_bsrmovf((__v64qi)__src1, (__v64qi)__src2);
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_bsr0_insertfull(__m512i __src1, __m512i __src2) {
+ __builtin_ia32_bsr0movf((__v64qi)__src1, (__v64qi)__src2);
}
/// Load the high half of BSR (64 A-operand scale bytes) from a ZMM register.
@@ -144,8 +148,9 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1,
///
/// \param __src
/// ZMM with 64 scale bytes to write to the BSR high half.
-static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) {
- __builtin_ia32_bsrmovh_set((__v64qi)__src);
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_bsr0_inserth(__m512i __src) {
+ __builtin_ia32_bsr0movhinsert((__v64qi)__src);
}
/// Read the high half of BSR (64 A-operand scale bytes) to a ZMM register.
@@ -156,8 +161,9 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) {
///
/// \returns
/// ZMM containing the 64 scale bytes from the BSR high half.
-static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) {
- return (__m512i)__builtin_ia32_bsrmovh_get();
+static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE
+_bsr0_extracth(void) {
+ return (__m512i)__builtin_ia32_bsr0movhextract();
}
/// Load the low half of BSR (64 B-operand scale bytes) from a ZMM register.
@@ -168,8 +174,9 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) {
///
/// \param __src
/// ZMM with 64 scale bytes to write to the BSR low half.
-static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) {
- __builtin_ia32_bsrmovl_set((__v64qi)__src);
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_bsr0_insertl(__m512i __src) {
+ __builtin_ia32_bsr0movlinsert((__v64qi)__src);
}
/// Read the low half of BSR (64 B-operand scale bytes) to a ZMM register.
@@ -180,8 +187,9 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) {
///
/// \returns
/// ZMM containing the 64 scale bytes from the BSR low half.
-static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
- return (__m512i)__builtin_ia32_bsrmovl_get();
+static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE
+_bsr0_extractl(void) {
+ return (__m512i)__builtin_ia32_bsr0movlextract();
}
/// Compute 2-way outer product of BF16 pairs, accumulating to FP32.
@@ -198,7 +206,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// First source ZMM vector (__m512bh) containing 32 BF16 values.
/// \param src2
/// Second source ZMM vector (__m512bh) containing 32 BF16 values.
-#define _tile_top2bf16ps(dst, src1, src2) \
+#define _tile_op2bf16_ps(dst, src1, src2) \
__builtin_ia32_top2bf16ps((dst), (__v32bf)(src1), (__v32bf)(src2))
/// Compute 4-way outer product of unsigned x unsigned bytes to INT32.
@@ -215,8 +223,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// First source ZMM vector (__m512i) containing 64 unsigned bytes.
/// \param src2
/// Second source ZMM vector (__m512i) containing 64 unsigned bytes.
-#define _tile_top4buud(dst, src1, src2) \
- __builtin_ia32_top4buud((dst), (__v64qs)(src1), (__v64qs)(src2))
+#define _tile_op4buud_epi32(dst, src1, src2) \
+ __builtin_ia32_top4buud((dst), (__v64qi)(src1), (__v64qi)(src2))
/// Compute 4-way outer product of unsigned x signed bytes to INT32.
/// Each group of 4 byte pairs (unsigned from src1, signed from src2)
@@ -232,8 +240,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// First source ZMM vector (__m512i) containing 64 unsigned bytes.
/// \param src2
/// Second source ZMM vector (__m512i) containing 64 signed bytes.
-#define _tile_top4busd(dst, src1, src2) \
- __builtin_ia32_top4busd((dst), (__v64qs)(src1), (__v64qs)(src2))
+#define _tile_op4busd_epi32(dst, src1, src2) \
+ __builtin_ia32_top4busd((dst), (__v64qi)(src1), (__v64qi)(src2))
/// Compute 4-way outer product of signed x signed bytes to INT32.
/// Each group of 4 signed byte pairs produces 4 products accumulated
@@ -249,8 +257,8 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// First source ZMM vector (__m512i) containing 64 signed bytes.
/// \param src2
/// Second source ZMM vector (__m512i) containing 64 signed bytes.
-#define _tile_top4bssd(dst, src1, src2) \
- __builtin_ia32_top4bssd((dst), (__v64qs)(src1), (__v64qs)(src2))
+#define _tile_op4bssd_epi32(dst, src1, src2) \
+ __builtin_ia32_top4bssd((dst), (__v64qi)(src1), (__v64qi)(src2))
/// Compute 4-way outer product of signed x unsigned bytes to INT32.
/// Each group of 4 byte pairs (signed from src1, unsigned from src2)
@@ -266,12 +274,35 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// First source ZMM vector (__m512i) containing 64 signed bytes.
/// \param src2
/// Second source ZMM vector (__m512i) containing 64 unsigned bytes.
-#define _tile_top4bsud(dst, src1, src2) \
- __builtin_ia32_top4bsud((dst), (__v64qs)(src1), (__v64qs)(src2))
+#define _tile_op4bsud_epi32(dst, src1, src2) \
+ __builtin_ia32_top4bsud((dst), (__v64qi)(src1), (__v64qi)(src2))
+
+/// Select the A-input scale group within the BSR for the mixed precision
+/// outer product intrinsics. Combine with \c _MM_ACE_SCALE_B using a bitwise
+/// OR to form the scale group selector.
+///
+/// \headerfile <immintrin.h>
+///
+/// \param g
+/// A-input scale group number (0-3).
+/// \returns
+/// The A-input field of the scale group selector.
+#define _MM_ACE_SCALE_A(g) ((g) << 0)
+
+/// Select the B-input scale group within the BSR for the mixed precision
+/// outer product intrinsics. Combine with \c _MM_ACE_SCALE_A using a bitwise
+/// OR to form the scale group selector.
+///
+/// \headerfile <immintrin.h>
+///
+/// \param g
+/// B-input scale group number (0-3).
+/// \returns
+/// The B-input field of the scale group selector.
+#define _MM_ACE_SCALE_B(g) ((g) << 3)
-/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
-/// Multiplies HF8 values from both sources, converting to FP32 and
-/// accumulating into the destination tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E4M3, B = FP8 E4M3. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -280,17 +311,17 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM vector (__m512i) containing HF8 values.
+/// The A input: ZMM vector (__m512i) of FP8 E4M3 (HF8) values.
/// \param src2
-/// Second source ZMM vector (__m512i) containing HF8 values.
+/// The B input: ZMM vector (__m512i) of FP8 E4M3 (HF8) values.
/// \param imm
-/// 8-bit control immediate for scaling/rounding options.
-#define _tile_top4mxhf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxhf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
+#define _tile_op4mxhf8_ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxhf8ps((dst), (__v64qi)(src1), (__v64qi)(src2), (imm))
-/// Compute 4-way mixed precision outer product with BF8/HF8 format.
-/// Multiplies BF8 values from src1 with HF8 (E4M3) values from src2,
-/// converting to FP32 and accumulating into the destination tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E5M2, B = FP8 E4M3. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -299,17 +330,17 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM vector (__m512i) containing BF8 values.
+/// The A input: ZMM vector (__m512i) of FP8 E5M2 (BF8) values.
/// \param src2
-/// Second source ZMM vector (__m512i) containing HF8 values.
+/// The B input: ZMM vector (__m512i) of FP8 E4M3 (HF8) values.
/// \param imm
-/// 8-bit control immediate for scaling/rounding options.
-#define _tile_top4mxbhf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbhf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
+#define _tile_op4mxbhf8_ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbhf8ps((dst), (__v64qi)(src1), (__v64qi)(src2), (imm))
-/// Compute 4-way mixed precision outer product with HF8/BF8 format.
-/// Multiplies HF8 (E4M3) values from src1 with BF8 values from src2,
-/// converting to FP32 and accumulating into the destination tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E4M3, B = FP8 E5M2. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -318,17 +349,17 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM vector (__m512i) containing HF8 values.
+/// The A input: ZMM vector (__m512i) of FP8 E4M3 (HF8) values.
/// \param src2
-/// Second source ZMM vector (__m512i) containing BF8 values.
+/// The B input: ZMM vector (__m512i) of FP8 E5M2 (BF8) values.
/// \param imm
-/// 8-bit control immediate for scaling/rounding options.
-#define _tile_top4mxhbf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxhbf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
+#define _tile_op4mxhbf8_ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxhbf8ps((dst), (__v64qi)(src1), (__v64qi)(src2), (imm))
-/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
-/// Multiplies BF8 values from both sources, converting to FP32 and
-/// accumulating into the destination tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E5M2, B = FP8 E5M2. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -337,17 +368,18 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM vector (__m512i) containing BF8 values.
+/// The A input: ZMM vector (__m512i) of FP8 E5M2 (BF8) values.
/// \param src2
-/// Second source ZMM vector (__m512i) containing BF8 values.
+/// The B input: ZMM vector (__m512i) of FP8 E5M2 (BF8) values.
/// \param imm
-/// 8-bit control immediate for scaling/rounding options.
-#define _tile_top4mxbf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
+#define _tile_op4mxbf8_ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbf8ps((dst), (__v64qi)(src1), (__v64qi)(src2), (imm))
-/// Compute 4-way mixed precision outer product of signed INT8 with BSR
-/// scaling. Multiplies signed bytes, applies scale factors from the BSR,
-/// and accumulates FP32 results into the destination tile.
+/// Rank-4 MX INT8 outer product. A = MX INT8 signed, B = MX INT8 signed.
+/// OCP MX block scaling via the BSR, with FP32 accumulation into the
+/// destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -356,13 +388,102 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM vector (__m512i) containing signed bytes.
+/// The A input: ZMM vector (__m512i) of MX INT8 signed values.
/// \param src2
-/// Second source ZMM vector (__m512i) containing signed bytes.
+/// The B input: ZMM vector (__m512i) of MX INT8 signed values.
/// \param imm
-/// 8-bit immediate selecting BSR scale factors to apply.
-#define _tile_top4mxbssps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbssps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
+#define _tile_op4mxbss_ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbssps((dst), (__v64qi)(src1), (__v64qi)(src2), (imm))
+
+/// Read a row from a tile register and convert its int32 elements to FP32.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWD2PS </c> instruction.
+///
+/// \param tsrc
+/// Source tile register ID (0-7).
+/// \param row
+/// Row index selecting the tile row to read.
+/// \returns
+/// ZMM vector (__m512) holding the converted FP32 elements.
+#define _tile_cvtrow_epi32_ps(tsrc, row) _tile_cvtrowd2ps((tsrc), (row))
+
+/// Read a row from a tile register and convert its FP32 elements to BF16,
+/// placing the results in the high half of each destination dword.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2BF16H </c> instruction.
+///
+/// \param tsrc
+/// Source tile register ID (0-7).
+/// \param row
+/// Row index selecting the tile row to read.
+/// \returns
+/// ZMM vector (__m512bh) holding the converted BF16 elements.
+#define _tile_cvtrowh_ps_pbh(tsrc, row) _tile_cvtrowps2bf16h((tsrc), (row))
+
+/// Read a row from a tile register and convert its FP32 elements to BF16,
+/// placing the results in the low half of each destination dword.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2BF16L </c> instruction.
+///
+/// \param tsrc
+/// Source tile register ID (0-7).
+/// \param row
+/// Row index selecting the tile row to read.
+/// \returns
+/// ZMM vector (__m512bh) holding the converted BF16 elements.
+#define _tile_cvtrowl_ps_pbh(tsrc, row) _tile_cvtrowps2bf16l((tsrc), (row))
+
+/// Read a row from a tile register and convert its FP32 elements to FP16,
+/// placing the results in the high half of each destination dword.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2PHH </c> instruction.
+///
+/// \param tsrc
+/// Source tile register ID (0-7).
+/// \param row
+/// Row index selecting the tile row to read.
+/// \returns
+/// ZMM vector (__m512h) holding the converted FP16 elements.
+#define _tile_cvtrowh_ps_ph(tsrc, row) _tile_cvtrowps2phh((tsrc), (row))
+
+/// Read a row from a tile register and convert its FP32 elements to FP16,
+/// placing the results in the low half of each destination dword.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2PHL </c> instruction.
+///
+/// \param tsrc
+/// Source tile register ID (0-7).
+/// \param row
+/// Row index selecting the tile row to read.
+/// \returns
+/// ZMM vector (__m512h) holding the converted FP16 elements.
+#define _tile_cvtrowl_ps_ph(tsrc, row) _tile_cvtrowps2phl((tsrc), (row))
+
+/// Extract one row of a tile register into a ZMM destination.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param tsrc
+/// Source tile register ID (0-7).
+/// \param row
+/// Row index selecting the tile row to read.
+/// \returns
+/// ZMM vector (__m512i) holding the extracted row.
+#define _tile_extractrow(tsrc, row) _tile_movrow((tsrc), (row))
/// ACE tile type with fixed dimensions (16 rows x 64 bytes = 1024 bytes).
/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1.
@@ -381,8 +502,7 @@ typedef struct __attribute__((__packed__, __aligned__(64))) {
/// Use with _tile_ace_loadconfig to configure tiles for ACE operations.
static __inline__ void __DEFAULT_FN_ATTRS_ACE
__ace_init_config(__ace_tile_config *cfg) {
- for (int i = 0; i < 64; i++)
- ((unsigned char *)cfg)[i] = 0;
+ __builtin_memset(cfg, 0, sizeof(*cfg));
cfg->palette_id = 2;
}
@@ -416,8 +536,8 @@ static __inline__ void __tile_ace_zero(__acetile *dst) {
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_top4buud(__acetile *dst, __m512i src1,
__m512i src2) {
- *dst = __builtin_ia32_top4buud_internal(16, 64, 64, *dst, (__v64qs)src1,
- (__v64qs)src2);
+ *dst = __builtin_ia32_top4buud_internal(16, 64, 64, *dst, (__v64qi)src1,
+ (__v64qi)src2);
}
/// Compute 4-way outer product of unsigned x signed bytes to INT32.
@@ -437,8 +557,8 @@ static __inline__ void __tile_ace_top4buud(__acetile *dst, __m512i src1,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_top4busd(__acetile *dst, __m512i src1,
__m512i src2) {
- *dst = __builtin_ia32_top4busd_internal(16, 64, 64, *dst, (__v64qs)src1,
- (__v64qs)src2);
+ *dst = __builtin_ia32_top4busd_internal(16, 64, 64, *dst, (__v64qi)src1,
+ (__v64qi)src2);
}
/// Compute 4-way outer product of signed x signed bytes to INT32.
@@ -458,8 +578,8 @@ static __inline__ void __tile_ace_top4busd(__acetile *dst, __m512i src1,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_top4bssd(__acetile *dst, __m512i src1,
__m512i src2) {
- *dst = __builtin_ia32_top4bssd_internal(16, 64, 64, *dst, (__v64qs)src1,
- (__v64qs)src2);
+ *dst = __builtin_ia32_top4bssd_internal(16, 64, 64, *dst, (__v64qi)src1,
+ (__v64qi)src2);
}
/// Compute 4-way outer product of signed x unsigned bytes to INT32.
@@ -479,8 +599,8 @@ static __inline__ void __tile_ace_top4bssd(__acetile *dst, __m512i src1,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_top4bsud(__acetile *dst, __m512i src1,
__m512i src2) {
- *dst = __builtin_ia32_top4bsud_internal(16, 64, 64, *dst, (__v64qs)src1,
- (__v64qs)src2);
+ *dst = __builtin_ia32_top4bsud_internal(16, 64, 64, *dst, (__v64qi)src1,
+ (__v64qi)src2);
}
/// Compute 2-way outer product of BF16 to FP32.
@@ -504,9 +624,8 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
(__v32bf)src2);
}
-/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
-/// Multiplies HF8 values from src1 with HF8 values from src2, applies
-/// BSR scaling, converts to FP32 and accumulates into the ACE tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E4M3, B = FP8 E4M3. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -515,18 +634,18 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
/// \param dst
/// Pointer to destination/accumulator __acetile.
/// \param src1
-/// First source ZMM vector containing HF8 (E4M3) values.
+/// The A input: ZMM vector of FP8 E4M3 (HF8) values.
/// \param src2
-/// Second source ZMM vector containing HF8 (E4M3) values.
+/// The B input: ZMM vector of FP8 E4M3 (HF8) values.
/// \param imm
-/// 8-bit immediate selecting BSR scale factors to apply.
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
#define __tile_ace_top4mxhf8ps(dst, src1, src2, imm) \
(*(dst) = __builtin_ia32_top4mxhf8ps_internal( \
- 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+ 16, 64, 64, (imm), *(dst), (__v64qi)(src1), (__v64qi)(src2)))
-/// Compute 4-way mixed precision outer product with BF8/HF8 format.
-/// Multiplies BF8 (E5M2) values from src1 with HF8 (E4M3) values from src2,
-/// applies BSR scaling, converts to FP32 and accumulates into the ACE tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E5M2, B = FP8 E4M3. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -535,18 +654,18 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
/// \param dst
/// Pointer to destination/accumulator __acetile.
/// \param src1
-/// First source ZMM vector containing BF8 (E5M2) values.
+/// The A input: ZMM vector of FP8 E5M2 (BF8) values.
/// \param src2
-/// Second source ZMM vector containing HF8 (E4M3) values.
+/// The B input: ZMM vector of FP8 E4M3 (HF8) values.
/// \param imm
-/// 8-bit immediate selecting BSR scale factors to apply.
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
#define __tile_ace_top4mxbhf8ps(dst, src1, src2, imm) \
(*(dst) = __builtin_ia32_top4mxbhf8ps_internal( \
- 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+ 16, 64, 64, (imm), *(dst), (__v64qi)(src1), (__v64qi)(src2)))
-/// Compute 4-way mixed precision outer product with HF8/BF8 format.
-/// Multiplies HF8 (E4M3) values from src1 with BF8 (E5M2) values from src2,
-/// applies BSR scaling, converts to FP32 and accumulates into the ACE tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E4M3, B = FP8 E5M2. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -555,18 +674,18 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
/// \param dst
/// Pointer to destination/accumulator __acetile.
/// \param src1
-/// First source ZMM vector containing HF8 (E4M3) values.
+/// The A input: ZMM vector of FP8 E4M3 (HF8) values.
/// \param src2
-/// Second source ZMM vector containing BF8 (E5M2) values.
+/// The B input: ZMM vector of FP8 E5M2 (BF8) values.
/// \param imm
-/// 8-bit immediate selecting BSR scale factors to apply.
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
#define __tile_ace_top4mxhbf8ps(dst, src1, src2, imm) \
(*(dst) = __builtin_ia32_top4mxhbf8ps_internal( \
- 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+ 16, 64, 64, (imm), *(dst), (__v64qi)(src1), (__v64qi)(src2)))
-/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
-/// Multiplies BF8 values from both sources, applies BSR scaling,
-/// converts to FP32 and accumulates into the ACE tile.
+/// Rank-4 MX FP8 outer product. A = FP8 E5M2, B = FP8 E5M2. OCP MX block
+/// scaling via the BSR, with FP32 accumulation into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -575,18 +694,19 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
/// \param dst
/// Pointer to destination/accumulator __acetile.
/// \param src1
-/// First source ZMM vector containing BF8 (E5M2) values.
+/// The A input: ZMM vector of FP8 E5M2 (BF8) values.
/// \param src2
-/// Second source ZMM vector containing BF8 (E5M2) values.
+/// The B input: ZMM vector of FP8 E5M2 (BF8) values.
/// \param imm
-/// 8-bit immediate selecting BSR scale factors to apply.
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
#define __tile_ace_top4mxbf8ps(dst, src1, src2, imm) \
(*(dst) = __builtin_ia32_top4mxbf8ps_internal( \
- 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+ 16, 64, 64, (imm), *(dst), (__v64qi)(src1), (__v64qi)(src2)))
-/// Compute 4-way mixed precision outer product of MX INT8 with BSR scaling.
-/// Multiplies signed MX INT8 bytes from both sources, applies BSR scaling,
-/// converts to FP32 and accumulates into the ACE tile.
+/// Rank-4 MX INT8 outer product. A = MX INT8 signed, B = MX INT8 signed.
+/// OCP MX block scaling via the BSR, with FP32 accumulation into the
+/// destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -595,14 +715,15 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
/// \param dst
/// Pointer to destination/accumulator __acetile.
/// \param src1
-/// First source ZMM vector containing signed MX INT8 values.
+/// The A input: ZMM vector of MX INT8 signed values.
/// \param src2
-/// Second source ZMM vector containing signed MX INT8 values.
+/// The B input: ZMM vector of MX INT8 signed values.
/// \param imm
-/// 8-bit immediate selecting BSR scale factors to apply.
+/// Scale group selector, formed from \c _MM_ACE_SCALE_A and
+/// \c _MM_ACE_SCALE_B; all other bits are reserved and must be zero.
#define __tile_ace_top4mxbssps(dst, src1, src2, imm) \
(*(dst) = __builtin_ia32_top4mxbssps_internal( \
- 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+ 16, 64, 64, (imm), *(dst), (__v64qi)(src1), (__v64qi)(src2)))
/// Write a ZMM vector as a column in an ACE tile.
/// The 16 doublewords from src are written vertically at column idx.
@@ -620,7 +741,7 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
unsigned int idx) {
- *dst = __builtin_ia32_tilemovcol_set_internal(16, 64, (__v16si)src, idx);
+ *dst = __builtin_ia32_tilemovcolinsert_internal(16, 64, (__v16si)src, idx);
}
/// Write a ZMM vector as a row in an ACE tile.
@@ -639,7 +760,7 @@ static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_setrow(__acetile *dst, __m512i src,
unsigned int idx) {
- *dst = __builtin_ia32_tilemovrow_set_internal(16, 64, (__v16si)src, idx);
+ *dst = __builtin_ia32_tilemovrowinsert_internal(16, 64, (__v16si)src, idx);
}
/// Read a row from an ACE tile to a ZMM vector.
@@ -760,7 +881,9 @@ static __inline__ __m512h __tile_ace_cvtrowps2phl(__acetile *src,
return __builtin_ia32_tcvtrowps2phl_internal(16, 64, *src, idx);
}
+// clang-format on
+
#undef __DEFAULT_FN_ATTRS_ACE
-#endif /* __x86_64__ */
+#endif /* __x86_64__ && __SSE2__ */
#endif /* __ACEV1INTRIN_H */
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index 3b598225240c3a..e7c7d55b2cc4fa 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -478,6 +478,10 @@ bool SemaX86::CheckBuiltinGatherScatterScale(unsigned BuiltinID,
enum { TileRegLow = 0, TileRegHigh = 7 };
+// The ACE scale group selector encodes the A group in bits [1:0] and the B
+// group in bits [4:3]; every other bit is reserved and must be zero.
+enum { ScaleGroupReservedMask = 0xE4 };
+
bool SemaX86::CheckBuiltinTileArgumentsRange(CallExpr *TheCall,
ArrayRef<int> ArgNums) {
for (int ArgNum : ArgNums) {
@@ -488,6 +492,21 @@ bool SemaX86::CheckBuiltinTileArgumentsRange(CallExpr *TheCall,
return false;
}
+bool SemaX86::CheckBuiltinScaleGroup(CallExpr *TheCall, int ArgNum) {
+ Expr *Arg = TheCall->getArg(ArgNum);
+ if (Arg->isTypeDependent() || Arg->isValueDependent())
+ return false;
+
+ llvm::APSInt Result;
+ if (SemaRef.BuiltinConstantArg(TheCall, ArgNum, Result))
+ return true;
+ if (Result.getExtValue() & ScaleGroupReservedMask)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_x86_builtin_invalid_scale_group)
+ << Arg->getSourceRange();
+ return false;
+}
+
bool SemaX86::CheckBuiltinTileDuplicate(CallExpr *TheCall,
ArrayRef<int> ArgNums) {
// Because the max number of tile register is TileRegHigh + 1, so here we use
@@ -535,19 +554,26 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tcvtrowps2phl:
case X86::BI__builtin_ia32_tcvtrowd2ps:
case X86::BI__builtin_ia32_tilemovrow:
- // The ACE outer products name only the accumulator tile; their two sources
- // are ZMM values, so neither the range nor the duplicate rule applies there.
+ // The ACE tile inserts and outer products name only the destination tile;
+ // their remaining sources are ZMM values or a row/column index, so neither
+ // the range nor the duplicate rule applies there.
+ case X86::BI__builtin_ia32_tilemovcolinsert:
+ case X86::BI__builtin_ia32_tilemovrowinsert:
case X86::BI__builtin_ia32_top2bf16ps:
case X86::BI__builtin_ia32_top4buud:
case X86::BI__builtin_ia32_top4busd:
case X86::BI__builtin_ia32_top4bssd:
case X86::BI__builtin_ia32_top4bsud:
+ return CheckBuiltinTileArgumentsRange(TheCall, 0);
+ // The mixed-precision forms take an additional scale group selector, which
+ // only encodes two 2-bit groups; the remaining bits are reserved.
case X86::BI__builtin_ia32_top4mxhf8ps:
case X86::BI__builtin_ia32_top4mxbhf8ps:
case X86::BI__builtin_ia32_top4mxhbf8ps:
case X86::BI__builtin_ia32_top4mxbf8ps:
case X86::BI__builtin_ia32_top4mxbssps:
- return CheckBuiltinTileArgumentsRange(TheCall, 0);
+ return CheckBuiltinTileArgumentsRange(TheCall, 0) ||
+ CheckBuiltinScaleGroup(TheCall, 3);
case X86::BI__builtin_ia32_tdpbssd:
case X86::BI__builtin_ia32_tdpbsud:
case X86::BI__builtin_ia32_tdpbusd:
diff --git a/clang/test/CIR/CodeGen/attr-target-x86.c b/clang/test/CIR/CodeGen/attr-target-x86.c
index e063313dae5a11..2b388fa5adf558 100644
--- a/clang/test/CIR/CodeGen/attr-target-x86.c
+++ b/clang/test/CIR/CodeGen/attr-target-x86.c
@@ -54,11 +54,11 @@ void f_fpmath_387(void) {}
// CIR: cir.func{{.*}} @f_no_sse2()
// CIR-SAME: "cir.target-cpu" = "i686"
-// CIR-SAME: "cir.target-features" = "+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop"
+// CIR-SAME: "cir.target-features" = "+cmov,+cx8,+x87,-acev1,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop"
// CIR-SAME: "cir.tune-cpu" = "i686"
// LLVM-NOT: tune-cpu
-// LLVM: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
+// LLVM: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-acev1,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse2")))
void f_no_sse2(void) {}
@@ -74,10 +74,10 @@ void f_sse4(void) {}
// CIR: cir.func{{.*}} @f_no_sse4()
// CIR-SAME: "cir.target-cpu" = "i686"
-// CIR-SAME: "cir.target-features" = "+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop"
+// CIR-SAME: "cir.target-features" = "+cmov,+cx8,+x87,-acev1,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop"
// CIR-SAME: "cir.tune-cpu" = "i686"
-// LLVM: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
+// LLVM: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-acev1,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx10v2aux,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse4")))
void f_no_sse4(void) {}
diff --git a/clang/test/CodeGen/X86/acev1.c b/clang/test/CodeGen/X86/acev1.c
index 50efae140ab569..72b54afbc63129 100644
--- a/clang/test/CodeGen/X86/acev1.c
+++ b/clang/test/CodeGen/X86/acev1.c
@@ -10,16 +10,31 @@
void test_acev1_outer_products(__m512bh bf1, __m512bh bf2, __m512i i1,
__m512i i2) {
// CHECK-LABEL: @test_acev1_outer_products
- // CHECK: call void @llvm.x86.top2bf16ps(i8 0, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
- // CHECK: call void @llvm.x86.top4buud(i8 1, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
- // CHECK: call void @llvm.x86.top4busd(i8 2, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
- // CHECK: call void @llvm.x86.top4bssd(i8 3, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
- // CHECK: call void @llvm.x86.top4bsud(i8 4, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
- _tile_top2bf16ps(0, bf1, bf2);
- _tile_top4buud(1, i1, i2);
- _tile_top4busd(2, i1, i2);
- _tile_top4bssd(3, i1, i2);
- _tile_top4bsud(4, i1, i2);
+ // CHECK: call void @llvm.x86.acev1.top2bf16ps(i8 0, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
+ // CHECK: call void @llvm.x86.acev1.top4buud(i8 1, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.acev1.top4busd(i8 2, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.acev1.top4bssd(i8 3, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.acev1.top4bsud(i8 4, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ _tile_op2bf16_ps(0, bf1, bf2);
+ _tile_op4buud_epi32(1, i1, i2);
+ _tile_op4busd_epi32(2, i1, i2);
+ _tile_op4bssd_epi32(3, i1, i2);
+ _tile_op4bsud_epi32(4, i1, i2);
+}
+
+// The mixed precision forms take an additional scale group selector.
+void test_acev1_mx_outer_products(__m512i i1, __m512i i2) {
+ // CHECK-LABEL: @test_acev1_mx_outer_products
+ // CHECK: call void @llvm.x86.acev1.top4mxhf8ps(i8 0, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, i8 0)
+ // CHECK: call void @llvm.x86.acev1.top4mxbhf8ps(i8 1, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, i8 1)
+ // CHECK: call void @llvm.x86.acev1.top4mxhbf8ps(i8 2, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, i8 0)
+ // CHECK: call void @llvm.x86.acev1.top4mxbf8ps(i8 3, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, i8 1)
+ // CHECK: call void @llvm.x86.acev1.top4mxbssps(i8 4, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, i8 0)
+ _tile_op4mxhf8_ps(0, i1, i2, 0);
+ _tile_op4mxbhf8_ps(1, i1, i2, 1);
+ _tile_op4mxhbf8_ps(2, i1, i2, 0);
+ _tile_op4mxbf8_ps(3, i1, i2, 1);
+ _tile_op4mxbss_ps(4, i1, i2, 0);
}
void test_acev1_tile_config(void *data) {
@@ -38,14 +53,14 @@ void test_acev1_tile_config(void *data) {
// TILEMOVROW/TILEMOVCOL with an explicit tile register ID.
void test_acev1_tile_movement(__m512i src) {
// CHECK-LABEL: @test_acev1_tile_movement
- // CHECK: call void @llvm.x86.tilemovrow.set(i8 0, <16 x i32> %{{.*}}, i32 5)
- // CHECK: call void @llvm.x86.tilemovcol.set(i8 1, <16 x i32> %{{.*}}, i32 3)
- _tile_setrow(0, src, 5);
- _tile_setcol(1, src, 3);
+ // CHECK: call void @llvm.x86.acev1.tilemovrowinsert(i8 0, <16 x i32> %{{.*}}, i32 5)
+ // CHECK: call void @llvm.x86.acev1.tilemovcolinsert(i8 1, <16 x i32> %{{.*}}, i32 3)
+ _tile_insertrow(0, src, 5);
+ _tile_insertcol(1, src, 3);
}
void test_acev1_bsr(void) {
// CHECK-LABEL: @test_acev1_bsr
- // CHECK: call void @llvm.x86.bsrinit()
+ // CHECK: call void @llvm.x86.acev1.bsr0init()
_bsr0_init();
}
diff --git a/clang/test/CodeGen/X86/acev1_api.c b/clang/test/CodeGen/X86/acev1_api.c
index 55127fc23fa15b..0ffc5a58b22f7d 100644
--- a/clang/test/CodeGen/X86/acev1_api.c
+++ b/clang/test/CodeGen/X86/acev1_api.c
@@ -8,165 +8,164 @@
#include <immintrin.h>
// Test __acetile zero
-void test_tile_ace_zero(void) {
+void test_tile_ace_zero(__acetile *dst) {
// CHECK-LABEL: @test_tile_ace_zero
// CHECK: call x86_amx @llvm.x86.tilezero.internal
// CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __acetile c;
- __tile_ace_zero(&c);
+ __tile_ace_zero(dst);
}
// Test TOP4BUUD ACE API
-void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4buud(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4buud
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64,
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4buud(&dst, src1, src2);
+ __tile_ace_top4buud(dst, src1, src2);
}
// Test TOP4BUSD ACE API
-void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4busd(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4busd
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64,
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4busd.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4busd(&dst, src1, src2);
+ __tile_ace_top4busd(dst, src1, src2);
}
// Test TOP4BSSD ACE API
-void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4bssd(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4bssd
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64,
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4bssd(&dst, src1, src2);
+ __tile_ace_top4bssd(dst, src1, src2);
}
// Test TOP4BSUD ACE API
-void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4bsud(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4bsud
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64,
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4bsud.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4bsud(&dst, src1, src2);
+ __tile_ace_top4bsud(dst, src1, src2);
}
// Test TOP2BF16PS ACE API
-void test_tile_ace_top2bf16ps(__acetile dst, __m512bh src1, __m512bh src2) {
+void test_tile_ace_top2bf16ps(__acetile *dst, __m512bh src1, __m512bh src2) {
// CHECK-LABEL: @test_tile_ace_top2bf16ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64,
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top2bf16ps(&dst, src1, src2);
+ __tile_ace_top2bf16ps(dst, src1, src2);
}
// Test TOP4MXHF8PS ACE API (HF8 x HF8 -> FP32 with BSR scaling)
-void test_tile_ace_top4mxhf8ps(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4mxhf8ps(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4mxhf8ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4mxhf8ps(&dst, src1, src2, 5);
+ __tile_ace_top4mxhf8ps(dst, src1, src2, 5);
}
// Test TOP4MXBHF8PS ACE API (BF8 x HF8 -> FP32 with BSR scaling)
-void test_tile_ace_top4mxbhf8ps(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4mxbhf8ps(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4mxbhf8ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4mxbhf8ps(&dst, src1, src2, 3);
+ __tile_ace_top4mxbhf8ps(dst, src1, src2, 3);
}
// Test TOP4MXHBF8PS ACE API (HF8 x BF8 -> FP32 with BSR scaling)
-void test_tile_ace_top4mxhbf8ps(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4mxhbf8ps(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4mxhbf8ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4mxhbf8ps(&dst, src1, src2, 7);
+ __tile_ace_top4mxhbf8ps(dst, src1, src2, 7);
}
// Test TOP4MXBF8PS ACE API (BF8 x BF8 -> FP32 with BSR scaling)
-void test_tile_ace_top4mxbf8ps(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4mxbf8ps(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4mxbf8ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4mxbf8ps(&dst, src1, src2, 1);
+ __tile_ace_top4mxbf8ps(dst, src1, src2, 1);
}
// Test TOP4MXBSSPS ACE API (MX INT8 SxS -> FP32 with BSR scaling)
-void test_tile_ace_top4mxbssps(__acetile dst, __m512i src1, __m512i src2) {
+void test_tile_ace_top4mxbssps(__acetile *dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4mxbssps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9
+ // CHECK-DAG: call x86_amx @llvm.x86.acev1.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
- __tile_ace_top4mxbssps(&dst, src1, src2, 9);
+ __tile_ace_top4mxbssps(dst, src1, src2, 9);
}
// Test tile setrow (ACE-specific: ZMM to tile row)
-void test_tile_ace_setrow(__acetile dst, __m512i src) {
+void test_tile_ace_setrow(__acetile *dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- __tile_ace_setrow(&dst, src, 0);
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal
+ __tile_ace_setrow(dst, src, 0);
}
// Test tile setcol (ACE-specific: ZMM to tile column)
-void test_tile_ace_setcol(__acetile dst, __m512i src) {
+void test_tile_ace_setcol(__acetile *dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol
- // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
- __tile_ace_setcol(&dst, src, 0);
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovcolinsert.internal
+ __tile_ace_setcol(dst, src, 0);
}
// Test tile getrow (ACE-specific: tile row to ZMM)
-__m512i test_tile_ace_getrow(__acetile src) {
+__m512i test_tile_ace_getrow(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_getrow
// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
- return __tile_ace_getrow(&src, 0);
+ return __tile_ace_getrow(src, 0);
}
// Test tile row read with INT32 to FP32 conversion (ACE-specific)
-__m512 test_tile_ace_cvtrowd2ps(__acetile src) {
+__m512 test_tile_ace_cvtrowd2ps(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowd2ps
// CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
- return __tile_ace_cvtrowd2ps(&src, 2);
+ return __tile_ace_cvtrowd2ps(src, 2);
}
// Test tile row read with FP32 to BF16 conversion, high half of each dword
-__m512bh test_tile_ace_cvtrowps2bf16h(__acetile src) {
+__m512bh test_tile_ace_cvtrowps2bf16h(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2bf16h
// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
- return __tile_ace_cvtrowps2bf16h(&src, 4);
+ return __tile_ace_cvtrowps2bf16h(src, 4);
}
// Test tile row read with FP32 to BF16 conversion, low half of each dword
-__m512bh test_tile_ace_cvtrowps2bf16l(__acetile src) {
+__m512bh test_tile_ace_cvtrowps2bf16l(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2bf16l
// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
- return __tile_ace_cvtrowps2bf16l(&src, 6);
+ return __tile_ace_cvtrowps2bf16l(src, 6);
}
// Test tile row read with FP32 to FP16 conversion, high half of each dword
-__m512h test_tile_ace_cvtrowps2phh(__acetile src) {
+__m512h test_tile_ace_cvtrowps2phh(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2phh
// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
- return __tile_ace_cvtrowps2phh(&src, 8);
+ return __tile_ace_cvtrowps2phh(src, 8);
}
// Test tile row read with FP32 to FP16 conversion, low half of each dword
-__m512h test_tile_ace_cvtrowps2phl(__acetile src) {
+__m512h test_tile_ace_cvtrowps2phl(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2phl
// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
- return __tile_ace_cvtrowps2phl(&src, 10);
+ return __tile_ace_cvtrowps2phl(src, 10);
}
// Integration test: ACE computation workflow
void test_ace_workflow(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_ace_workflow
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.top4buud.internal
__acetile acc;
__tile_ace_zero(&acc);
__tile_ace_top4buud(&acc, src1, src2);
diff --git a/clang/test/CodeGen/X86/acev1_bsr.c b/clang/test/CodeGen/X86/acev1_bsr.c
index be7d1d8dfa4162..e2e4193b9ece18 100644
--- a/clang/test/CodeGen/X86/acev1_bsr.c
+++ b/clang/test/CodeGen/X86/acev1_bsr.c
@@ -6,57 +6,57 @@
#include <immintrin.h>
#include <stdint.h>
-void test_bsr_init(void) {
- // CHECK-LABEL: @test_bsr_init
- // CHECK: call void @llvm.x86.bsrinit()
+void test_bsr0_init(void) {
+ // CHECK-LABEL: @test_bsr0_init
+ // CHECK: call void @llvm.x86.acev1.bsr0init()
_bsr0_init();
}
-void test_bsr_movf(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_bsr_movf
- // CHECK: call void @llvm.x86.bsrmovf(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
- _bsr0_movf(src1, src2);
+void test_bsr0_insertfull(__m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_bsr0_insertfull
+ // CHECK: call void @llvm.x86.acev1.bsr0movf(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ _bsr0_insertfull(src1, src2);
}
-void test_bsr_movh_set(__m512i src) {
- // CHECK-LABEL: @test_bsr_movh_set
- // CHECK: call void @llvm.x86.bsrmovh.set(<64 x i8> %{{.*}})
- _bsr0_movh_set(src);
+void test_bsr0_inserth(__m512i src) {
+ // CHECK-LABEL: @test_bsr0_inserth
+ // CHECK: call void @llvm.x86.acev1.bsr0movhinsert(<64 x i8> %{{.*}})
+ _bsr0_inserth(src);
}
-__m512i test_bsr_movh_get(void) {
- // CHECK-LABEL: @test_bsr_movh_get
- // CHECK: call <64 x i8> @llvm.x86.bsrmovh.get()
- return _bsr0_movh_get();
+__m512i test_bsr0_extracth(void) {
+ // CHECK-LABEL: @test_bsr0_extracth
+ // CHECK: call <64 x i8> @llvm.x86.acev1.bsr0movhextract()
+ return _bsr0_extracth();
}
-void test_bsr_movl_set(__m512i src) {
- // CHECK-LABEL: @test_bsr_movl_set
- // CHECK: call void @llvm.x86.bsrmovl.set(<64 x i8> %{{.*}})
- _bsr0_movl_set(src);
+void test_bsr0_insertl(__m512i src) {
+ // CHECK-LABEL: @test_bsr0_insertl
+ // CHECK: call void @llvm.x86.acev1.bsr0movlinsert(<64 x i8> %{{.*}})
+ _bsr0_insertl(src);
}
-__m512i test_bsr_movl_get(void) {
- // CHECK-LABEL: @test_bsr_movl_get
- // CHECK: call <64 x i8> @llvm.x86.bsrmovl.get()
- return _bsr0_movl_get();
+__m512i test_bsr0_extractl(void) {
+ // CHECK-LABEL: @test_bsr0_extractl
+ // CHECK: call <64 x i8> @llvm.x86.acev1.bsr0movlextract()
+ return _bsr0_extractl();
}
// Test BSR operations sequence
void test_bsr_sequence(void) {
// CHECK-LABEL: @test_bsr_sequence
- // CHECK: call void @llvm.x86.bsrinit()
- // CHECK: call void @llvm.x86.bsrmovf
- // CHECK: call <64 x i8> @llvm.x86.bsrmovh.get()
- // CHECK: call <64 x i8> @llvm.x86.bsrmovl.get()
+ // CHECK: call void @llvm.x86.acev1.bsr0init()
+ // CHECK: call void @llvm.x86.acev1.bsr0movf
+ // CHECK: call <64 x i8> @llvm.x86.acev1.bsr0movhextract()
+ // CHECK: call <64 x i8> @llvm.x86.acev1.bsr0movlextract()
_bsr0_init();
__m512i zmm1 = _mm512_set1_epi32(1);
__m512i zmm2 = _mm512_set1_epi32(2);
- _bsr0_movf(zmm1, zmm2);
+ _bsr0_insertfull(zmm1, zmm2);
- __m512i high = _bsr0_movh_get();
- __m512i low = _bsr0_movl_get();
+ __m512i high = _bsr0_extracth();
+ __m512i low = _bsr0_extractl();
(void)high;
(void)low;
diff --git a/clang/test/CodeGen/X86/acev1_errors.c b/clang/test/CodeGen/X86/acev1_errors.c
index 0776e3c026d3e9..31f4cb083d124b 100644
--- a/clang/test/CodeGen/X86/acev1_errors.c
+++ b/clang/test/CodeGen/X86/acev1_errors.c
@@ -5,13 +5,116 @@
#include <immintrin.h>
+unsigned char TILE_ID = 2;
+unsigned char SCALE_GROUP = 0;
+
void test_tile_range_errors(__m512bh bf, __m512i i) {
// Tile index must be in range [0, 7]
_tile_ace_zero(8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
_tile_ace_zero(16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ // Tile insert index errors
+ _tile_insertcol(8, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_insertrow(16, i, 0); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+
// Outer product tile index errors
- _tile_top4buud(8, i, i); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_top4bssd(16, i, i); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
- _tile_top2bf16ps(9, bf, bf); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
+ _tile_op4buud_epi32(8, i, i); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op4busd_epi32(8, i, i); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op4bssd_epi32(16, i, i); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ _tile_op4bsud_epi32(8, i, i); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op2bf16_ps(9, bf, bf); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
+
+ // Mixed precision outer product tile index errors
+ _tile_op4mxhf8_ps(8, i, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op4mxbhf8_ps(8, i, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op4mxhbf8_ps(8, i, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op4mxbf8_ps(8, i, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_op4mxbss_ps(8, i, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+
+ // Row read tile index errors
+ _tile_cvtrow_epi32_ps(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_cvtrowh_ps_pbh(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_cvtrowl_ps_pbh(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_cvtrowh_ps_ph(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_cvtrowl_ps_ph(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_extractrow(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+}
+
+void test_tile_id_constness(__m512bh bf, __m512i i) {
+ // The tile index must be an integer constant expression.
+ _tile_ace_zero(TILE_ID); // expected-error {{argument to '__builtin_ia32_tilezero' must be a constant integer}}
+
+ _tile_insertcol(TILE_ID, i, 0); // expected-error {{argument to '__builtin_ia32_tilemovcolinsert' must be a constant integer}}
+ _tile_insertrow(TILE_ID, i, 0); // expected-error {{argument to '__builtin_ia32_tilemovrowinsert' must be a constant integer}}
+
+ _tile_op4buud_epi32(TILE_ID, i, i); // expected-error {{argument to '__builtin_ia32_top4buud' must be a constant integer}}
+ _tile_op4busd_epi32(TILE_ID, i, i); // expected-error {{argument to '__builtin_ia32_top4busd' must be a constant integer}}
+ _tile_op4bssd_epi32(TILE_ID, i, i); // expected-error {{argument to '__builtin_ia32_top4bssd' must be a constant integer}}
+ _tile_op4bsud_epi32(TILE_ID, i, i); // expected-error {{argument to '__builtin_ia32_top4bsud' must be a constant integer}}
+ _tile_op2bf16_ps(TILE_ID, bf, bf); // expected-error {{argument to '__builtin_ia32_top2bf16ps' must be a constant integer}}
+
+ _tile_op4mxhf8_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxhf8ps' must be a constant integer}}
+ _tile_op4mxbhf8_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxbhf8ps' must be a constant integer}}
+ _tile_op4mxhbf8_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxhbf8ps' must be a constant integer}}
+ _tile_op4mxbf8_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxbf8ps' must be a constant integer}}
+ _tile_op4mxbss_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxbssps' must be a constant integer}}
+
+ _tile_cvtrow_epi32_ps(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowd2ps' must be a constant integer}}
+ _tile_cvtrowh_ps_pbh(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2bf16h' must be a constant integer}}
+ _tile_cvtrowl_ps_pbh(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2bf16l' must be a constant integer}}
+ _tile_cvtrowh_ps_ph(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2phh' must be a constant integer}}
+ _tile_cvtrowl_ps_ph(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2phl' must be a constant integer}}
+ _tile_extractrow(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tilemovrow' must be a constant integer}}
+}
+
+void test_scale_group_errors(__m512i i) {
+ // The scale group selector encodes the A group in bits [1:0] and the B group
+ // in bits [4:3]; bits [7:6], [5] and [2] are reserved, leaving 16 of the 256
+ // encodings valid.
+ _tile_op4mxhf8_ps(0, i, i, 0x04); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+ _tile_op4mxbhf8_ps(0, i, i, 0x20); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+ _tile_op4mxhbf8_ps(0, i, i, 0x40); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+ _tile_op4mxbf8_ps(0, i, i, 0x80); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+ _tile_op4mxbss_ps(0, i, i, 0xFF); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+
+ // -1 wraps to 0xFF, which has reserved bits set.
+ _tile_op4mxbf8_ps(0, i, i, -1); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+
+ // The tile index and the scale group are both checked.
+ _tile_op4mxbf8_ps(8, i, i, 0x04); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+}
+
+void test_scale_group_constness(__m512i i) {
+ // The scale group selector must also be an integer constant expression.
+ _tile_op4mxhf8_ps(0, i, i, SCALE_GROUP); // expected-error {{argument to '__builtin_ia32_top4mxhf8ps' must be a constant integer}}
+ _tile_op4mxbhf8_ps(0, i, i, SCALE_GROUP); // expected-error {{argument to '__builtin_ia32_top4mxbhf8ps' must be a constant integer}}
+ _tile_op4mxhbf8_ps(0, i, i, SCALE_GROUP); // expected-error {{argument to '__builtin_ia32_top4mxhbf8ps' must be a constant integer}}
+ _tile_op4mxbf8_ps(0, i, i, SCALE_GROUP); // expected-error {{argument to '__builtin_ia32_top4mxbf8ps' must be a constant integer}}
+ _tile_op4mxbss_ps(0, i, i, SCALE_GROUP); // expected-error {{argument to '__builtin_ia32_top4mxbssps' must be a constant integer}}
+}
+
+void test_scale_group_valid(__m512i i) {
+ // All 16 legal encodings: A group in bits [1:0], B group in bits [4:3].
+ _tile_op4mxbf8_ps(0, i, i, 0x00);
+ _tile_op4mxbf8_ps(0, i, i, 0x03);
+ _tile_op4mxbf8_ps(0, i, i, 0x18);
+ _tile_op4mxbf8_ps(0, i, i, 0x1B);
+}
+
+void test_scale_group_helpers(__m512i i) {
+ // The helper macros only ever set the two group fields, so every
+ // combination of in-range group numbers is accepted.
+ _tile_op4mxbf8_ps(0, i, i, _MM_ACE_SCALE_A(0) | _MM_ACE_SCALE_B(0));
+ _tile_op4mxbf8_ps(0, i, i, _MM_ACE_SCALE_A(3) | _MM_ACE_SCALE_B(3));
+ _tile_op4mxbss_ps(0, i, i, _MM_ACE_SCALE_A(1) | _MM_ACE_SCALE_B(2));
+
+ // Either field alone is valid; the other group defaults to 0.
+ _tile_op4mxbf8_ps(0, i, i, _MM_ACE_SCALE_A(2));
+ _tile_op4mxbf8_ps(0, i, i, _MM_ACE_SCALE_B(2));
+
+ // The macros deliberately do not mask their argument, so an out-of-range
+ // group number spills into a reserved bit and is diagnosed rather than
+ // silently truncated.
+ _tile_op4mxbf8_ps(0, i, i, _MM_ACE_SCALE_A(4)); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
+ _tile_op4mxbf8_ps(0, i, i, _MM_ACE_SCALE_B(4)); // expected-error {{scale group argument must have bits 2, 5, 6, and 7 clear}}
}
diff --git a/clang/test/CodeGen/X86/acev1_feature_errors.c b/clang/test/CodeGen/X86/acev1_feature_errors.c
new file mode 100644
index 00000000000000..d3767e0ecf898a
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_feature_errors.c
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +avx10.1 -emit-llvm -o /dev/null -verify=nofeature
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx10.1 -emit-llvm -o /dev/null -verify=feature
+
+// Tests that the ACE v1 intrinsics require the acev1 target feature.
+
+// feature-no-diagnostics
+
+#include <immintrin.h>
+
+void test_needs_acev1(__m512bh bf, __m512i i) {
+ _tile_insertcol(0, i, 0); // nofeature-error {{'__builtin_ia32_tilemovcolinsert' needs target feature acev1}}
+ _tile_insertrow(0, i, 0); // nofeature-error {{'__builtin_ia32_tilemovrowinsert' needs target feature acev1}}
+ _tile_op4buud_epi32(0, i, i); // nofeature-error {{'__builtin_ia32_top4buud' needs target feature acev1}}
+ _tile_op2bf16_ps(0, bf, bf); // nofeature-error {{'__builtin_ia32_top2bf16ps' needs target feature acev1}}
+ _tile_op4mxbf8_ps(0, i, i, 0); // nofeature-error {{'__builtin_ia32_top4mxbf8ps' needs target feature acev1}}
+
+ _tile_ace_zero(0); // nofeature-error {{'__builtin_ia32_tilezero' needs target feature amx-tile|acev1}}
+ _tile_cvtrow_epi32_ps(0, 0); // nofeature-error {{'__builtin_ia32_tcvtrowd2ps' needs target feature amx-avx512,avx10.2|acev1}}
+ _tile_extractrow(0, 0); // nofeature-error {{'__builtin_ia32_tilemovrow' needs target feature amx-avx512,avx10.2|acev1}}
+
+ _tile_ace_loadconfig(&i); // nofeature-error {{always_inline function '_tile_ace_loadconfig' requires target feature 'acev1'}}
+ _tile_ace_storeconfig(&i); // nofeature-error {{always_inline function '_tile_ace_storeconfig' requires target feature 'acev1'}}
+ _tile_ace_release(); // nofeature-error {{always_inline function '_tile_ace_release' requires target feature 'acev1'}}
+ _bsr0_init(); // nofeature-error {{always_inline function '_bsr0_init' requires target feature 'acev1'}}
+ _bsr0_insertfull(i, i); // nofeature-error {{always_inline function '_bsr0_insertfull' requires target feature 'acev1'}}
+ _bsr0_inserth(i); // nofeature-error {{always_inline function '_bsr0_inserth' requires target feature 'acev1'}}
+ _bsr0_insertl(i); // nofeature-error {{always_inline function '_bsr0_insertl' requires target feature 'acev1'}}
+}
diff --git a/clang/test/CodeGen/X86/acev1_inline_asm.c b/clang/test/CodeGen/X86/acev1_inline_asm.c
index 57da62d269b756..7ebed9b590a4ff 100644
--- a/clang/test/CodeGen/X86/acev1_inline_asm.c
+++ b/clang/test/CodeGen/X86/acev1_inline_asm.c
@@ -1,133 +1,79 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
// RUN: -target-feature +avx10.1 -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
-// Tests inline assembly with ACE v1 tile registers
+// Tests ACE tile and BSR register state in inline-asm clobbers and operands.
#include <immintrin.h>
-// TILESTORED is not available in palette 2, so the result leaves the tile one
-// row at a time through TILEMOVROW and is stored from the ZMM.
-void test_acev1_inline_asm_outer_product(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_outer_product
- // CHECK: call void asm sideeffect "tilezero %tmm0
- // CHECK: top4buud %zmm0, %zmm1, %tmm0
- // CHECK: tilemovrow $$0, %tmm0, %zmm2
- // CHECK: vmovups %zmm2, 0(%rdi)
- __asm__ volatile (
- "tilezero %%tmm0 \n\t"
+#define SRCDEST 7
+
+// ACE has no TILESTORED; the result leaves the tile through TILEMOVROW.
+void test_outer_product(void) {
+ // CHECK-LABEL: @test_outer_product
+ // CHECK: call void asm sideeffect "tilezero %tmm0 \0A\09
+ // CHECK-SAME: top4buud %zmm0, %zmm1, %tmm0 \0A\09
+ // CHECK-SAME: tilemovrow $$0, %tmm0, %zmm2 \0A\09
+ // CHECK-SAME: vmovups %zmm2, 0(%rdi) \0A\09",
+ // CHECK-SAME: "~{memory},~{tmm0},~{zmm0},~{zmm1},~{zmm2},~{dirflag},~{fpsr},~{flags}"()
+ __asm__ volatile(
+ "tilezero %%tmm0 \n\t"
"top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
- "tilemovrow $0, %%tmm0, %%zmm2 \n\t"
- "vmovups %%zmm2, 0(%%rdi) \n\t"
+ "tilemovrow $0, %%tmm0, %%zmm2 \n\t"
+ "vmovups %%zmm2, 0(%%rdi) \n\t"
::: "memory", "tmm0", "zmm0", "zmm1", "zmm2"
);
}
-void test_acev1_inline_asm_bf16(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_bf16
- // CHECK: call void asm sideeffect "tilezero %tmm1
- // CHECK: top2bf16ps %zmm2, %zmm3, %tmm1
- __asm__ volatile (
- "tilezero %%tmm1 \n\t"
- "top2bf16ps %%zmm2, %%zmm3, %%tmm1 \n\t"
- ::: "memory", "tmm1", "zmm2", "zmm3"
- );
-}
-
-void test_acev1_inline_asm_bsr(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_bsr
- // CHECK: call void asm sideeffect "bsrinit %bsr0
- // CHECK: bsrmovf %zmm0, %zmm1
- __asm__ volatile (
- "bsrinit %%bsr0 \n\t"
- "bsrmovf %%zmm0, %%zmm1 \n\t"
- ::: "memory", "zmm0", "zmm1"
- );
-}
-
-void test_acev1_inline_asm_tile_config(void *cfg) {
- // CHECK-LABEL: @test_acev1_inline_asm_tile_config
- // CHECK: call void asm sideeffect "ldtilecfg ($0)
- // CHECK: tilerelease
- __asm__ volatile (
- "ldtilecfg (%0) \n\t"
- "tilezero %%tmm0 \n\t"
- "tilezero %%tmm1 \n\t"
- "tilerelease \n\t"
- :: "r"(cfg) : "memory", "tmm0", "tmm1"
- );
+void test_bsr_clobber(void) {
+ // CHECK-LABEL: @test_bsr_clobber
+ // CHECK: call void asm sideeffect "bsrinit %bsr0 \0A\09",
+ // CHECK-SAME: "~{memory},~{bsr0},~{dirflag},~{fpsr},~{flags}"()
+ __asm__ volatile("bsrinit %%bsr0 \n\t" ::: "memory", "bsr0");
}
-// Tile movement is the only way in or out of an ACE tile, since ACE v1 has no
-// TILELOADD/TILESTORED. Immediate index form.
-void test_acev1_inline_asm_tile_movement(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_tile_movement
- // CHECK: call void asm sideeffect "tilemovrow $$3, %zmm0, %tmm2
- // CHECK: tilemovcol $$5, %zmm1, %tmm3
- __asm__ volatile (
- "tilemovrow $3, %%zmm0, %%tmm2 \n\t"
- "tilemovcol $5, %%zmm1, %%tmm3 \n\t"
- ::: "memory", "tmm2", "tmm3", "zmm0", "zmm1"
+void test_tile_and_bsr_clobber(void) {
+ // CHECK-LABEL: @test_tile_and_bsr_clobber
+ // CHECK: call void asm sideeffect "tilezero %tmm1 \0A\09
+ // CHECK-SAME: top4mxhf8ps $$7, %zmm3, %zmm2, %tmm1 \0A\09",
+ // CHECK-SAME: "~{memory},~{tmm1},~{bsr0},~{zmm2},~{zmm3},~{dirflag},~{fpsr},~{flags}"()
+ __asm__ volatile(
+ "tilezero %%tmm1 \n\t"
+ "top4mxhf8ps $7, %%zmm3, %%zmm2, %%tmm1 \n\t"
+ ::: "memory", "tmm1", "bsr0", "zmm2", "zmm3"
);
}
-// Tile movement with the row/column index supplied in a GPR.
-void test_acev1_inline_asm_tile_movement_reg_index(int idx) {
- // CHECK-LABEL: @test_acev1_inline_asm_tile_movement_reg_index
- // CHECK: call void asm sideeffect "tilemovrow $0, %zmm0, %tmm0
- // CHECK: tilemovcol $0, %zmm0, %tmm1
- __asm__ volatile (
+void test_reg_index_operand(int idx) {
+ // CHECK-LABEL: @test_reg_index_operand
+ // CHECK: call void asm sideeffect "tilemovrow $0, %zmm0, %tmm0 \0A\09
+ // CHECK-SAME: tilemovcol $0, %zmm0, %tmm1 \0A\09",
+ // CHECK-SAME: "r,~{memory},~{tmm0},~{tmm1},~{zmm0},~{dirflag},~{fpsr},~{flags}"(i32 %{{.*}})
+ __asm__ volatile(
"tilemovrow %0, %%zmm0, %%tmm0 \n\t"
"tilemovcol %0, %%zmm0, %%tmm1 \n\t"
:: "r"(idx) : "memory", "tmm0", "tmm1", "zmm0"
);
}
-// BSR high/low halves, both set (ZMM to BSR) and get (BSR to ZMM) directions.
-void test_acev1_inline_asm_bsr_movh_movl(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_bsr_movh_movl
- // CHECK: call void asm sideeffect "bsrmovh %zmm1, %bsr0
- // CHECK: bsrmovl %zmm2, %bsr0
- // CHECK: bsrmovh %bsr0, %zmm3
- // CHECK: bsrmovl %bsr0, %zmm4
- __asm__ volatile (
- "bsrmovh %%zmm1, %%bsr0 \n\t"
- "bsrmovl %%zmm2, %%bsr0 \n\t"
- "bsrmovh %%bsr0, %%zmm3 \n\t"
- "bsrmovl %%bsr0, %%zmm4 \n\t"
- ::: "memory", "zmm1", "zmm2", "zmm3", "zmm4"
- );
-}
-
-// MX outer product, which takes the BSR scaling mode as an immediate.
-void test_acev1_inline_asm_mx_outer_product(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_mx_outer_product
- // CHECK: call void asm sideeffect "tilezero %tmm1
- // CHECK: top4mxhf8ps $$7, %zmm3, %zmm2, %tmm1
- __asm__ volatile (
- "tilezero %%tmm1 \n\t"
- "top4mxhf8ps $7, %%zmm3, %%zmm2, %%tmm1 \n\t"
- ::: "memory", "tmm1", "zmm2", "zmm3"
- );
+void test_imm_tile_number(__m512 q) {
+ // CHECK-LABEL: @test_imm_tile_number
+ // CHECK: call void asm sideeffect "tilemovrow %eax, $0, %tmm${1:c}",
+ // CHECK-SAME: "v,i,~{dirflag},~{fpsr},~{flags}"(<16 x float> %{{.*}}, i32 7)
+ __asm__ volatile("tilemovrow %%eax, %0, %%tmm%c1" :: "v"(q), "i"(SRCDEST));
}
-void test_acev1_inline_asm_all_outer_products(void) {
- // CHECK-LABEL: @test_acev1_inline_asm_all_outer_products
- // CHECK: top4buud
- // CHECK: top4busd
- // CHECK: top4bssd
- // CHECK: top4bsud
- // CHECK: top2bf16ps
- __asm__ volatile (
- "tilezero %%tmm0 \n\t"
- "tilezero %%tmm1 \n\t"
- "tilezero %%tmm2 \n\t"
- "tilezero %%tmm3 \n\t"
- "tilezero %%tmm4 \n\t"
- "top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
- "top4busd %%zmm0, %%zmm1, %%tmm1 \n\t"
- "top4bssd %%zmm0, %%zmm1, %%tmm2 \n\t"
- "top4bsud %%zmm0, %%zmm1, %%tmm3 \n\t"
- "top2bf16ps %%zmm0, %%zmm1, %%tmm4 \n\t"
- ::: "memory", "tmm0", "tmm1", "tmm2", "tmm3", "tmm4", "zmm0", "zmm1"
+void test_tile_config(void *cfg) {
+ // CHECK-LABEL: @test_tile_config
+ // CHECK: call void asm sideeffect "ldtilecfg ($0) \0A\09
+ // CHECK-SAME: tilezero %tmm0 \0A\09
+ // CHECK-SAME: tilezero %tmm1 \0A\09
+ // CHECK-SAME: tilerelease \0A\09",
+ // CHECK-SAME: "r,~{memory},~{tmm0},~{tmm1},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}})
+ __asm__ volatile(
+ "ldtilecfg (%0) \n\t"
+ "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm1 \n\t"
+ "tilerelease \n\t"
+ :: "r"(cfg) : "memory", "tmm0", "tmm1"
);
}
diff --git a/clang/test/CodeGen/X86/acev1_mxfp8.c b/clang/test/CodeGen/X86/acev1_mxfp8.c
deleted file mode 100644
index dc0d04cdf76ab8..00000000000000
--- a/clang/test/CodeGen/X86/acev1_mxfp8.c
+++ /dev/null
@@ -1,51 +0,0 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
-
-// Tests ACE v1 MX FP8 outer product macro-based intrinsics
-
-#include <immintrin.h>
-
-void test_top4mxhf8ps(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_top4mxhf8ps
- // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
- _tile_top4mxhf8ps(0, src1, src2, 0);
-}
-
-void test_top4mxbhf8ps(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_top4mxbhf8ps
- // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 1)
- _tile_top4mxbhf8ps(1, src1, src2, 1);
-}
-
-void test_top4mxhbf8ps(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_top4mxhbf8ps
- // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
- _tile_top4mxhbf8ps(2, src1, src2, 0);
-}
-
-void test_top4mxbf8ps(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_top4mxbf8ps
- // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 1)
- _tile_top4mxbf8ps(3, src1, src2, 1);
-}
-
-void test_top4mxbssps(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_top4mxbssps
- // CHECK: call void @llvm.x86.top4mxbssps(i8 4, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
- _tile_top4mxbssps(4, src1, src2, 0);
-}
-
-// Test all MX FP8 variants in sequence
-void test_all_mxfp8_variants(__m512i src1, __m512i src2) {
- // CHECK-LABEL: @test_all_mxfp8_variants
- // CHECK: call void @llvm.x86.top4mxhf8ps
- // CHECK: call void @llvm.x86.top4mxbhf8ps
- // CHECK: call void @llvm.x86.top4mxhbf8ps
- // CHECK: call void @llvm.x86.top4mxbf8ps
- // CHECK: call void @llvm.x86.top4mxbssps
- _tile_top4mxhf8ps(0, src1, src2, 0);
- _tile_top4mxbhf8ps(3, src1, src2, 0);
- _tile_top4mxhbf8ps(6, src1, src2, 0);
- _tile_top4mxbf8ps(1, src1, src2, 1);
- _tile_top4mxbssps(4, src1, src2, 1);
-}
diff --git a/clang/test/CodeGen/X86/acev1_tile_movement.c b/clang/test/CodeGen/X86/acev1_tile_movement.c
index a2648b41b1d382..404e5e3f72f44e 100644
--- a/clang/test/CodeGen/X86/acev1_tile_movement.c
+++ b/clang/test/CodeGen/X86/acev1_tile_movement.c
@@ -9,96 +9,46 @@
#include <immintrin.h>
// Test tile setrow - write ZMM to tile row (ACE-specific)
-void test_tile_ace_setrow(__acetile dst, __m512i src) {
+void test_tile_ace_setrow(__acetile *dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- __tile_ace_setrow(&dst, src, 0);
-}
-
-// Test tile setrow with different row indices
-void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
- // CHECK-LABEL: @test_tile_ace_setrow_indices
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- __tile_ace_setrow(&dst, src, 0);
- __tile_ace_setrow(&dst, src, 8);
- __tile_ace_setrow(&dst, src, 15);
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal
+ __tile_ace_setrow(dst, src, 0);
}
// Test tile setcol - write ZMM to tile column (ACE-specific)
-void test_tile_ace_setcol(__acetile dst, __m512i src) {
+void test_tile_ace_setcol(__acetile *dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol
- // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
- __tile_ace_setcol(&dst, src, 0);
-}
-
-// Test tile setcol with different column indices
-void test_tile_ace_setcol_indices(__acetile dst, __m512i src) {
- // CHECK-LABEL: @test_tile_ace_setcol_indices
- // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
- // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
- __tile_ace_setcol(&dst, src, 0);
- __tile_ace_setcol(&dst, src, 8);
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovcolinsert.internal
+ __tile_ace_setcol(dst, src, 0);
}
// Test tile getrow - read tile row to ZMM (ACE-specific)
-__m512i test_tile_ace_getrow(__acetile src) {
+__m512i test_tile_ace_getrow(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_getrow
// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
- return __tile_ace_getrow(&src, 0);
-}
-
-// Test tile getrow with different row indices
-void test_tile_ace_getrow_indices(__acetile src, __m512i *out) {
- // CHECK-LABEL: @test_tile_ace_getrow_indices
- // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
- // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
- // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
- out[0] = __tile_ace_getrow(&src, 0);
- out[1] = __tile_ace_getrow(&src, 8);
- out[2] = __tile_ace_getrow(&src, 15);
-}
-
-// Integration test: initialize tile using ACE tile zero and setrow
-void test_tile_init_via_setrow(void) {
- // CHECK-LABEL: @test_tile_init_via_setrow
- // CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- __acetile tile;
- __tile_ace_zero(&tile);
-
- __m512i zmm = _mm512_set1_epi32(0x12345678);
- __tile_ace_setrow(&tile, zmm, 0);
+ return __tile_ace_getrow(src, 0);
}
// Test combined tile operations
void test_tile_movement_combined(void) {
// CHECK-LABEL: @test_tile_movement_combined
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovcolinsert.internal
__acetile tile;
__tile_ace_zero(&tile);
- __m512i zmm1 = _mm512_set1_epi32(1);
- __m512i zmm2 = _mm512_set1_epi32(2);
-
- // Set multiple rows
- __tile_ace_setrow(&tile, zmm1, 0);
- __tile_ace_setrow(&tile, zmm2, 1);
-
- // Set columns
- __tile_ace_setcol(&tile, zmm1, 0);
+ __m512i zmm = _mm512_set1_epi32(1);
+ __tile_ace_setrow(&tile, zmm, 0);
+ __tile_ace_setcol(&tile, zmm, 0);
}
// Integration test: complete ACE tile workflow with data movement
void test_ace_data_workflow(__m512i *input, __m512i *output) {
// CHECK-LABEL: @test_ace_data_workflow
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
- // CHECK: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.top4buud.internal
// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
__acetile tile;
__tile_ace_zero(&tile);
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index d0c0fdd1e97105..29bdf72879428c 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -316,6 +316,13 @@
// AMX-AVX512: "-target-feature" "+amx-avx512"
// NO-AMX-AVX512: "-target-feature" "-amx-avx512"
+// RUN: %clang --target=x86_64-unknown-linux-gnu -macev1 %s \
+// RUN: -### -o %t.o 2>&1 | FileCheck -check-prefix=ACEV1 %s
+// RUN: %clang --target=x86_64-unknown-linux-gnu -mno-acev1 %s \
+// RUN: -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-ACEV1 %s
+// ACEV1: "-target-feature" "+acev1"
+// NO-ACEV1: "-target-feature" "-acev1"
+
// RUN: %clang --target=i386 -march=i386 -mhreset %s -### 2>&1 | FileCheck -check-prefix=HRESET %s
// RUN: %clang --target=i386 -march=i386 -mno-hreset %s -### 2>&1 | FileCheck -check-prefix=NO-HRESET %s
// HRESET: "-target-feature" "+hreset"
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index fb7f6d68cc1774..352187bc9aa05e 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -547,32 +547,33 @@
// NO-AMX-AVX512-NOT: #define __AMX_AVX512__ 1
-// ACE v1 requires AVX10.1 as its vector baseline and AMX-TILE for the tile
-// registers and tile management instructions, but nothing above either.
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 -x c \
// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 \
-// RUN: --implicit-check-not=__AVX10_2 --implicit-check-not=__AMX_AVX512__ %s
+// RUN: --implicit-check-not=__AVX10_2 --implicit-check-not=__AMX_AVX512__ \
+// RUN: --implicit-check-not=__AMX_TILE__ %s
// ACEV1-DAG: #define __ACEV1__ 1
// ACEV1-DAG: #define __AVX10_1__ 1
// ACEV1-DAG: #define __AVX512F__ 1
-// ACEV1-DAG: #define __AMX_TILE__ 1
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mno-acev1 -x c \
// RUN: -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
// NO-ACEV1-NOT: #define __ACEV1__ 1
-// AVX10.2 is not required: disabling it keeps ACE v1 and its baseline.
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
// RUN: -mno-avx10.2 -x c -E -dM -o - %s | FileCheck \
// RUN: -check-prefix=ACEV1-BASELINE %s
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-amx-tile -x c -E -dM -o - %s | FileCheck \
+// RUN: -check-prefix=ACEV1-BASELINE %s
// ACEV1-BASELINE-DAG: #define __ACEV1__ 1
// ACEV1-BASELINE-DAG: #define __AVX10_1__ 1
-// ACEV1-BASELINE-DAG: #define __AMX_TILE__ 1
-// Both baselines are necessary: dropping either drops ACE v1 with it.
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
// RUN: -mno-avx10.1 -x c -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
-// RUN: -mno-amx-tile -x c -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+// RUN: -mamx-tile -x c -E -dM -o - %s | FileCheck -check-prefix=ACEV1-AMX %s
+// ACEV1-AMX-DAG: #define __ACEV1__ 1
+// ACEV1-AMX-DAG: #define __AMX_TILE__ 1
// RUN: %clang -target i386-unknown-unknown -march=atom -mavxvnni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVXVNNI %s
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 8276e2ce7ad2c7..ee5dc56aae8810 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5737,196 +5737,132 @@ let TargetPrefix = "x86" in {
}
//===----------------------------------------------------------------------===//
-// ACE (AI Compute Extensions) Intrinsics
-//===----------------------------------------------------------------------===//
+// ACE - AMD AI Compute Extensions
+
let TargetPrefix = "x86" in {
- // BSRINIT - Initialize Block Scale Register
- def int_x86_bsrinit : ClangBuiltin<"__builtin_ia32_bsrinit">,
+ // ACE - block scale register
+ def int_x86_acev1_bsr0init : ClangBuiltin<"__builtin_ia32_bsr0init">,
Intrinsic<[], [], []>;
-
- // BSRMOVF - Move Full to BSR
- // Takes two ZMM registers of E8M0 scale bytes, writes to BSR (implicit)
- def int_x86_bsrmovf : ClangBuiltin<"__builtin_ia32_bsrmovf">,
+ def int_x86_acev1_bsr0movf : ClangBuiltin<"__builtin_ia32_bsr0movf">,
Intrinsic<[], [llvm_v64i8_ty, llvm_v64i8_ty], []>;
-
- // BSRMOVH - Move Half to/from BSR
- // Load variant: read 64 scale bytes from ZMM, update BSR
- def int_x86_bsrmovh_set : ClangBuiltin<"__builtin_ia32_bsrmovh_set">,
+ def int_x86_acev1_bsr0movhinsert : ClangBuiltin<"__builtin_ia32_bsr0movhinsert">,
Intrinsic<[], [llvm_v64i8_ty], []>;
- // Store variant: read from BSR, return 64 scale bytes in a ZMM
- def int_x86_bsrmovh_get : ClangBuiltin<"__builtin_ia32_bsrmovh_get">,
+ def int_x86_acev1_bsr0movhextract : ClangBuiltin<"__builtin_ia32_bsr0movhextract">,
Intrinsic<[llvm_v64i8_ty], [], []>;
-
- // BSRMOVL - Move Low to/from BSR
- // Load variant: read 64 scale bytes from ZMM, write to BSR
- def int_x86_bsrmovl_set : ClangBuiltin<"__builtin_ia32_bsrmovl_set">,
+ def int_x86_acev1_bsr0movlinsert : ClangBuiltin<"__builtin_ia32_bsr0movlinsert">,
Intrinsic<[], [llvm_v64i8_ty], []>;
- // Store variant: read from BSR, return 64 scale bytes in a ZMM
- def int_x86_bsrmovl_get : ClangBuiltin<"__builtin_ia32_bsrmovl_get">,
+ def int_x86_acev1_bsr0movlextract : ClangBuiltin<"__builtin_ia32_bsr0movlextract">,
Intrinsic<[llvm_v64i8_ty], [], []>;
- // TILEMOVCOL - Move Column from Vector to Tile
- // Single intrinsic handles both immediate and register index forms.
- // Lowering checks if index is constant and selects appropriate instruction.
- def int_x86_tilemovcol_set : ClangBuiltin<"__builtin_ia32_tilemovcol_set">,
+ // ACE - tile movement. The index operand may be constant or dynamic;
+ // lowering selects the immediate or register form accordingly.
+ def int_x86_acev1_tilemovcolinsert : ClangBuiltin<"__builtin_ia32_tilemovcolinsert">,
Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
-
- // ACE TILEMOVROW - Move Row from Vector to Tile (extends AMX TILEMOVROW)
- // AMX has int_x86_tilemovrow, which reads from the tile into a vector; ACE
- // adds the reverse direction, writing to the tile from a vector.
- // Single intrinsic handles both immediate and register index forms.
- // Lowering checks if index is constant and selects appropriate instruction.
- def int_x86_tilemovrow_set : ClangBuiltin<"__builtin_ia32_tilemovrow_set">,
+ def int_x86_acev1_tilemovrowinsert : ClangBuiltin<"__builtin_ia32_tilemovrowinsert">,
Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
- // TOP2BF16PS - Outer Product BF16 to Single Precision
- def int_x86_top2bf16ps : ClangBuiltin<"__builtin_ia32_top2bf16ps">,
+ // ACE - outer products
+ def int_x86_acev1_top2bf16ps : ClangBuiltin<"__builtin_ia32_top2bf16ps">,
Intrinsic<[], [llvm_i8_ty, llvm_v32bf16_ty, llvm_v32bf16_ty],
[ImmArg<ArgIndex<0>>]>;
-
- // TOP4 - Tile Outer Product (4x4 block operations)
- // TOP4BUUD - Unsigned/Unsigned
- def int_x86_top4buud : ClangBuiltin<"__builtin_ia32_top4buud">,
+ def int_x86_acev1_top4buud : ClangBuiltin<"__builtin_ia32_top4buud">,
Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
[ImmArg<ArgIndex<0>>]>;
-
- // TOP4BUSD - Unsigned/Signed
- def int_x86_top4busd : ClangBuiltin<"__builtin_ia32_top4busd">,
+ def int_x86_acev1_top4busd : ClangBuiltin<"__builtin_ia32_top4busd">,
Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
[ImmArg<ArgIndex<0>>]>;
-
- // TOP4BSSD - Signed/Signed
- def int_x86_top4bssd : ClangBuiltin<"__builtin_ia32_top4bssd">,
+ def int_x86_acev1_top4bssd : ClangBuiltin<"__builtin_ia32_top4bssd">,
Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
[ImmArg<ArgIndex<0>>]>;
-
- // TOP4BSUD - Signed/Unsigned
- def int_x86_top4bsud : ClangBuiltin<"__builtin_ia32_top4bsud">,
+ def int_x86_acev1_top4bsud : ClangBuiltin<"__builtin_ia32_top4bsud">,
Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
[ImmArg<ArgIndex<0>>]>;
- // TOP4MX - Mixed Precision Outer Products
- // TOP4MXHF8PS - MX FP8 HF8 (E4M3) to FP32
- def int_x86_top4mxhf8ps : ClangBuiltin<"__builtin_ia32_top4mxhf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ // ACE - MX outer products. Operand 3 selects the BSR scale group.
+ def int_x86_acev1_top4mxhf8ps : ClangBuiltin<"__builtin_ia32_top4mxhf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,
llvm_i8_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXBHF8PS - MX FP8 BF8xHF8 to FP32
- def int_x86_top4mxbhf8ps : ClangBuiltin<"__builtin_ia32_top4mxbhf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ def int_x86_acev1_top4mxbhf8ps : ClangBuiltin<"__builtin_ia32_top4mxbhf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,
llvm_i8_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXHBF8PS - MX FP8 HF8xBF8 to FP32
- def int_x86_top4mxhbf8ps : ClangBuiltin<"__builtin_ia32_top4mxhbf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ def int_x86_acev1_top4mxhbf8ps : ClangBuiltin<"__builtin_ia32_top4mxhbf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,
llvm_i8_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXBF8PS - MX FP8 BF8xBF8 to FP32
- def int_x86_top4mxbf8ps : ClangBuiltin<"__builtin_ia32_top4mxbf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ def int_x86_acev1_top4mxbf8ps : ClangBuiltin<"__builtin_ia32_top4mxbf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,
llvm_i8_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXBSSPS - MX INT8 SxS to FP32 (BSR scaled)
- def int_x86_top4mxbssps : ClangBuiltin<"__builtin_ia32_top4mxbssps">,
- Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ def int_x86_acev1_top4mxbssps : ClangBuiltin<"__builtin_ia32_top4mxbssps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty,
llvm_i8_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
- // ACE - internal intrinsics (for IR-level optimization)
- // These take tile/ZMM values directly instead of immediate tile IDs
-
- // TILEMOVCOL internal - takes dimensions and returns tile
- def int_x86_tilemovcol_set_internal :
- ClangBuiltin<"__builtin_ia32_tilemovcol_set_internal">,
+ // ACE - internal intrinsics
+ def int_x86_acev1_tilemovcolinsert_internal :
+ ClangBuiltin<"__builtin_ia32_tilemovcolinsert_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
[]>;
-
- // ACE TILEMOVROW internal - takes dimensions, vector, index and returns tile
- // Note: AMX has int_x86_tilemovrow_internal which extracts from tile to vector
- // ACE adds the reverse: write vector to tile row
- def int_x86_tilemovrow_set_internal :
- ClangBuiltin<"__builtin_ia32_tilemovrow_set_internal">,
+ def int_x86_acev1_tilemovrowinsert_internal :
+ ClangBuiltin<"__builtin_ia32_tilemovrowinsert_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
[]>;
-
- // TOP2BF16PS internal - operates on BF16 data (32 BF16 elements per ZMM)
- def int_x86_top2bf16ps_internal :
+ def int_x86_acev1_top2bf16ps_internal :
ClangBuiltin<"__builtin_ia32_top2bf16ps_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
llvm_x86amx_ty, llvm_v32bf16_ty, llvm_v32bf16_ty], []>;
-
- // TOP4BUUD internal - unsigned bytes (64 elements per ZMM)
- def int_x86_top4buud_internal :
+ def int_x86_acev1_top4buud_internal :
ClangBuiltin<"__builtin_ia32_top4buud_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
-
- // TOP4BUSD internal - unsigned/signed bytes (64 elements per ZMM)
- def int_x86_top4busd_internal :
+ def int_x86_acev1_top4busd_internal :
ClangBuiltin<"__builtin_ia32_top4busd_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
-
- // TOP4BSSD internal - signed bytes (64 elements per ZMM)
- def int_x86_top4bssd_internal :
+ def int_x86_acev1_top4bssd_internal :
ClangBuiltin<"__builtin_ia32_top4bssd_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
-
- // TOP4BSUD internal - signed/unsigned bytes (64 elements per ZMM)
- def int_x86_top4bsud_internal :
+ def int_x86_acev1_top4bsud_internal :
ClangBuiltin<"__builtin_ia32_top4bsud_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
-
- // TOP4MXHFBPS internal - 4th arg is BSR index (i8 immediate)
- // FP8 data packed in dwords (16 dwords per ZMM)
- def int_x86_top4mxhf8ps_internal :
+ def int_x86_acev1_top4mxhf8ps_internal :
ClangBuiltin<"__builtin_ia32_top4mxhf8ps_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
- llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXBHFBPS internal - FP8 data packed in dwords
- def int_x86_top4mxbhf8ps_internal :
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], [ImmArg<ArgIndex<3>>]>;
+ def int_x86_acev1_top4mxbhf8ps_internal :
ClangBuiltin<"__builtin_ia32_top4mxbhf8ps_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
- llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXHBFBPS internal - FP8 data packed in dwords
- def int_x86_top4mxhbf8ps_internal :
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], [ImmArg<ArgIndex<3>>]>;
+ def int_x86_acev1_top4mxhbf8ps_internal :
ClangBuiltin<"__builtin_ia32_top4mxhbf8ps_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
- llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXBF8PS internal - MX FP8 BF8xBF8 data packed in dwords
- def int_x86_top4mxbf8ps_internal :
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], [ImmArg<ArgIndex<3>>]>;
+ def int_x86_acev1_top4mxbf8ps_internal :
ClangBuiltin<"__builtin_ia32_top4mxbf8ps_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
- llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
-
- // TOP4MXBSSPS internal - MX INT8 SxS to FP32 (BSR scaled)
- def int_x86_top4mxbssps_internal :
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], [ImmArg<ArgIndex<3>>]>;
+ def int_x86_acev1_top4mxbssps_internal :
ClangBuiltin<"__builtin_ia32_top4mxbssps_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
- llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], [ImmArg<ArgIndex<3>>]>;
}
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 018c7144c75ffd..2173a2b4b5e23b 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -3975,56 +3975,6 @@ bool X86AsmParser::parseInstruction(ParseInstructionInfo &Info, StringRef Name,
}
}
- // ACE BSR instructions: Strip %bsr0 operands and distinguish load/store.
- // BSR (Block Scale Register) is implicit in the encoding (always BSR0),
- // but the spec requires explicit %bsr0 in assembly syntax for disambiguation.
- //
- // For BSRMOVH and BSRMOVL, load vs store is determined by operand order:
- // - ATT load: zmm/mem, bsr0 (ZMM first, BSR last) - move TO bsr0
- // - ATT store: bsr0, zmm/mem (BSR first, ZMM last) - move FROM bsr0
- // - Intel syntax reverses source/dest from ATT
- //
- // Note: Check for specific ACE BSR instructions to avoid matching standard
- // x86 "bsrl"/"bsrq" (bit scan reverse).
- if (Name == "bsrinit" || Name == "bsrmovf" || Name == "bsrmovh" ||
- Name == "bsrmovl") {
- // Single pass: track positions and filter operands simultaneously
- int FirstBsrPos = -1;
- int FirstNonBsrPos = -1;
- SmallVector<std::unique_ptr<MCParsedAsmOperand>, 4> NewOperands;
- NewOperands.push_back(std::move(Operands[0])); // Keep the mnemonic token
-
- for (size_t I = 1; I < Operands.size(); ++I) {
- X86Operand &Op = static_cast<X86Operand &>(*Operands[I]);
- if (Op.isReg() && Op.getReg() == X86::BSR0) {
- if (FirstBsrPos < 0)
- FirstBsrPos = I;
- // Skip BSR operands (don't add to NewOperands)
- } else {
- if (FirstNonBsrPos < 0)
- FirstNonBsrPos = I;
- NewOperands.push_back(std::move(Operands[I]));
- }
- }
-
- // BSR instructions require explicit %bsr0 operand
- if (FirstBsrPos < 0)
- return Error(NameLoc, "BSR instruction requires explicit %bsr0 operand");
-
- Operands = std::move(NewOperands);
-
- // For BSRMOVH and BSRMOVL, detect set vs get by operand order
- if (Name == "bsrmovh" || Name == "bsrmovl") {
- // Intel: BSR first = set (ZMM->BSR); ATT: non-BSR first = set
- bool IsLoad = isParsingIntelSyntax() ? (FirstBsrPos < FirstNonBsrPos)
- : (FirstNonBsrPos < FirstBsrPos);
- StringRef NewMnemonic =
- IsLoad ? (Name == "bsrmovh" ? "bsrmovh_set" : "bsrmovl_set")
- : (Name == "bsrmovh" ? "bsrmovh_get" : "bsrmovl_get");
- static_cast<X86Operand &>(*Operands[0]).setTokenValue(NewMnemonic);
- }
- }
-
if (Flags)
Operands.push_back(X86Operand::CreatePrefix(Flags, NameLoc, NameLoc));
return false;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
index e310276f4d75f8..564636959f00fe 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
@@ -84,9 +84,8 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, uint64_t Address,
STI.hasFeature(X86::Is16Bit)) {
OS << "\tdata32";
}
- // Try to print any aliases first, then BSR instructions, then compare instrs.
- else if (!printAliasInstr(MI, Address, OS) && !printBSRInstr(MI, OS) &&
- !printVecCompareInstr(MI, OS))
+ // Try to print any aliases first.
+ else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
printInstruction(MI, Address, OS);
// Next always print the annotation.
@@ -403,93 +402,6 @@ bool X86ATTInstPrinter::printVecCompareInstr(const MCInst *MI,
return false;
}
-bool X86ATTInstPrinter::printBSRInstr(const MCInst *MI, raw_ostream &OS) {
- // ACE BSR instructions: Print with explicit %bsr0 operand as per spec.
- // BSR is implicit in the encoding, but the spec requires explicit syntax.
- if (!MI)
- return false;
- unsigned Opcode = MI->getOpcode();
-
- switch (Opcode) {
- default:
- return false;
-
- case X86::BSRINIT:
- OS << "\tbsrinit\t%bsr0";
- return true;
-
- case X86::BSRMOVFrr:
- // Syntax: bsrmovf %zmm2, %zmm1, %bsr0
- OS << "\tbsrmovf\t";
- printOperand(MI, 1, OS); // src2
- OS << ", ";
- printOperand(MI, 0, OS); // src1
- OS << ", %bsr0";
- return true;
-
- case X86::BSRMOVFrm:
- // Syntax: bsrmovf mem, %zmm1, %bsr0
- OS << "\tbsrmovf\t";
- printMemReference(MI, 1, OS); // src2 (memory)
- OS << ", ";
- printOperand(MI, 0, OS); // src1
- OS << ", %bsr0";
- return true;
-
- case X86::BSRMOVHrr_set:
- // Syntax: bsrmovh %zmm1, %bsr0 (set BSR high from zmm)
- OS << "\tbsrmovh\t";
- printOperand(MI, 0, OS); // src
- OS << ", %bsr0";
- return true;
-
- case X86::BSRMOVHrm_set:
- // Syntax: bsrmovh mem, %bsr0 (set BSR high from memory)
- OS << "\tbsrmovh\t";
- printMemReference(MI, 0, OS); // src (memory)
- OS << ", %bsr0";
- return true;
-
- case X86::BSRMOVHrr_get:
- // Syntax: bsrmovh %bsr0, %zmm1 (get BSR high to zmm)
- OS << "\tbsrmovh\t%bsr0, ";
- printOperand(MI, 0, OS); // dst
- return true;
-
- case X86::BSRMOVHmr_get:
- // Syntax: bsrmovh %bsr0, mem
- OS << "\tbsrmovh\t%bsr0, ";
- printMemReference(MI, 0, OS); // dst (memory)
- return true;
-
- case X86::BSRMOVLrr_set:
- // Syntax: bsrmovl %zmm2, %bsr0 (set BSR low from zmm)
- OS << "\tbsrmovl\t";
- printOperand(MI, 0, OS); // src
- OS << ", %bsr0";
- return true;
-
- case X86::BSRMOVLrm_set:
- // Syntax: bsrmovl mem, %bsr0
- OS << "\tbsrmovl\t";
- printMemReference(MI, 0, OS); // src (memory)
- OS << ", %bsr0";
- return true;
-
- case X86::BSRMOVLrr_get:
- // Syntax: bsrmovl %bsr0, %zmm3 (get BSR low to zmm)
- OS << "\tbsrmovl\t%bsr0, ";
- printOperand(MI, 0, OS); // dst
- return true;
-
- case X86::BSRMOVLmr_get:
- // Syntax: bsrmovl %bsr0, mem
- OS << "\tbsrmovl\t%bsr0, ";
- printMemReference(MI, 0, OS); // dst (memory)
- return true;
- }
-}
-
void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
index f28024a50a78c0..1452622ebcea84 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
@@ -28,7 +28,6 @@ class X86ATTInstPrinter final : public X86InstPrinterCommon {
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
- bool printBSRInstr(const MCInst *MI, raw_ostream &OS);
// Autogenerated by tblgen, returns true if we successfully printed an
// alias.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
index def7784ea7b5f8..ff27005f52ea8f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
@@ -46,8 +46,7 @@ void X86IntelInstPrinter::printInst(const MCInst *MI, uint64_t Address,
if (MI->getOpcode() == X86::DATA16_PREFIX &&
STI.hasFeature(X86::Is16Bit)) {
OS << "\tdata32";
- } else if (!printAliasInstr(MI, Address, OS) && !printBSRInstr(MI, OS) &&
- !printVecCompareInstr(MI, OS))
+ } else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
printInstruction(MI, Address, OS);
// Next always print the annotation.
@@ -364,91 +363,6 @@ bool X86IntelInstPrinter::printVecCompareInstr(const MCInst *MI, raw_ostream &OS
return false;
}
-bool X86IntelInstPrinter::printBSRInstr(const MCInst *MI, raw_ostream &OS) {
- // ACE BSR instructions: Print with explicit bsr0 operand as per spec.
- // BSR is implicit in the encoding, but the spec requires explicit syntax.
- if (!MI)
- return false;
- unsigned Opcode = MI->getOpcode();
-
- switch (Opcode) {
- default:
- return false;
-
- case X86::BSRINIT:
- OS << "\tbsrinit\tbsr0";
- return true;
-
- case X86::BSRMOVFrr:
- // Intel Syntax: bsrmovf bsr0, zmm1, zmm2
- OS << "\tbsrmovf\tbsr0, ";
- printOperand(MI, 0, OS); // src1
- OS << ", ";
- printOperand(MI, 1, OS); // src2
- return true;
-
- case X86::BSRMOVFrm:
- // Intel Syntax: bsrmovf bsr0, zmm1, zmmword ptr [mem]
- OS << "\tbsrmovf\tbsr0, ";
- printOperand(MI, 0, OS); // src1
- OS << ", ";
- printzmmwordmem(MI, 1, OS); // src2 (memory)
- return true;
-
- case X86::BSRMOVHrr_set:
- // Intel Syntax: bsrmovh bsr0, zmm1 (set BSR high from zmm)
- OS << "\tbsrmovh\tbsr0, ";
- printOperand(MI, 0, OS); // src
- return true;
-
- case X86::BSRMOVHrm_set:
- // Intel Syntax: bsrmovh bsr0, zmmword ptr [mem] (set BSR high from memory)
- OS << "\tbsrmovh\tbsr0, ";
- printzmmwordmem(MI, 0, OS); // src (memory)
- return true;
-
- case X86::BSRMOVHrr_get:
- // Intel Syntax: bsrmovh zmm1, bsr0 (get BSR high to zmm)
- OS << "\tbsrmovh\t";
- printOperand(MI, 0, OS); // dst
- OS << ", bsr0";
- return true;
-
- case X86::BSRMOVHmr_get:
- // Intel Syntax: bsrmovh zmmword ptr [mem], bsr0
- OS << "\tbsrmovh\t";
- printzmmwordmem(MI, 0, OS); // dst (memory)
- OS << ", bsr0";
- return true;
-
- case X86::BSRMOVLrr_set:
- // Intel Syntax: bsrmovl bsr0, zmm2 (set BSR low from zmm)
- OS << "\tbsrmovl\tbsr0, ";
- printOperand(MI, 0, OS); // src
- return true;
-
- case X86::BSRMOVLrm_set:
- // Intel Syntax: bsrmovl bsr0, zmmword ptr [mem]
- OS << "\tbsrmovl\tbsr0, ";
- printzmmwordmem(MI, 0, OS); // src (memory)
- return true;
-
- case X86::BSRMOVLrr_get:
- // Intel Syntax: bsrmovl zmm3, bsr0 (get BSR low to zmm)
- OS << "\tbsrmovl\t";
- printOperand(MI, 0, OS); // dst
- OS << ", bsr0";
- return true;
-
- case X86::BSRMOVLmr_get:
- // Intel Syntax: bsrmovl zmmword ptr [mem], bsr0
- OS << "\tbsrmovl\t";
- printzmmwordmem(MI, 0, OS); // dst (memory)
- OS << ", bsr0";
- return true;
- }
-}
-
void X86IntelInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
index 85830cc0c02cb0..324c56c1329da1 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
@@ -28,7 +28,6 @@ class X86IntelInstPrinter final : public X86InstPrinterCommon {
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
- bool printBSRInstr(const MCInst *MI, raw_ostream &OS);
// Autogenerated by tblgen, returns true if we successfully printed an
// alias.
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 4f1d0d76c8423c..7d15c41cc69a3d 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -363,16 +363,12 @@ def FeatureAVX10_2_512 : SubtargetFeature<"avx10.2-512", "HasAVX10_2_512", "true
def FeatureAVX10_V2_AUX : SubtargetFeature<"avx10v2aux", "HasAVX10_V2_AUX", "true",
"Support AVX10 V2 AUX instructions",
[FeatureAVX10_1]>;
-// Defined after AVX10.1 since ACE v1 builds on it as its vector baseline.
-// ACE v1 reuses the AMX tile architecture: per spec section 15.5.5 an
-// implementation that supports only ACE still reports AMX-TILE, which is what
-// enumerates the tile registers and LDTILECFG/STTILECFG/TILEZERO/TILERELEASE.
-// The AMX compute features and palette 1 are not required, so AMX-AVX512 is not
-// implied; TILEMOVROW and the TCVTROW* family are enumerated under
-// "AMX-AVX512 || ACE_VSN >= 1".
+// AMX-TILE is not implied. ACE configures tile palette 2, which supports only
+// LDTILECFG, STTILECFG, TILEZERO and TILERELEASE; those are predicated on
+// HasAMXTILE_Or_ACEV1. A target with both palettes needs -mamx-tile as well.
def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
"Support ACEV1 (AI Compute Extensions v1) instructions",
- [FeatureAMXTILE, FeatureAVX10_1]>;
+ [FeatureAVX10_1]>;
def FeatureEGPR : SubtargetFeature<"egpr", "HasEGPR", "true",
"Support extended general purpose register">;
def FeaturePush2Pop2 : SubtargetFeature<"push2pop2", "HasPush2Pop2", "true",
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 478f750e662ace..ddb40dc2d07c32 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -714,32 +714,30 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
MI.tieOperands(0, 1);
return true;
}
- // ACE internal pseudo instructions
- // Pattern: Remove dimension operands (1-3), change opcode, re-tie operands
- case X86::PTOP2BF16PSV:
- case X86::PTOP4BUUDV:
- case X86::PTOP4BUSDV:
- case X86::PTOP4BSSDV:
- case X86::PTOP4BSUDV: {
+ case X86::PTOP2BF16PStrrV:
+ case X86::PTOP4BUUDtrrV:
+ case X86::PTOP4BUSDtrrV:
+ case X86::PTOP4BSSDtrrV:
+ case X86::PTOP4BSUDtrrV: {
MI.untieRegOperand(4);
for (unsigned i = 3; i > 0; --i)
MI.removeOperand(i);
unsigned Opc = 0;
switch (Opcode) {
- case X86::PTOP2BF16PSV:
- Opc = X86::TOP2BF16PSrrr;
+ case X86::PTOP2BF16PStrrV:
+ Opc = X86::TOP2BF16PStrr;
break;
- case X86::PTOP4BUUDV:
- Opc = X86::TOP4BUUDrrr;
+ case X86::PTOP4BUUDtrrV:
+ Opc = X86::TOP4BUUDtrr;
break;
- case X86::PTOP4BUSDV:
- Opc = X86::TOP4BUSDrrr;
+ case X86::PTOP4BUSDtrrV:
+ Opc = X86::TOP4BUSDtrr;
break;
- case X86::PTOP4BSSDV:
- Opc = X86::TOP4BSSDrrr;
+ case X86::PTOP4BSSDtrrV:
+ Opc = X86::TOP4BSSDtrr;
break;
- case X86::PTOP4BSUDV:
- Opc = X86::TOP4BSUDrrr;
+ case X86::PTOP4BSUDtrrV:
+ Opc = X86::TOP4BSUDtrr;
break;
default:
llvm_unreachable("Unexpected ACE opcode");
@@ -748,38 +746,32 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
MI.tieOperands(0, 1);
return true;
}
- // ACE TOP4MX internal pseudo instructions - have 4 dimension operands (row,
- // col, k, imm)
- case X86::PTOP4MXHF8PSV:
- case X86::PTOP4MXBHF8PSV:
- case X86::PTOP4MXHBF8PSV:
- case X86::PTOP4MXBF8PSV:
- case X86::PTOP4MXBSSPSV: {
+ case X86::PTOP4MXHF8PStrriV:
+ case X86::PTOP4MXBHF8PStrriV:
+ case X86::PTOP4MXHBF8PStrriV:
+ case X86::PTOP4MXBF8PStrriV:
+ case X86::PTOP4MXBSSPStrriV: {
MI.untieRegOperand(5);
- // Remove dimension operands 1-3, keep imm at position 4
for (unsigned i = 3; i > 0; --i)
MI.removeOperand(i);
- // Now operand layout is: dst, imm, src1(acc), src2, src3
- // Real instruction expects: dst, src1(acc), src2, src3, imm
- // Need to move imm from position 1 to end
unsigned ImmVal = MI.getOperand(1).getImm();
MI.removeOperand(1);
unsigned Opc = 0;
switch (Opcode) {
- case X86::PTOP4MXHF8PSV:
- Opc = X86::TOP4MXHF8PSrrri;
+ case X86::PTOP4MXHF8PStrriV:
+ Opc = X86::TOP4MXHF8PStrri;
break;
- case X86::PTOP4MXBHF8PSV:
- Opc = X86::TOP4MXBHF8PSrrri;
+ case X86::PTOP4MXBHF8PStrriV:
+ Opc = X86::TOP4MXBHF8PStrri;
break;
- case X86::PTOP4MXHBF8PSV:
- Opc = X86::TOP4MXHBF8PSrrri;
+ case X86::PTOP4MXHBF8PStrriV:
+ Opc = X86::TOP4MXHBF8PStrri;
break;
- case X86::PTOP4MXBF8PSV:
- Opc = X86::TOP4MXBF8PSrrri;
+ case X86::PTOP4MXBF8PStrriV:
+ Opc = X86::TOP4MXBF8PStrri;
break;
- case X86::PTOP4MXBSSPSV:
- Opc = X86::TOP4MXBSSPSrrri;
+ case X86::PTOP4MXBSSPStrriV:
+ Opc = X86::TOP4MXBSSPStrri;
break;
default:
llvm_unreachable("Unexpected ACE TOP4MX opcode");
@@ -789,17 +781,12 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
MI.tieOperands(0, 1);
return true;
}
- // ACE TILEMOV internal pseudo instructions - output TILE, remove dimension
- // operands
- case X86::PTILEMOVCOLV:
- case X86::PTILEMOVROWV: {
- // Operand layout: dst, row, col, src(VR512), idx(GR32)
- // Remove row, col (operands 1, 2)
+ case X86::PTILEMOVCOLtreV:
+ case X86::PTILEMOVROWtreV: {
for (unsigned i = 2; i > 0; --i)
MI.removeOperand(i);
- // Now layout: dst, src, idx - matches real instruction
- unsigned Opc =
- (Opcode == X86::PTILEMOVCOLV) ? X86::TILEMOVCOLrr : X86::TILEMOVROWrr;
+ unsigned Opc = (Opcode == X86::PTILEMOVCOLtreV) ? X86::TILEMOVCOLtre
+ : X86::TILEMOVROWtre;
MI.setDesc(TII->get(Opc));
return true;
}
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 41c10a84b8badd..80cadba2e51c04 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -268,7 +268,6 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
int FI = getStackSpaceFor(OrigReg);
const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg);
Register TileReg;
-
// Fold copy to tileload
// BB1:
// spill src to s
@@ -299,7 +298,7 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
addFrameReference(MIB, FI, Row * RowSize);
// tilemovrow $row, %zmm, %tmm
- BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWri),
+ BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWtri),
TileReg)
.addReg(ScratchZMM)
.addImm(Row);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 3b5bb70e3db02b..d9824f4142d22a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28777,8 +28777,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
// ACE Tile Movement Intrinsics - handle both immediate and register index
// forms using a single intrinsic. Check if index is constant to select
// the appropriate instruction form.
- case Intrinsic::x86_tilemovrow_set:
- case Intrinsic::x86_tilemovcol_set: {
+ case Intrinsic::x86_acev1_tilemovrowinsert:
+ case Intrinsic::x86_acev1_tilemovcolinsert: {
SDLoc DL(Op);
SDValue Chain = Op.getOperand(0);
unsigned TileID = Op.getConstantOperandVal(2);
@@ -28786,12 +28786,12 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDValue Idx = Op.getOperand(4);
unsigned PseudoImm, PseudoReg;
- if (IntNo == Intrinsic::x86_tilemovrow_set) {
- PseudoImm = X86::PTILEMOVROW;
- PseudoReg = X86::PTILEMOVROW_REG;
+ if (IntNo == Intrinsic::x86_acev1_tilemovrowinsert) {
+ PseudoImm = X86::PTILEMOVROWtri;
+ PseudoReg = X86::PTILEMOVROWtre;
} else {
- PseudoImm = X86::PTILEMOVCOL;
- PseudoReg = X86::PTILEMOVCOL_REG;
+ PseudoImm = X86::PTILEMOVCOLtri;
+ PseudoReg = X86::PTILEMOVCOLtre;
}
MachineSDNode *Node;
@@ -39163,34 +39163,34 @@ static const TileOpcodeModelEntry TileOpcodeModels[] = {
{X86::PTILEZEROV, TileProgModelType::AMX_ManagedRA},
// ACE-only DirectReg (macro API)
- {X86::PTILEMOVCOL, TileProgModelType::ACE_DirectReg},
- {X86::PTILEMOVCOL_REG, TileProgModelType::ACE_DirectReg},
- {X86::PTILEMOVROW, TileProgModelType::ACE_DirectReg},
- {X86::PTILEMOVROW_REG, TileProgModelType::ACE_DirectReg},
- {X86::PTOP2BF16PS, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4BUUD, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4BUSD, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4BSSD, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4BSUD, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4MXHF8PS, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4MXBHF8PS, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4MXHBF8PS, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4MXBF8PS, TileProgModelType::ACE_DirectReg},
- {X86::PTOP4MXBSSPS, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVCOLtri, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVCOLtre, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVROWtri, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVROWtre, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP2BF16PStrr, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BUUDtrr, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BUSDtrr, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BSSDtrr, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BSUDtrr, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXHF8PStrri, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBHF8PStrri, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXHBF8PStrri, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBF8PStrri, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBSSPStrri, TileProgModelType::ACE_DirectReg},
// ACE-only ManagedRA (struct API)
- {X86::PTILEMOVCOLV, TileProgModelType::ACE_ManagedRA},
- {X86::PTILEMOVROWV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP2BF16PSV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4BUUDV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4BUSDV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4BSSDV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4BSUDV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4MXHF8PSV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4MXBHF8PSV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4MXHBF8PSV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4MXBF8PSV, TileProgModelType::ACE_ManagedRA},
- {X86::PTOP4MXBSSPSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTILEMOVCOLtreV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTILEMOVROWtreV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP2BF16PStrrV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BUUDtrrV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BUSDtrrV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BSSDtrrV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BSUDtrrV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXHF8PStrriV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBHF8PStrriV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXHBF8PStrriV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBF8PStrriV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBSSPStrriV, TileProgModelType::ACE_ManagedRA},
};
// Set tile program model based on opcode. Returns true if model was set.
@@ -39663,18 +39663,18 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
return BB;
}
case X86::PTILEZEROV:
- case X86::PTILEMOVCOLV:
- case X86::PTILEMOVROWV:
- case X86::PTOP2BF16PSV:
- case X86::PTOP4BUUDV:
- case X86::PTOP4BUSDV:
- case X86::PTOP4BSSDV:
- case X86::PTOP4BSUDV:
- case X86::PTOP4MXHF8PSV:
- case X86::PTOP4MXBHF8PSV:
- case X86::PTOP4MXHBF8PSV:
- case X86::PTOP4MXBF8PSV:
- case X86::PTOP4MXBSSPSV:
+ case X86::PTILEMOVCOLtreV:
+ case X86::PTILEMOVROWtreV:
+ case X86::PTOP2BF16PStrrV:
+ case X86::PTOP4BUUDtrrV:
+ case X86::PTOP4BUSDtrrV:
+ case X86::PTOP4BSSDtrrV:
+ case X86::PTOP4BSUDtrrV:
+ case X86::PTOP4MXHF8PStrriV:
+ case X86::PTOP4MXBHF8PStrriV:
+ case X86::PTOP4MXHBF8PStrriV:
+ case X86::PTOP4MXBF8PStrriV:
+ case X86::PTOP4MXBSSPStrriV:
return BB;
case X86::PTILELOADDRS:
case X86::PTILELOADDRST1:
@@ -39806,67 +39806,67 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
}
// Note: BSRMOVF, BSRMOVH, BSRMOVL are handled directly by patterns on real
// instructions
- case X86::PTILEMOVCOL: {
+ case X86::PTILEMOVCOLtri: {
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
Register SrcReg = MI.getOperand(1).getReg();
unsigned Idx = MI.getOperand(2).getImm();
- BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLri), DstReg)
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLtri), DstReg)
.addReg(SrcReg)
.addImm(Idx);
MI.eraseFromParent();
return BB;
}
- case X86::PTILEMOVCOL_REG: {
+ case X86::PTILEMOVCOLtre: {
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
Register SrcReg = MI.getOperand(1).getReg();
- BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLrr), DstReg)
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLtre), DstReg)
.addReg(SrcReg)
.addReg(MI.getOperand(2).getReg());
MI.eraseFromParent();
return BB;
}
- case X86::PTILEMOVROW: {
+ case X86::PTILEMOVROWtri: {
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
Register SrcReg = MI.getOperand(1).getReg();
unsigned Idx = MI.getOperand(2).getImm();
- BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWri), DstReg)
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWtri), DstReg)
.addReg(SrcReg)
.addImm(Idx);
MI.eraseFromParent();
return BB;
}
- case X86::PTILEMOVROW_REG: {
+ case X86::PTILEMOVROWtre: {
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
Register SrcReg = MI.getOperand(1).getReg();
- BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWrr), DstReg)
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWtre), DstReg)
.addReg(SrcReg)
.addReg(MI.getOperand(2).getReg());
MI.eraseFromParent();
return BB;
}
- case X86::PTOP2BF16PS:
- case X86::PTOP4BUUD:
- case X86::PTOP4BUSD:
- case X86::PTOP4BSSD:
- case X86::PTOP4BSUD: {
+ case X86::PTOP2BF16PStrr:
+ case X86::PTOP4BUUDtrr:
+ case X86::PTOP4BUSDtrr:
+ case X86::PTOP4BSSDtrr:
+ case X86::PTOP4BSUDtrr: {
unsigned Opc;
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected opcode!");
- case X86::PTOP2BF16PS:
- Opc = X86::TOP2BF16PSrrr;
+ case X86::PTOP2BF16PStrr:
+ Opc = X86::TOP2BF16PStrr;
break;
- case X86::PTOP4BUUD:
- Opc = X86::TOP4BUUDrrr;
+ case X86::PTOP4BUUDtrr:
+ Opc = X86::TOP4BUUDtrr;
break;
- case X86::PTOP4BUSD:
- Opc = X86::TOP4BUSDrrr;
+ case X86::PTOP4BUSDtrr:
+ Opc = X86::TOP4BUSDtrr;
break;
- case X86::PTOP4BSSD:
- Opc = X86::TOP4BSSDrrr;
+ case X86::PTOP4BSSDtrr:
+ Opc = X86::TOP4BSSDtrr;
break;
- case X86::PTOP4BSUD:
- Opc = X86::TOP4BSUDrrr;
+ case X86::PTOP4BSUDtrr:
+ Opc = X86::TOP4BSUDtrr;
break;
}
// These instructions: TILE dst (rw), ZMM src1, ZMM src2
@@ -39879,29 +39879,29 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MI.eraseFromParent();
return BB;
}
- case X86::PTOP4MXHF8PS:
- case X86::PTOP4MXBHF8PS:
- case X86::PTOP4MXHBF8PS:
- case X86::PTOP4MXBF8PS:
- case X86::PTOP4MXBSSPS: {
+ case X86::PTOP4MXHF8PStrri:
+ case X86::PTOP4MXBHF8PStrri:
+ case X86::PTOP4MXHBF8PStrri:
+ case X86::PTOP4MXBF8PStrri:
+ case X86::PTOP4MXBSSPStrri: {
unsigned Opc;
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected opcode!");
- case X86::PTOP4MXHF8PS:
- Opc = X86::TOP4MXHF8PSrrri;
+ case X86::PTOP4MXHF8PStrri:
+ Opc = X86::TOP4MXHF8PStrri;
break;
- case X86::PTOP4MXBHF8PS:
- Opc = X86::TOP4MXBHF8PSrrri;
+ case X86::PTOP4MXBHF8PStrri:
+ Opc = X86::TOP4MXBHF8PStrri;
break;
- case X86::PTOP4MXHBF8PS:
- Opc = X86::TOP4MXHBF8PSrrri;
+ case X86::PTOP4MXHBF8PStrri:
+ Opc = X86::TOP4MXHBF8PStrri;
break;
- case X86::PTOP4MXBF8PS:
- Opc = X86::TOP4MXBF8PSrrri;
+ case X86::PTOP4MXBF8PStrri:
+ Opc = X86::TOP4MXBF8PStrri;
break;
- case X86::PTOP4MXBSSPS:
- Opc = X86::TOP4MXBSSPSrrri;
+ case X86::PTOP4MXBSSPStrri:
+ Opc = X86::TOP4MXBSSPStrri;
break;
}
// These instructions: TILE dst (rw), ZMM src1, ZMM src2, imm8
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
index 26f531a19ff0be..d64d7a053c2c66 100644
--- a/llvm/lib/Target/X86/X86InstrACE.td
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -17,237 +17,175 @@
let Predicates = [HasACEV1, In64BitMode] in {
// BSRINIT - Initialize Block Scale Register
-// VEX.128.F2.0F38.W1 49 11:000:000 - BSRINIT bsr0
-// BSR is implicit destination, ModRM = 11:000:000 (mod=11, reg=000, r/m=000)
-// BSR0 is implicit (via Defs) rather than an operand, so the asm syntax takes
-// no explicit operand.
let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
def BSRINIT : I<0x49, MRM_C0, (outs), (ins),
- "bsrinit", []>,
- VEX, XD, T8, REX_W;
+ "bsrinit\t{%bsr0|bsr0}", []>,
+ VEX, XD, T8, REX_W;
}
// BSRMOVF - Move Full to BSR
-// EVEX.512.NP.MAP6.W1 95 11:000:bbb - BSRMOVF bsr0, zmm1, zmm2/m512
-// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1 and ModRM.r/m
-// encodes zmm2
-// BSR is implicit destination (via Defs)
-let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
+let SchedRW = [WriteSystem], Defs = [BSR0] in {
def BSRMOVFrr : I<0x95, MRM0r, (outs),
(ins VR512:$src1, VR512:$src2),
- "bsrmovf\t{$src2, $src1|$src1, $src2}",
- [(int_x86_bsrmovf (v64i8 VR512:$src1), (v64i8 VR512:$src2))]>,
- EVEX, T_MAP6, REX_W, EVEX_V512, VVVV;
+ "bsrmovf\t{$src2, $src1, %bsr0|bsr0, $src1, $src2}",
+ [(int_x86_acev1_bsr0movf (v64i8 VR512:$src1), (v64i8 VR512:$src2))]>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, VVVV;
let mayLoad = 1 in
def BSRMOVFrm : I<0x95, MRM0m, (outs),
(ins VR512:$src1, f512mem:$src2),
- "bsrmovf\t{$src2, $src1|$src1, $src2}", []>,
- EVEX, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>, VVVV;
+ "bsrmovf\t{$src2, $src1, %bsr0|bsr0, $src1, $src2}", []>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>, VVVV;
}
// BSRMOVH - Move Half to/from BSR
-// EVEX.512.F2.MAP6.W1 95 /r - BSRMOVH bsr0, zmm1/m512, bsr0 (load, W1)
-// EVEX.512.F2.MAP6.W0 95 /r - BSRMOVH zmm1/m512, bsr0 (store, W0)
-// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
-// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
-// AsmString uses a "_set"/"_get" suffix for asm matching; the printer emits the
-// real mnemonic.
-let SchedRW = [WriteVecLogic] in {
- // Set BSR high half (W1): BSR is both input and output (read-modify-write)
+let SchedRW = [WriteSystem] in {
let Defs = [BSR0], Uses = [BSR0] in {
def BSRMOVHrr_set : I<0x95, MRM0r, (outs),
(ins VR512:$src),
- "bsrmovh_set\t$src",
- [(int_x86_bsrmovh_set (v64i8 VR512:$src))]>,
- EVEX, XD, T_MAP6, REX_W, EVEX_V512;
+ "bsrmovh\t{$src, %bsr0|bsr0, $src}",
+ [(int_x86_acev1_bsr0movhinsert (v64i8 VR512:$src))]>,
+ EVEX, XD, T_MAP6, REX_W, EVEX_V512;
let mayLoad = 1 in
def BSRMOVHrm_set : I<0x95, MRM0m, (outs),
(ins f512mem:$src),
- "bsrmovh_set\t$src", []>,
- EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ "bsrmovh\t{$src, %bsr0|bsr0, $src}", []>,
+ EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
- // Get BSR high half (W0): BSR is the implicit source, ZMM/mem the explicit
- // destination
let Uses = [BSR0] in {
def BSRMOVHrr_get : I<0x95, MRM0r, (outs VR512:$dst),
(ins),
- "bsrmovh_get\t$dst",
- [(set (v64i8 VR512:$dst), (int_x86_bsrmovh_get))]>,
- EVEX, XD, T_MAP6, EVEX_V512;
+ "bsrmovh\t{%bsr0, $dst|$dst, bsr0}",
+ [(set (v64i8 VR512:$dst), (int_x86_acev1_bsr0movhextract))]>,
+ EVEX, XD, T_MAP6, EVEX_V512;
let mayStore = 1 in
def BSRMOVHmr_get : I<0x95, MRM0m, (outs),
(ins f512mem:$dst),
- "bsrmovh_get\t$dst",
- [(store (v64i8 (int_x86_bsrmovh_get)), addr:$dst)]>,
- EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ "bsrmovh\t{%bsr0, $dst|$dst, bsr0}",
+ [(store (v64i8 (int_x86_acev1_bsr0movhextract)), addr:$dst)]>,
+ EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
}
// BSRMOVL - Move Low to/from BSR
-// EVEX.512.F3.MAP6.W1 95 /r - BSRMOVL bsr0, zmm1/m512 (load, W1)
-// EVEX.512.F3.MAP6.W0 95 /r - BSRMOVL zmm1/m512, bsr0 (store, W0)
-// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
-// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
-// AsmString uses a "_set"/"_get" suffix for asm matching; the printer emits the
-// real mnemonic.
-let SchedRW = [WriteVecLogic] in {
- // Set BSR low half (W1): BSR is both input and output (read-modify-write)
+let SchedRW = [WriteSystem] in {
let Defs = [BSR0], Uses = [BSR0] in {
def BSRMOVLrr_set : I<0x95, MRM0r, (outs),
(ins VR512:$src),
- "bsrmovl_set\t$src",
- [(int_x86_bsrmovl_set (v64i8 VR512:$src))]>,
- EVEX, XS, T_MAP6, REX_W, EVEX_V512;
+ "bsrmovl\t{$src, %bsr0|bsr0, $src}",
+ [(int_x86_acev1_bsr0movlinsert (v64i8 VR512:$src))]>,
+ EVEX, XS, T_MAP6, REX_W, EVEX_V512;
let mayLoad = 1 in
def BSRMOVLrm_set : I<0x95, MRM0m, (outs),
(ins f512mem:$src),
- "bsrmovl_set\t$src", []>,
- EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ "bsrmovl\t{$src, %bsr0|bsr0, $src}", []>,
+ EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
- // Get BSR low half (W0): BSR is the implicit source, ZMM/mem the explicit
- // destination
let Uses = [BSR0] in {
def BSRMOVLrr_get : I<0x95, MRM0r, (outs VR512:$dst),
(ins),
- "bsrmovl_get\t$dst",
- [(set (v64i8 VR512:$dst), (int_x86_bsrmovl_get))]>,
- EVEX, XS, T_MAP6, EVEX_V512;
+ "bsrmovl\t{%bsr0, $dst|$dst, bsr0}",
+ [(set (v64i8 VR512:$dst), (int_x86_acev1_bsr0movlextract))]>,
+ EVEX, XS, T_MAP6, EVEX_V512;
let mayStore = 1 in
def BSRMOVLmr_get : I<0x95, MRM0m, (outs),
(ins f512mem:$dst),
- "bsrmovl_get\t$dst",
- [(store (v64i8 (int_x86_bsrmovl_get)), addr:$dst)]>,
- EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ "bsrmovl\t{%bsr0, $dst|$dst, bsr0}",
+ [(store (v64i8 (int_x86_acev1_bsr0movlextract)), addr:$dst)]>,
+ EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
}
// TILEMOVCOL - Move Column from Vector to Tile
-// EVEX.512.66.0F3A.W1 2F /r ib - TILEMOVCOL tmm1, zmm2, imm8
-// EVEX.512.66.0F38.W1 4B /r - TILEMOVCOL tmm1, zmm2, r32
-// For r32 variant: GR32 is encoded in EVEX.vvvv (use MRMSrcReg4VOp3 + VVVV)
-let SchedRW = [WriteVecLogic] in {
- def TILEMOVCOLri : Ii8<0x2F, MRMSrcReg, (outs TILE:$dst),
- (ins VR512:$src, u8imm:$idx),
- "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, TA, REX_W, EVEX_V512;
-
- def TILEMOVCOLrr : I<0x4B, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins VR512:$src, GR32:$idx),
- "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+let SchedRW = [WriteSystem] in {
+ def TILEMOVCOLtri : Ii8<0x2F, MRMSrcReg, (outs TILE:$dst),
+ (ins VR512:$src, u8imm:$idx),
+ "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def TILEMOVCOLtre : I<0x4B, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins VR512:$src, GR32:$idx),
+ "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
}
// TILEMOVROW - Move Row from Vector to Tile
-// EVEX.512.66.0F3A.W1 07 /r ib - TILEMOVROW tmm1, zmm2, imm8
-// EVEX.512.66.0F38.W1 4A /r - TILEMOVROW tmm1, zmm2, r32
-// For r32 variant: GR32 is encoded in EVEX.vvvv (use MRMSrcReg4VOp3 + VVVV)
-let SchedRW = [WriteVecLogic] in {
- def TILEMOVROWri : Ii8<0x07, MRMSrcReg, (outs TILE:$dst),
- (ins VR512:$src, u8imm:$idx),
- "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, TA, REX_W, EVEX_V512;
-
- def TILEMOVROWrr : I<0x4A, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins VR512:$src, GR32:$idx),
- "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+let SchedRW = [WriteSystem] in {
+ def TILEMOVROWtri : Ii8<0x07, MRMSrcReg, (outs TILE:$dst),
+ (ins VR512:$src, u8imm:$idx),
+ "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def TILEMOVROWtre : I<0x4A, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins VR512:$src, GR32:$idx),
+ "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
}
// Outer Product Instructions
// TOP2BF16PS - Outer Product BF16 to Single Precision
-// EVEX.512.F3.0F38.W0 5C /r - TOP2BF16PS tmm1, zmm2, zmm3
-// Operand encoding from ACE spec page 52:
-// Operand 1: ModRM:reg (rw) = tmm1
-// Operand 2: ModRM:r/m (r) = zmm2
-// Operand 3: EVEX.vvvv (r) = zmm3
-// Use MRMSrcReg4VOp3: operand 1 goes to r/m and operand 2 to vvvv, so src2
-// and src3 are listed in the opposite order from the spec.
-let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
- def TOP2BF16PSrrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
+let Constraints = "$src1 = $dst", SchedRW = [WriteSystem] in {
+ def TOP2BF16PStrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
"top2bf16ps\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, XS, T8, EVEX_V512;
+ EVEX, VVVV, XS, T8, EVEX_V512;
}
-// TOP4 Instructions - All use opcode 0x5E, differentiated by prefix
-// EVEX.512.NP.0F38.W0 5E /r - TOP4BUUD tmm1, zmm2, zmm3 (unsigned/unsigned)
-// EVEX.512.66.0F38.W0 5E /r - TOP4BUSD tmm1, zmm2, zmm3 (unsigned/signed)
-// EVEX.512.F2.0F38.W0 5E /r - TOP4BSSD tmm1, zmm2, zmm3 (signed/signed)
-// EVEX.512.F3.0F38.W0 5E /r - TOP4BSUD tmm1, zmm2, zmm3 (signed/unsigned)
-// Operand encoding from ACE spec page 54:
-// Operand 1: ModRM:reg (rw) = tmm1
-// Operand 2: ModRM:r/m (r) = zmm2
-// Operand 3: EVEX.vvvv (r) = zmm3
-// Use MRMSrcReg4VOp3: operand 1 goes to r/m and operand 2 to vvvv, so src2
-// and src3 are listed in the opposite order from the spec.
-let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
- def TOP4BUUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+// TOP4 Instructions - integer outer products
+let Constraints = "$src1 = $dst", SchedRW = [WriteSystem] in {
+ def TOP4BUUDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
"top4buud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, T8, EVEX_V512;
+ EVEX, VVVV, T8, EVEX_V512;
- def TOP4BUSDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ def TOP4BUSDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
"top4busd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, PD, T8, EVEX_V512;
+ EVEX, VVVV, PD, T8, EVEX_V512;
- def TOP4BSSDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ def TOP4BSSDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
"top4bssd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, XD, T8, EVEX_V512;
+ EVEX, VVVV, XD, T8, EVEX_V512;
- def TOP4BSUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ def TOP4BSUDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
"top4bsud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, XS, T8, EVEX_V512;
+ EVEX, VVVV, XS, T8, EVEX_V512;
}
// TOP4MX Instructions - Mixed Precision Outer Products with Immediate
-// All use opcode 0x8D with different prefixes, or 0x8F
-// EVEX.512.NP.0F3A.W0 8D /r ib - TOP4MXBF8PS tmm1, zmm2, zmm3, imm8
-// EVEX.512.66.0F3A.W0 8D /r ib - TOP4MXHF8PS tmm1, zmm2, zmm3, imm8
-// EVEX.512.F2.0F3A.W0 8D /r ib - TOP4MXBHF8PS tmm1, zmm2, zmm3, imm8
-// EVEX.512.F3.0F3A.W0 8D /r ib - TOP4MXHBF8PS tmm1, zmm2, zmm3, imm8
-// EVEX.512.F2.0F3A.W0 8F /r ib - TOP4MXBSSPS tmm1, zmm2, zmm3, imm8
-// Operand encoding from ACE spec page 47:
-// Operand 1: ModRM:reg (rw) = tmm1
-// Operand 2: ModRM:r/m (r) = zmm2
-// Operand 3: EVEX.vvvv (r) = zmm3
-// Operand 4: imm8
-// MRMSrcReg encodes src1 in reg, src2 in r/m and src3 in vvvv, so the operands
-// are listed swapped to match the spec order.
-let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic], Uses = [BSR0] in {
- def TOP4MXBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+let Constraints = "$src1 = $dst", SchedRW = [WriteSystem], Uses = [BSR0] in {
+ def TOP4MXBF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, TA, EVEX_V512;
+ EVEX, VVVV, TA, EVEX_V512;
- def TOP4MXHF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ def TOP4MXHF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, PD, TA, EVEX_V512;
+ EVEX, VVVV, PD, TA, EVEX_V512;
- def TOP4MXBHF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ def TOP4MXBHF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxbhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, XD, TA, EVEX_V512;
+ EVEX, VVVV, XD, TA, EVEX_V512;
- def TOP4MXHBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ def TOP4MXHBF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxhbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, XS, TA, EVEX_V512;
+ EVEX, VVVV, XS, TA, EVEX_V512;
- def TOP4MXBSSPSrrri : Ii8<0x8F, MRMSrcReg, (outs TILE:$dst),
+ def TOP4MXBSSPStrri : Ii8<0x8F, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxbssps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, XD, TA, EVEX_V512;
+ EVEX, VVVV, XD, TA, EVEX_V512;
}
//===----------------------------------------------------------------------===//
@@ -258,177 +196,144 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic], Uses = [BSR0] in {
// Pseudo instructions for user-facing intrinsics (immediate tile IDs).
// usesCustomInserter expands these to real instructions in X86ISelLowering.
let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
- // BSRINIT pseudo - no operands, just triggers BSR initialization
def PBSRINIT : PseudoI<(outs), (ins),
- [(int_x86_bsrinit)]>;
+ [(int_x86_acev1_bsr0init)]>;
- // Note: BSRMOVF, BSRMOVH, BSRMOVL intrinsics are handled directly
- // by patterns on the real instructions (BSRMOVFrr, BSRMOVHrr_set, etc.)
+ def PTILEMOVCOLtri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
- // TILEMOVCOL pseudo - immediate index variant
- // dst: tile ID (imm), src: ZMM value (VR512), idx: column index (imm)
- // Custom lowering picks the imm or reg variant, so there is no pattern here.
- def PTILEMOVCOL : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+ def PTILEMOVCOLtre : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
- // TILEMOVCOL pseudo - register index variant
- def PTILEMOVCOL_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+ def PTILEMOVROWtri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
- // TILEMOVROW pseudo - immediate index variant (ACE: write to tile)
- // dst: tile ID (imm), src: ZMM value (VR512), idx: row index (imm)
- // Custom lowering picks the imm or reg variant, so there is no pattern here.
- def PTILEMOVROW : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+ def PTILEMOVROWtre : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
- // TILEMOVROW pseudo - register index variant (ACE: write to tile)
- def PTILEMOVROW_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+ def PTOP2BF16PStrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_acev1_top2bf16ps timm:$dst, VR512:$src1, VR512:$src2)]>;
- // TOP2BF16PS pseudo - tile dst by ID, ZMM sources by value
- def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_top2bf16ps timm:$dst, VR512:$src1, VR512:$src2)]>;
+ def PTOP4BUUDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_acev1_top4buud timm:$dst, VR512:$src1, VR512:$src2)]>;
- // TOP4 pseudos - tile dst by ID (also the accumulator), ZMM sources by value
- def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_top4buud timm:$dst, VR512:$src1, VR512:$src2)]>;
+ def PTOP4BUSDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_acev1_top4busd timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_top4busd timm:$dst, VR512:$src1, VR512:$src2)]>;
+ def PTOP4BSSDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_acev1_top4bssd timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_top4bssd timm:$dst, VR512:$src1, VR512:$src2)]>;
+ def PTOP4BSUDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_acev1_top4bsud timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_top4bsud timm:$dst, VR512:$src1, VR512:$src2)]>;
+ def PTOP4MXHF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_acev1_top4mxhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- // TOP4MX pseudos - tile dst by ID, ZMM sources by value, plus imm8
- def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_top4mxhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
+ def PTOP4MXBHF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_acev1_top4mxbhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_top4mxbhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
+ def PTOP4MXHBF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_acev1_top4mxhbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_top4mxhbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
+ def PTOP4MXBF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_acev1_top4mxbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_top4mxbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
-
- def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_top4mxbssps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
+ def PTOP4MXBSSPStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_acev1_top4mxbssps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
}
// Pseudo instructions for internal intrinsics (register allocation).
-// These are used for IR-level optimization and register allocation.
-// usesCustomInserter is needed to set ACEProgModel in
-// EmitInstrWithCustomInserter.
let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
- // TILEMOVCOL internal - output is TILE, input is ZMM + index
- def PTILEMOVCOLV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
- [(set TILE:$dst,
- (int_x86_tilemovcol_set_internal GR16:$row, GR16:$col,
- VR512:$src, GR32:$idx))]>;
-
- // TILEMOVROW internal - output is TILE, input is ZMM + index
- def PTILEMOVROWV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
- [(set TILE:$dst,
- (int_x86_tilemovrow_set_internal GR16:$row, GR16:$col,
- VR512:$src, GR32:$idx))]>;
-
- // TOP2BF16PS internal - output TILE, inputs: dimensions, TILE acc, two ZMMs
- let Constraints = "$src1 = $dst" in
- def PTOP2BF16PSV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top2bf16ps_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4BUUD internal
- let Constraints = "$src1 = $dst" in
- def PTOP4BUUDV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top4buud_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4BUSD internal
+ def PTILEMOVCOLtreV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (int_x86_acev1_tilemovcolinsert_internal GR16:$row, GR16:$col,
+ VR512:$src, GR32:$idx))]>;
+
+ def PTILEMOVROWtreV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (int_x86_acev1_tilemovrowinsert_internal GR16:$row, GR16:$col,
+ VR512:$src, GR32:$idx))]>;
+
let Constraints = "$src1 = $dst" in
- def PTOP4BUSDV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top4busd_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4BSSD internal
+ def PTOP2BF16PStrrV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top2bf16ps_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
let Constraints = "$src1 = $dst" in
- def PTOP4BSSDV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top4bssd_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4BSUD internal
+ def PTOP4BUUDtrrV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top4buud_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
let Constraints = "$src1 = $dst" in
- def PTOP4BSUDV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top4bsud_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4MXHFBPS internal - imm is BSR index (u8imm matches timm in pattern)
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXHF8PSV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ def PTOP4BUSDtrrV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
TILE:$src1, VR512:$src2, VR512:$src3),
[(set TILE:$dst,
- (int_x86_top4mxhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ (int_x86_acev1_top4busd_internal GR16:$row, GR16:$col, GR16:$k,
TILE:$src1, VR512:$src2, VR512:$src3))]>;
- // TOP4MXBHFBPS internal
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXBHF8PSV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top4mxbhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4MXHBFBPS internal
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXHBF8PSV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_top4mxhbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- // TOP4MXBF8PS internal
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXBF8PSV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BSSDtrrV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
TILE:$src1, VR512:$src2, VR512:$src3),
[(set TILE:$dst,
- (int_x86_top4mxbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ (int_x86_acev1_top4bssd_internal GR16:$row, GR16:$col, GR16:$k,
TILE:$src1, VR512:$src2, VR512:$src3))]>;
- // TOP4MXBSSPS internal
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXBSSPSV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BSUDtrrV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
TILE:$src1, VR512:$src2, VR512:$src3),
[(set TILE:$dst,
- (int_x86_top4mxbssps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ (int_x86_acev1_top4bsud_internal GR16:$row, GR16:$col, GR16:$k,
TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
+ def PTOP4MXHF8PStrriV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top4mxhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
+ def PTOP4MXBHF8PStrriV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top4mxbhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
+ def PTOP4MXHBF8PStrriV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top4mxhbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
+ def PTOP4MXBF8PStrriV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top4mxbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
+ def PTOP4MXBSSPStrriV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_acev1_top4mxbssps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
}
// Pseudo instructions for tile spill and reload (register allocation)
-// ACE has no TILELOADD/TILESTORED, so the register allocator emits these and
-// X86LowerTileCopy expands them into a row-by-row TILEMOVROW sequence.
-// Operand order matches storeRegToStackSlot/loadRegFromStackSlot.
let isPseudo = true, hasNoSchedulingInfo = 1 in {
let mayStore = 1 in
def ACE_TILESPILL : PseudoI<(outs), (ins opaquemem:$dst, TILE:$src), []>;
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index 56df999d5249f2..7c0d568ec2dca8 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -110,20 +110,20 @@ static bool isACEOnlyIntrinsic(Instruction *I) {
return false;
switch (II->getIntrinsicID()) {
// ACE outer product intrinsics
- case Intrinsic::x86_top2bf16ps_internal:
- case Intrinsic::x86_top4buud_internal:
- case Intrinsic::x86_top4busd_internal:
- case Intrinsic::x86_top4bssd_internal:
- case Intrinsic::x86_top4bsud_internal:
+ case Intrinsic::x86_acev1_top2bf16ps_internal:
+ case Intrinsic::x86_acev1_top4buud_internal:
+ case Intrinsic::x86_acev1_top4busd_internal:
+ case Intrinsic::x86_acev1_top4bssd_internal:
+ case Intrinsic::x86_acev1_top4bsud_internal:
// ACE mixed precision intrinsics
- case Intrinsic::x86_top4mxhf8ps_internal:
- case Intrinsic::x86_top4mxbhf8ps_internal:
- case Intrinsic::x86_top4mxhbf8ps_internal:
- case Intrinsic::x86_top4mxbf8ps_internal:
- case Intrinsic::x86_top4mxbssps_internal:
+ case Intrinsic::x86_acev1_top4mxhf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxbhf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxhbf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxbf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxbssps_internal:
// ACE tile movement intrinsics
- case Intrinsic::x86_tilemovcol_set_internal:
- case Intrinsic::x86_tilemovrow_set_internal:
+ case Intrinsic::x86_acev1_tilemovcolinsert_internal:
+ case Intrinsic::x86_acev1_tilemovrowinsert_internal:
return true;
default:
return false;
@@ -254,11 +254,11 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
}
// ACE internal intrinsics - outer products with ZMM sources
// Pattern: (m, n, k, acc_tile, zmm1, zmm2) -> acc_tile += zmm1 * zmm2
- case Intrinsic::x86_top2bf16ps_internal:
- case Intrinsic::x86_top4buud_internal:
- case Intrinsic::x86_top4busd_internal:
- case Intrinsic::x86_top4bssd_internal:
- case Intrinsic::x86_top4bsud_internal: {
+ case Intrinsic::x86_acev1_top2bf16ps_internal:
+ case Intrinsic::x86_acev1_top4buud_internal:
+ case Intrinsic::x86_acev1_top4busd_internal:
+ case Intrinsic::x86_acev1_top4bssd_internal:
+ case Intrinsic::x86_acev1_top4bsud_internal: {
switch (OpNo) {
case 3: // Accumulator tile
Row = II->getArgOperand(0);
@@ -276,12 +276,11 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
break;
}
// ACE TOP4MX intrinsics - mixed precision with BSR index
- // Pattern: (m, n, k, bsr_idx, acc_tile, zmm1, zmm2)
- case Intrinsic::x86_top4mxhf8ps_internal:
- case Intrinsic::x86_top4mxbhf8ps_internal:
- case Intrinsic::x86_top4mxhbf8ps_internal:
- case Intrinsic::x86_top4mxbf8ps_internal:
- case Intrinsic::x86_top4mxbssps_internal: {
+ case Intrinsic::x86_acev1_top4mxhf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxbhf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxhbf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxbf8ps_internal:
+ case Intrinsic::x86_acev1_top4mxbssps_internal: {
switch (OpNo) {
case 4: // Accumulator tile
Row = II->getArgOperand(0);
@@ -298,8 +297,8 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
}
// ACE TILEMOV intrinsics - move ZMM to tile row/column
// Pattern: (m, n, zmm_src, idx) -> tile
- case Intrinsic::x86_tilemovcol_set_internal:
- case Intrinsic::x86_tilemovrow_set_internal: {
+ case Intrinsic::x86_acev1_tilemovcolinsert_internal:
+ case Intrinsic::x86_acev1_tilemovrowinsert_internal: {
// Output tile shape
Row = II->getArgOperand(0);
Col = II->getArgOperand(1);
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 4b9bf9639c2420..0c29d84c15d81d 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -75,7 +75,7 @@ static void emitACETileReload(MachineBasicBlock &MBB,
addFrameReference(MIB, TileSS, Row * ACETileRowSize);
// tilemovrow $row, %zmm, %tmm
- BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWri), DstReg)
+ BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWtri), DstReg)
.addReg(ScratchZMM, RegState::Kill)
.addImm(Row);
}
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index bcc7f463e48d8f..8b66598eb6e477 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -1147,19 +1147,18 @@ static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
case X86::PTDPBHF8PSV:
case X86::PTDPHBF8PSV:
case X86::PTDPHF8PSV:
- // ACE internal pseudos - same pattern: operands 1,2 are row,col
- case X86::PTOP2BF16PSV:
- case X86::PTOP4BUUDV:
- case X86::PTOP4BUSDV:
- case X86::PTOP4BSSDV:
- case X86::PTOP4BSUDV:
- case X86::PTOP4MXHF8PSV:
- case X86::PTOP4MXBHF8PSV:
- case X86::PTOP4MXHBF8PSV:
- case X86::PTOP4MXBF8PSV:
- case X86::PTOP4MXBSSPSV:
- case X86::PTILEMOVCOLV:
- case X86::PTILEMOVROWV: {
+ case X86::PTOP2BF16PStrrV:
+ case X86::PTOP4BUUDtrrV:
+ case X86::PTOP4BUSDtrrV:
+ case X86::PTOP4BSSDtrrV:
+ case X86::PTOP4BSUDtrrV:
+ case X86::PTOP4MXHF8PStrriV:
+ case X86::PTOP4MXBHF8PStrriV:
+ case X86::PTOP4MXHBF8PStrriV:
+ case X86::PTOP4MXBF8PStrriV:
+ case X86::PTOP4MXBSSPStrriV:
+ case X86::PTILEMOVCOLtreV:
+ case X86::PTILEMOVROWtreV: {
MachineOperand &MO1 = MI->getOperand(1);
MachineOperand &MO2 = MI->getOperand(2);
ShapeT Shape(&MO1, &MO2, MRI);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index 364979f3ce8cc9..be03118420ed26 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -854,13 +854,6 @@ let CopyCost = -1 in // Don't allow copying of tile registers
def TILE : RegisterClass<"X86", [x86amx], 8192,
(sequence "TMM%u", 0, 7)> {let Size = 8192;}
-// Block Scale Registers (ACE)
-let CopyCost = -1 in // Don't allow copying of BSR register
-def BSR : RegisterClass<"X86", [untyped], 0, (add BSR0)> {
- let Size = 1024;
- let isAllocatable = 0;
-}
-
//===----------------------------------------------------------------------===//
// Register categories.
//
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index a86f536fc2951e..bacbea6fbccac7 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -641,11 +641,8 @@ constexpr FeatureBitset ImpliedFeaturesAMX_AVX512 =
FeatureAMX_TILE | FeatureAVX10_2;
constexpr FeatureBitset ImpliedFeaturesHRESET = {};
-// ACE v1 Features. AVX10.1 is the vector baseline required by the ACE v1
-// feature detection algorithm. AMX-TILE provides the tile registers and tile
-// management instructions that ACE reuses; per spec section 15.5.5 an ACE-only
-// implementation still reports it.
-constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAVX10_1 | FeatureAMX_TILE;
+// ACE v1 Features
+constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAVX10_1;
constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {};
constexpr FeatureBitset ImpliedFeaturesCMPCCXADD = {};
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
index ab72e042e0b991..e3fc8db107404d 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
@@ -10,12 +10,12 @@ define void @test_acev1_ra(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; CHECK-LABEL: name: test_acev1_ra
; CHECK: PLDTILECFGV
; CHECK: PTILEZEROV
- ; CHECK: PTOP2BF16PSV
+ ; CHECK: PTOP2BF16PStrrV
; CHECK: PTILEMOVROWrtiV
entry:
%c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract using TILEMOVROW (ACE v1 pattern)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
@@ -30,13 +30,13 @@ define void @test_acev1_ra_multi(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %
; CHECK: PLDTILECFGV
; CHECK: PTILEZEROV
; CHECK: PTILEZEROV
- ; CHECK: PTOP2BF16PSV
+ ; CHECK: PTOP2BF16PStrrV
entry:
%c1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
%c2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
- %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c2, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d1 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d2 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c2, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract using TILEMOVROW
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
@@ -50,13 +50,13 @@ define void @test_acev1_ra_int(ptr %out, <64 x i8> %a_i8) nounwind {
; CHECK-LABEL: name: test_acev1_ra_int
; CHECK: PLDTILECFGV
; CHECK: PTILEZEROV
- ; CHECK: PTOP4BSSDV
- ; CHECK: PTOP4BUUDV
+ ; CHECK: PTOP4BSSDtrrV
+ ; CHECK: PTOP4BUUDtrrV
entry:
%c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d1 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
- %d2 = tail call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d1 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d2 = tail call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
; Extract using TILEMOVROW
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
@@ -76,11 +76,11 @@ entry:
br i1 %icmp, label %then, label %else
then:
- %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d1 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
br label %merge
else:
- %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_b, <32 x bfloat> %zmm_a)
+ %d2 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_b, <32 x bfloat> %zmm_a)
br label %merge
merge:
@@ -94,7 +94,7 @@ merge:
}
declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
-declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
-declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
index d7a913f8e22900..621b1e58fd1ba3 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
@@ -10,41 +10,41 @@
; anything.
; Test BSRINIT intrinsic
-define void @test_bsrinit() {
-; CHECK-LABEL: test_bsrinit:
+define void @test_bsr0init() {
+; CHECK-LABEL: test_bsr0init:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrinit %bsr0 # encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.bsrinit()
+ call void @llvm.x86.acev1.bsr0init()
ret void
}
; Test BSRMOVF intrinsic
-define void @test_bsrmovf(<64 x i8> %a, <64 x i8> %b) {
-; CHECK-LABEL: test_bsrmovf:
+define void @test_bsr0movf(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_bsr0movf:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0xc1]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.bsr0movf(<64 x i8> %a, <64 x i8> %b)
ret void
}
; Test BSRMOVF intrinsic with memory operand
-define void @test_bsrmovf_mem(<64 x i8> %a, ptr %p) {
-; CHECK-LABEL: test_bsrmovf_mem:
+define void @test_bsr0movf_mem(<64 x i8> %a, ptr %p) {
+; CHECK-LABEL: test_bsr0movf_mem:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovf (%rdi), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x07]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
%b = load <64 x i8>, ptr %p
- call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.bsr0movf(<64 x i8> %a, <64 x i8> %b)
ret void
}
; Test BSRMOVF intrinsic with memory operand (base+index*scale+disp)
-define void @test_bsrmovf_mem_amode(<64 x i8> %a, ptr %base, i64 %idx) {
-; CHECK-LABEL: test_bsrmovf_mem_amode:
+define void @test_bsr0movf_mem_amode(<64 x i8> %a, ptr %base, i64 %idx) {
+; CHECK-LABEL: test_bsr0movf_mem_amode:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovf 64(%rdi,%rsi,4), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x44,0xb7,0x01]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
@@ -52,128 +52,128 @@ define void @test_bsrmovf_mem_amode(<64 x i8> %a, ptr %base, i64 %idx) {
%elt = getelementptr i32, ptr %base, i64 %idx
%addr = getelementptr i8, ptr %elt, i64 64
%b = load <64 x i8>, ptr %addr
- call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.bsr0movf(<64 x i8> %a, <64 x i8> %b)
ret void
}
; Test BSRMOVF intrinsic with unaligned memory operand
-define void @test_bsrmovf_mem_unaligned(<64 x i8> %a, ptr %p) {
-; CHECK-LABEL: test_bsrmovf_mem_unaligned:
+define void @test_bsr0movf_mem_unaligned(<64 x i8> %a, ptr %p) {
+; CHECK-LABEL: test_bsr0movf_mem_unaligned:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovf (%rdi), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x07]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
%b = load <64 x i8>, ptr %p, align 1
- call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.bsr0movf(<64 x i8> %a, <64 x i8> %b)
ret void
}
; Test BSRMOVF intrinsic with memory operand in first source (not foldable)
-define void @test_bsrmovf_mem_src1(ptr %p, <64 x i8> %b) {
-; CHECK-LABEL: test_bsrmovf_mem_src1:
+define void @test_bsr0movf_mem_src1(ptr %p, <64 x i8> %b) {
+; CHECK-LABEL: test_bsr0movf_mem_src1:
; CHECK: # %bb.0:
; CHECK-NEXT: vmovaps (%rdi), %zmm1 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0x0f]
; CHECK-NEXT: bsrmovf %zmm0, %zmm1, %bsr0 # encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
%a = load <64 x i8>, ptr %p
- call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.bsr0movf(<64 x i8> %a, <64 x i8> %b)
ret void
}
; Test BSRMOVH set intrinsic
-define void @test_bsrmovh_set(<64 x i8> %a) {
-; CHECK-LABEL: test_bsrmovh_set:
+define void @test_bsr0movhinsert(<64 x i8> %a) {
+; CHECK-LABEL: test_bsr0movhinsert:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovh %zmm0, %bsr0 # encoding: [0x62,0xf6,0xff,0x48,0x95,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.bsrmovh.set(<64 x i8> %a)
+ call void @llvm.x86.acev1.bsr0movhinsert(<64 x i8> %a)
ret void
}
; Test BSRMOVH set intrinsic with memory operand
-define void @test_bsrmovh_set_mem(ptr %p) {
-; CHECK-LABEL: test_bsrmovh_set_mem:
+define void @test_bsr0movhinsert_mem(ptr %p) {
+; CHECK-LABEL: test_bsr0movhinsert_mem:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovh (%rdi), %bsr0 # encoding: [0x62,0xf6,0xff,0x48,0x95,0x07]
; CHECK-NEXT: retq # encoding: [0xc3]
%a = load <64 x i8>, ptr %p
- call void @llvm.x86.bsrmovh.set(<64 x i8> %a)
+ call void @llvm.x86.acev1.bsr0movhinsert(<64 x i8> %a)
ret void
}
; Test BSRMOVH get intrinsic
-define <64 x i8> @test_bsrmovh_get() {
-; CHECK-LABEL: test_bsrmovh_get:
+define <64 x i8> @test_bsr0movhextract() {
+; CHECK-LABEL: test_bsr0movhextract:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovh %bsr0, %zmm0 # encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc0]
; CHECK-NEXT: retq # encoding: [0xc3]
- %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ %result = call <64 x i8> @llvm.x86.acev1.bsr0movhextract()
ret <64 x i8> %result
}
; Test BSRMOVH get intrinsic with memory operand
-define void @test_bsrmovh_get_mem(ptr %p) {
-; CHECK-LABEL: test_bsrmovh_get_mem:
+define void @test_bsr0movhextract_mem(ptr %p) {
+; CHECK-LABEL: test_bsr0movhextract_mem:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovh %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7f,0x48,0x95,0x07]
; CHECK-NEXT: retq # encoding: [0xc3]
- %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ %result = call <64 x i8> @llvm.x86.acev1.bsr0movhextract()
store <64 x i8> %result, ptr %p
ret void
}
; Test BSRMOVH get intrinsic with unaligned memory operand
-define void @test_bsrmovh_get_mem_unaligned(ptr %p) {
-; CHECK-LABEL: test_bsrmovh_get_mem_unaligned:
+define void @test_bsr0movhextract_mem_unaligned(ptr %p) {
+; CHECK-LABEL: test_bsr0movhextract_mem_unaligned:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovh %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7f,0x48,0x95,0x07]
; CHECK-NEXT: retq # encoding: [0xc3]
- %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ %result = call <64 x i8> @llvm.x86.acev1.bsr0movhextract()
store <64 x i8> %result, ptr %p, align 1
ret void
}
; Test BSRMOVL set intrinsic
-define void @test_bsrmovl_set(<64 x i8> %a) {
-; CHECK-LABEL: test_bsrmovl_set:
+define void @test_bsr0movlinsert(<64 x i8> %a) {
+; CHECK-LABEL: test_bsr0movlinsert:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovl %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.bsrmovl.set(<64 x i8> %a)
+ call void @llvm.x86.acev1.bsr0movlinsert(<64 x i8> %a)
ret void
}
; Test BSRMOVL set intrinsic with memory operand
-define void @test_bsrmovl_set_mem(ptr %p) {
-; CHECK-LABEL: test_bsrmovl_set_mem:
+define void @test_bsr0movlinsert_mem(ptr %p) {
+; CHECK-LABEL: test_bsr0movlinsert_mem:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovl (%rdi), %bsr0 # encoding: [0x62,0xf6,0xfe,0x48,0x95,0x07]
; CHECK-NEXT: retq # encoding: [0xc3]
%a = load <64 x i8>, ptr %p
- call void @llvm.x86.bsrmovl.set(<64 x i8> %a)
+ call void @llvm.x86.acev1.bsr0movlinsert(<64 x i8> %a)
ret void
}
; Test BSRMOVL get intrinsic
-define <64 x i8> @test_bsrmovl_get() {
-; CHECK-LABEL: test_bsrmovl_get:
+define <64 x i8> @test_bsr0movlextract() {
+; CHECK-LABEL: test_bsr0movlextract:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovl %bsr0, %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc0]
; CHECK-NEXT: retq # encoding: [0xc3]
- %result = call <64 x i8> @llvm.x86.bsrmovl.get()
+ %result = call <64 x i8> @llvm.x86.acev1.bsr0movlextract()
ret <64 x i8> %result
}
; Test BSRMOVL get intrinsic with memory operand
-define void @test_bsrmovl_get_mem(ptr %p) {
-; CHECK-LABEL: test_bsrmovl_get_mem:
+define void @test_bsr0movlextract_mem(ptr %p) {
+; CHECK-LABEL: test_bsr0movlextract_mem:
; CHECK: # %bb.0:
; CHECK-NEXT: bsrmovl %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7e,0x48,0x95,0x07]
; CHECK-NEXT: retq # encoding: [0xc3]
- %result = call <64 x i8> @llvm.x86.bsrmovl.get()
+ %result = call <64 x i8> @llvm.x86.acev1.bsr0movlextract()
store <64 x i8> %result, ptr %p
ret void
}
@@ -188,7 +188,7 @@ define void @test_top2bf16ps(<32 x bfloat> %a, <32 x bfloat> %b) {
; CHECK-NEXT: top2bf16ps %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x76,0x48,0x5c,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top2bf16ps(i8 0, <32 x bfloat> %a, <32 x bfloat> %b)
+ call void @llvm.x86.acev1.top2bf16ps(i8 0, <32 x bfloat> %a, <32 x bfloat> %b)
ret void
}
@@ -199,7 +199,7 @@ define void @test_top4buud(<64 x i8> %a, <64 x i8> %b) {
; CHECK-NEXT: top4buud %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x74,0x48,0x5e,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4buud(i8 0, <64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.top4buud(i8 0, <64 x i8> %a, <64 x i8> %b)
ret void
}
@@ -210,7 +210,7 @@ define void @test_top4busd(<64 x i8> %a, <64 x i8> %b) {
; CHECK-NEXT: top4busd %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x75,0x48,0x5e,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4busd(i8 0, <64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.top4busd(i8 0, <64 x i8> %a, <64 x i8> %b)
ret void
}
@@ -221,7 +221,7 @@ define void @test_top4bssd(<64 x i8> %a, <64 x i8> %b) {
; CHECK-NEXT: top4bssd %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x77,0x48,0x5e,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4bssd(i8 0, <64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.top4bssd(i8 0, <64 x i8> %a, <64 x i8> %b)
ret void
}
@@ -232,126 +232,126 @@ define void @test_top4bsud(<64 x i8> %a, <64 x i8> %b) {
; CHECK-NEXT: top4bsud %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x76,0x48,0x5e,0xc0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4bsud(i8 0, <64 x i8> %a, <64 x i8> %b)
+ call void @llvm.x86.acev1.top4bsud(i8 0, <64 x i8> %a, <64 x i8> %b)
ret void
}
; Test TOP4MXHF8PS intrinsic
-define void @test_top4mxhf8ps(<16 x i32> %a, <16 x i32> %b) {
+define void @test_top4mxhf8ps(<64 x i8> %a, <64 x i8> %b) {
; CHECK-LABEL: test_top4mxhf8ps:
; CHECK: # %bb.0:
; CHECK-NEXT: top4mxhf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x8d,0xc1,0x05]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4mxhf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ call void @llvm.x86.acev1.top4mxhf8ps(i8 0, <64 x i8> %a, <64 x i8> %b, i8 5)
ret void
}
; Test TOP4MXBHF8PS intrinsic
-define void @test_top4mxbhf8ps(<16 x i32> %a, <16 x i32> %b) {
+define void @test_top4mxbhf8ps(<64 x i8> %a, <64 x i8> %b) {
; CHECK-LABEL: test_top4mxbhf8ps:
; CHECK: # %bb.0:
; CHECK-NEXT: top4mxbhf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x8d,0xc1,0x05]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4mxbhf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ call void @llvm.x86.acev1.top4mxbhf8ps(i8 0, <64 x i8> %a, <64 x i8> %b, i8 5)
ret void
}
; Test TOP4MXHBF8PS intrinsic
-define void @test_top4mxhbf8ps(<16 x i32> %a, <16 x i32> %b) {
+define void @test_top4mxhbf8ps(<64 x i8> %a, <64 x i8> %b) {
; CHECK-LABEL: test_top4mxhbf8ps:
; CHECK: # %bb.0:
; CHECK-NEXT: top4mxhbf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x8d,0xc1,0x05]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4mxhbf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ call void @llvm.x86.acev1.top4mxhbf8ps(i8 0, <64 x i8> %a, <64 x i8> %b, i8 5)
ret void
}
; Test TOP4MXBF8PS intrinsic
-define void @test_top4mxbf8ps(<16 x i32> %a, <16 x i32> %b) {
+define void @test_top4mxbf8ps(<64 x i8> %a, <64 x i8> %b) {
; CHECK-LABEL: test_top4mxbf8ps:
; CHECK: # %bb.0:
; CHECK-NEXT: top4mxbf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7c,0x48,0x8d,0xc1,0x05]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4mxbf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ call void @llvm.x86.acev1.top4mxbf8ps(i8 0, <64 x i8> %a, <64 x i8> %b, i8 5)
ret void
}
; Test TOP4MXBSSPS intrinsic (MX INT8 SxS with BSR scaling)
-define void @test_top4mxbssps(<16 x i32> %a, <16 x i32> %b) {
+define void @test_top4mxbssps(<64 x i8> %a, <64 x i8> %b) {
; CHECK-LABEL: test_top4mxbssps:
; CHECK: # %bb.0:
; CHECK-NEXT: top4mxbssps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x8f,0xc1,0x05]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.top4mxbssps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ call void @llvm.x86.acev1.top4mxbssps(i8 0, <64 x i8> %a, <64 x i8> %b, i8 5)
ret void
}
; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
-define void @test_tilemovcol(<16 x i32> %src) {
-; CHECK-LABEL: test_tilemovcol:
+define void @test_tilemovcolinsert(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovcolinsert:
; CHECK: # %bb.0:
; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xc0,0x03]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.tilemovcol.set(i8 0, <16 x i32> %src, i32 3)
+ call void @llvm.x86.acev1.tilemovcolinsert(i8 0, <16 x i32> %src, i32 3)
ret void
}
; Test TILEMOVROW intrinsic - write to tile (ACE extension)
; Uses constant index - generates immediate form
-define void @test_tilemovrow_to_tile(<16 x i32> %src) {
-; CHECK-LABEL: test_tilemovrow_to_tile:
+define void @test_tilemovrowinsert_to_tile(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovrowinsert_to_tile:
; CHECK: # %bb.0:
; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1 # encoding: [0x62,0xf3,0xfd,0x48,0x07,0xc8,0x05]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.tilemovrow.set(i8 1, <16 x i32> %src, i32 5)
+ call void @llvm.x86.acev1.tilemovrowinsert(i8 1, <16 x i32> %src, i32 5)
ret void
}
; Test TILEMOVCOL with register index
-define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
-; CHECK-LABEL: test_tilemovcol_reg:
+define void @test_tilemovcolinsert_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovcolinsert_reg:
; CHECK: # %bb.0:
; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2 # encoding: [0x62,0xf2,0xc5,0x48,0x4b,0xd0]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.tilemovcol.set(i8 2, <16 x i32> %src, i32 %idx)
+ call void @llvm.x86.acev1.tilemovcolinsert(i8 2, <16 x i32> %src, i32 %idx)
ret void
}
; Test TILEMOVROW with register index
-define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
-; CHECK-LABEL: test_tilemovrow_reg:
+define void @test_tilemovrowinsert_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovrowinsert_reg:
; CHECK: # %bb.0:
; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3 # encoding: [0x62,0xf2,0xc5,0x48,0x4a,0xd8]
; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
; CHECK-NEXT: retq # encoding: [0xc3]
- call void @llvm.x86.tilemovrow.set(i8 3, <16 x i32> %src, i32 %idx)
+ call void @llvm.x86.acev1.tilemovrowinsert(i8 3, <16 x i32> %src, i32 %idx)
ret void
}
; Intrinsic declarations
-declare void @llvm.x86.bsrinit()
-declare void @llvm.x86.bsrmovf(<64 x i8>, <64 x i8>)
-declare void @llvm.x86.bsrmovh.set(<64 x i8>)
-declare <64 x i8> @llvm.x86.bsrmovh.get()
-declare void @llvm.x86.bsrmovl.set(<64 x i8>)
-declare <64 x i8> @llvm.x86.bsrmovl.get()
-declare void @llvm.x86.tilemovcol.set(i8, <16 x i32>, i32)
-declare void @llvm.x86.tilemovrow.set(i8, <16 x i32>, i32)
-declare void @llvm.x86.top2bf16ps(i8, <32 x bfloat>, <32 x bfloat>)
-declare void @llvm.x86.top4buud(i8, <64 x i8>, <64 x i8>)
-declare void @llvm.x86.top4busd(i8, <64 x i8>, <64 x i8>)
-declare void @llvm.x86.top4bssd(i8, <64 x i8>, <64 x i8>)
-declare void @llvm.x86.top4bsud(i8, <64 x i8>, <64 x i8>)
-declare void @llvm.x86.top4mxhf8ps(i8, <16 x i32>, <16 x i32>, i8)
-declare void @llvm.x86.top4mxbhf8ps(i8, <16 x i32>, <16 x i32>, i8)
-declare void @llvm.x86.top4mxhbf8ps(i8, <16 x i32>, <16 x i32>, i8)
-declare void @llvm.x86.top4mxbf8ps(i8, <16 x i32>, <16 x i32>, i8)
-declare void @llvm.x86.top4mxbssps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.acev1.bsr0init()
+declare void @llvm.x86.acev1.bsr0movf(<64 x i8>, <64 x i8>)
+declare void @llvm.x86.acev1.bsr0movhinsert(<64 x i8>)
+declare <64 x i8> @llvm.x86.acev1.bsr0movhextract()
+declare void @llvm.x86.acev1.bsr0movlinsert(<64 x i8>)
+declare <64 x i8> @llvm.x86.acev1.bsr0movlextract()
+declare void @llvm.x86.acev1.tilemovcolinsert(i8, <16 x i32>, i32)
+declare void @llvm.x86.acev1.tilemovrowinsert(i8, <16 x i32>, i32)
+declare void @llvm.x86.acev1.top2bf16ps(i8, <32 x bfloat>, <32 x bfloat>)
+declare void @llvm.x86.acev1.top4buud(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.acev1.top4busd(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.acev1.top4bssd(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.acev1.top4bsud(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.acev1.top4mxhf8ps(i8, <64 x i8>, <64 x i8>, i8)
+declare void @llvm.x86.acev1.top4mxbhf8ps(i8, <64 x i8>, <64 x i8>, i8)
+declare void @llvm.x86.acev1.top4mxhbf8ps(i8, <64 x i8>, <64 x i8>, i8)
+declare void @llvm.x86.acev1.top4mxbf8ps(i8, <64 x i8>, <64 x i8>, i8)
+declare void @llvm.x86.acev1.top4mxbssps(i8, <64 x i8>, <64 x i8>, i8)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
index 7586363819a695..353c68d95c1c25 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
@@ -15,7 +15,7 @@ define void @test_top2bf16ps(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract using TILEMOVROW (ACE v1 pattern)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
@@ -32,7 +32,7 @@ define void @test_top4buud(ptr %out, <64 x i8> %a_i8) {
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d = call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -48,7 +48,7 @@ define void @test_top4bssd(ptr %out, <64 x i8> %a_i8) {
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d = call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -64,7 +64,7 @@ define void @test_top4busd(ptr %out, <64 x i8> %a_i8) {
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d = call x86_amx @llvm.x86.acev1.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -80,7 +80,7 @@ define void @test_top4bsud(ptr %out, <64 x i8> %a_i8) {
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d = call x86_amx @llvm.x86.acev1.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -88,7 +88,7 @@ define void @test_top4bsud(ptr %out, <64 x i8> %a_i8) {
}
; Test MX FP8 HF8xHF8 -> FP32 with BSR scaling (TOP4MXHF8PS)
-define void @test_top4mxhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+define void @test_top4mxhf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zmm_b) {
; CHECK-LABEL: test_top4mxhf8ps_internal:
; CHECK: ldtilecfg
; CHECK: tilezero
@@ -96,7 +96,7 @@ define void @test_top4mxhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -104,7 +104,7 @@ define void @test_top4mxhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %
}
; Test MX FP8 BF8xHF8 -> FP32 with BSR scaling (TOP4MXBHF8PS)
-define void @test_top4mxbhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+define void @test_top4mxbhf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zmm_b) {
; CHECK-LABEL: test_top4mxbhf8ps_internal:
; CHECK: ldtilecfg
; CHECK: tilezero
@@ -112,7 +112,7 @@ define void @test_top4mxbhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32>
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -120,7 +120,7 @@ define void @test_top4mxbhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32>
}
; Test MX FP8 HF8xBF8 -> FP32 with BSR scaling (TOP4MXHBF8PS)
-define void @test_top4mxhbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+define void @test_top4mxhbf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zmm_b) {
; CHECK-LABEL: test_top4mxhbf8ps_internal:
; CHECK: ldtilecfg
; CHECK: tilezero
@@ -128,7 +128,7 @@ define void @test_top4mxhbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32>
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -136,7 +136,7 @@ define void @test_top4mxhbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32>
}
; Test MX FP8 BF8xBF8 -> FP32 with BSR scaling (TOP4MXBF8PS)
-define void @test_top4mxbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+define void @test_top4mxbf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zmm_b) {
; CHECK-LABEL: test_top4mxbf8ps_internal:
; CHECK: ldtilecfg
; CHECK: tilezero
@@ -144,7 +144,7 @@ define void @test_top4mxbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -152,7 +152,7 @@ define void @test_top4mxbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %
}
; Test MX INT8 SxS -> FP32 with BSR scaling (TOP4MXBSSPS)
-define void @test_top4mxbssps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+define void @test_top4mxbssps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zmm_b) {
; CHECK-LABEL: test_top4mxbssps_internal:
; CHECK: ldtilecfg
; CHECK: tilezero
@@ -160,7 +160,7 @@ define void @test_top4mxbssps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %
; CHECK: tilemovrow
; CHECK: tilerelease
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d = call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
@@ -180,11 +180,11 @@ define void @test_chained_outer_products(ptr %out, <32 x bfloat> %zmm_bf16, <64
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
; BF16 outer product
- %d1 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_bf16, <32 x bfloat> %zmm_bf16)
+ %d1 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_bf16, <32 x bfloat> %zmm_bf16)
; Integer outer products
- %d2 = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
- %d3 = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d2, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d2 = call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d3 = call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d2, <64 x i8> %a_i8, <64 x i8> %a_i8)
; Extract using TILEMOVROW
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
@@ -193,14 +193,14 @@ define void @test_chained_outer_products(ptr %out, <32 x bfloat> %zmm_bf16, <64
}
declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
-declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
-declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare x86_amx @llvm.x86.top4busd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare x86_amx @llvm.x86.top4bsud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare x86_amx @llvm.x86.top4mxhf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
-declare x86_amx @llvm.x86.top4mxbhf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
-declare x86_amx @llvm.x86.top4mxhbf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
-declare x86_amx @llvm.x86.top4mxbf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
-declare x86_amx @llvm.x86.top4mxbssps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.acev1.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4busd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4bsud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4mxhf8ps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4mxbhf8ps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4mxhbf8ps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4mxbf8ps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4mxbssps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
index 711de8d3c98473..32f89f4592356e 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
@@ -16,7 +16,7 @@ define void @test_acev1_basic(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
; ACE BF16 outer product
- %d = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract result using TILEMOVROW (ACE v1 pattern - no TILESTORED)
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
@@ -37,10 +37,10 @@ define void @test_acev1_int_outer_products(ptr %out, <64 x i8> %a_i8, <64 x i8>
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
; Unsigned x Unsigned -> DWORD
- %d0 = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %b_i8)
+ %d0 = call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %b_i8)
; Signed x Signed -> DWORD
- %d1 = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d0, <64 x i8> %a_i8, <64 x i8> %b_i8)
+ %d1 = call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d0, <64 x i8> %a_i8, <64 x i8> %b_i8)
; Extract result using TILEMOVROW
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
@@ -62,9 +62,9 @@ define void @test_acev1_accumulation(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloa
%c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
; Multiple accumulations
- %d0 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
- %d1 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d0, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
- %d2 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d0 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d1 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d0, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d2 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract result using TILEMOVROW
%row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
@@ -90,7 +90,7 @@ define void @test_acev1_tilezero(ptr %out) {
}
declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
-declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
-declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.acev1.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
index 52098739b8d8a4..58887c9623cca0 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
@@ -35,15 +35,15 @@ entry:
%t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
%t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d0 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t0, <32 x bfloat> %a, <32 x bfloat> %b)
- %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t1, <32 x bfloat> %a, <32 x bfloat> %b)
- %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t2, <32 x bfloat> %a, <32 x bfloat> %b)
- %d3 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t3, <32 x bfloat> %a, <32 x bfloat> %b)
- %d4 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t4, <32 x bfloat> %a, <32 x bfloat> %b)
- %d5 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t5, <32 x bfloat> %a, <32 x bfloat> %b)
- %d6 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t6, <32 x bfloat> %a, <32 x bfloat> %b)
- %d7 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t7, <32 x bfloat> %a, <32 x bfloat> %b)
- %d8 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t8, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d0 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t0, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d1 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t1, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d2 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t2, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d3 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t3, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d4 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t4, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d5 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t5, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d6 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t6, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d7 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t7, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d8 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t8, <32 x bfloat> %a, <32 x bfloat> %b)
; Keep all nine results live to the end so the allocator has to spill.
%r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
@@ -86,15 +86,15 @@ entry:
%t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
%t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %d0 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t0, <64 x i8> %a, <64 x i8> %a)
- %d1 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t1, <64 x i8> %a, <64 x i8> %a)
- %d2 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t2, <64 x i8> %a, <64 x i8> %a)
- %d3 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t3, <64 x i8> %a, <64 x i8> %a)
- %d4 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t4, <64 x i8> %a, <64 x i8> %a)
- %d5 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t5, <64 x i8> %a, <64 x i8> %a)
- %d6 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t6, <64 x i8> %a, <64 x i8> %a)
- %d7 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t7, <64 x i8> %a, <64 x i8> %a)
- %d8 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t8, <64 x i8> %a, <64 x i8> %a)
+ %d0 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t0, <64 x i8> %a, <64 x i8> %a)
+ %d1 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t1, <64 x i8> %a, <64 x i8> %a)
+ %d2 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t2, <64 x i8> %a, <64 x i8> %a)
+ %d3 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t3, <64 x i8> %a, <64 x i8> %a)
+ %d4 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t4, <64 x i8> %a, <64 x i8> %a)
+ %d5 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t5, <64 x i8> %a, <64 x i8> %a)
+ %d6 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t6, <64 x i8> %a, <64 x i8> %a)
+ %d7 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t7, <64 x i8> %a, <64 x i8> %a)
+ %d8 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t8, <64 x i8> %a, <64 x i8> %a)
%r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
store volatile <16 x i32> %r0, ptr %out, align 64
@@ -119,6 +119,6 @@ entry:
}
declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
-declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
-declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/MC/X86/ACEV1/acev1-att.s b/llvm/test/MC/X86/ACEV1/acev1-att.s
index a6f13e2c79ab32..81a2b378a0e325 100644
--- a/llvm/test/MC/X86/ACEV1/acev1-att.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-att.s
@@ -143,3 +143,23 @@
// CHECK: top4bsud %zmm0, %zmm16, %tmm1
// CHECK: encoding: [0x62,0xb2,0x7e,0x48,0x5e,0xc8]
top4bsud %zmm0, %zmm16, %tmm1
+
+// CHECK: ldtilecfg (%rax)
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x00]
+ ldtilecfg (%rax)
+
+// CHECK: sttilecfg (%rax)
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x00]
+ sttilecfg (%rax)
+
+// CHECK: tilezero %tmm0
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xc0]
+ tilezero %tmm0
+
+// CHECK: tilezero %tmm7
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf8]
+ tilezero %tmm7
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+ tilerelease
diff --git a/llvm/test/MC/X86/ACEV1/acev1-error.s b/llvm/test/MC/X86/ACEV1/acev1-error.s
index e939b843564033..6e94143ac39295 100644
--- a/llvm/test/MC/X86/ACEV1/acev1-error.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-error.s
@@ -1,39 +1,24 @@
// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1 %s -o /dev/null 2>&1 | FileCheck %s
// Test missing explicit BSR operand
-// CHECK: error: BSR instruction requires explicit %bsr0 operand
+// CHECK: [[#@LINE+1]]:1: error: too few operands for instruction
bsrmovf %zmm1, %zmm2
-// CHECK: error:
+// CHECK: [[#@LINE+1]]:12: error: invalid operand for instruction
tilemovcol %tmm1, %xmm2, $5
-// CHECK: error:
+// CHECK: [[#@LINE+1]]:10: error: invalid operand for instruction
top4busd %tmm1, %ymm2, %zmm3
-// CHECK: error:
+// CHECK: [[#@LINE+1]]:12: error: invalid operand for instruction
tilemovrow $256, %zmm2, %tmm1
-// Test feature gating - ACEV1 instructions require ACEV1 feature
-// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=-acev1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOACEV1
-
-// NOACEV1: error:
-bsrinit %bsr0
-
-// NOACEV1: error:
-top2bf16ps %zmm3, %zmm2, %tmm1
-
-// Test that ACEV1 requires AMX-TILE (dependency check)
-// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1,-amx-tile %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOAMX
-
-// NOAMX: error:
-tilemovcol $5, %zmm2, %tmm1
-
// Test invalid BSR register (only bsr0 exists)
-// CHECK: error:
+// CHECK: [[#@LINE+1]]:23: error: invalid register name
bsrmovf %zmm1, %zmm2, %bsr1
// Test 32-bit mode rejection (ACEV1 is 64-bit only)
// RUN: not llvm-mc -triple i386-unknown-unknown -mattr=+acev1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO32BIT
-// NO32BIT: error:
+// NO32BIT: [[#@LINE+1]]:1: error: instruction requires: 64-bit mode
bsrinit %bsr0
diff --git a/llvm/test/MC/X86/ACEV1/acev1-intel.s b/llvm/test/MC/X86/ACEV1/acev1-intel.s
index daab4536709ece..a1312e65438774 100644
--- a/llvm/test/MC/X86/ACEV1/acev1-intel.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-intel.s
@@ -143,3 +143,23 @@
// CHECK: top4bsud tmm1, zmm16, zmm0
// CHECK: encoding: [0x62,0xb2,0x7e,0x48,0x5e,0xc8]
top4bsud tmm1, zmm16, zmm0
+
+// CHECK: ldtilecfg [rax]
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0x00]
+ ldtilecfg [rax]
+
+// CHECK: sttilecfg [rax]
+// CHECK: encoding: [0xc4,0xe2,0x79,0x49,0x00]
+ sttilecfg [rax]
+
+// CHECK: tilezero tmm0
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xc0]
+ tilezero tmm0
+
+// CHECK: tilezero tmm7
+// CHECK: encoding: [0xc4,0xe2,0x7b,0x49,0xf8]
+ tilezero tmm7
+
+// CHECK: tilerelease
+// CHECK: encoding: [0xc4,0xe2,0x78,0x49,0xc0]
+ tilerelease
>From ea7bf44b64d9a9ec90f3b9ba1801bf5c856174d2 Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Mon, 16 Nov 2026 07:48:50 +0530
Subject: [PATCH 4/6] [X86][ACEV1] Give ACE spellings to the shared
palette-bearing instructions
ACE configures tile palette 2 and AMX palette 1, so an instruction both
extensions spell the same way must stay distinguishable after ISel. Asking
the subtarget answers wrongly for a function built for an ACE target that
holds only plain AMX code, and silently miscompiled it: a tile loaded
through a vector became a tilezero, dropping the load, and a tile read back
as a vector became poison.
Give the seven shared palette-bearing instructions - TILEZERO, TCVTROWD2PS,
TCVTROWPS2BF16H/L, TCVTROWPS2PHH/L and the TILEMOVROW read form - ACE
intrinsics and pseudos of their own, so the palette is chosen from the
intrinsic while still in IR and, after selection, from the ACE programming
model that the pseudo's opcode records in X86MachineFunctionInfo beside the
AMX one. The read direction of TILEMOVROW also gets a user-facing name,
llvm.x86.acev1.tilemovrowextract, as GCC gives it.
The six row reads take the row index by value in one intrinsic, as GCC's
builtins do, and each has an imm8 encoding alongside the r32 one, so they
get an immediate-index pseudo too and a constant index stops being
materialised into a GPR. The index is not an ImmArg, so the constant
arrives as an ISD::Constant and that pattern matches imm, not timm.
---
clang/include/clang/Basic/BuiltinsX86_64.td | 22 +++
clang/lib/Headers/acev1intrin.h | 35 +++--
clang/lib/Sema/SemaX86.cpp | 8 ++
clang/test/CodeGen/X86/acev1.c | 21 ++-
clang/test/CodeGen/X86/acev1_api.c | 16 +--
clang/test/CodeGen/X86/acev1_errors.c | 26 ++--
clang/test/CodeGen/X86/acev1_feature_errors.c | 6 +-
clang/test/CodeGen/X86/acev1_tile_movement.c | 8 +-
llvm/include/llvm/IR/IntrinsicsX86.td | 66 +++++++++
llvm/lib/Target/X86/X86ExpandPseudo.cpp | 29 +++-
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 3 +-
llvm/lib/Target/X86/X86FastTileConfig.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 101 +++++++++++---
llvm/lib/Target/X86/X86InstrACE.td | 132 ++++++++++++++++++
llvm/lib/Target/X86/X86InstrInfo.cpp | 1 +
llvm/lib/Target/X86/X86LowerAMXType.cpp | 89 +++++++-----
llvm/lib/Target/X86/X86PreTileConfig.cpp | 15 +-
llvm/lib/Target/X86/X86RegisterInfo.cpp | 1 +
llvm/lib/Target/X86/X86TileConfig.cpp | 2 +-
.../CodeGen/X86/ACEV1/acev1-amx-interop.ll | 35 +++++
.../X86/ACEV1/acev1-direct-prog-model.ll | 49 +++++++
.../test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll | 22 +--
.../CodeGen/X86/ACEV1/acev1-intrinsics.ll | 82 +++++++++++
.../CodeGen/X86/ACEV1/acev1-outer-product.ll | 48 +++----
llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll | 20 +--
.../CodeGen/X86/ACEV1/acev1-tile-basic.ll | 78 +++++++++--
.../CodeGen/X86/ACEV1/acev1-tile-spill.ll | 76 +++++-----
27 files changed, 793 insertions(+), 200 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-direct-prog-model.ll
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index 8d1d19367e0c6e..ffcda4ba603480 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -399,6 +399,28 @@ let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def tilemovcolinsert : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
// ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
def tilemovrowinsert : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ def acev1_tilemovrowextract : X86Builtin<"_Vector<16, int>(_Constant unsigned char, unsigned int)">;
+}
+
+// Shared instructions - user-facing (immediate tile ID).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def acev1_tilezero : X86Builtin<"void(_Constant unsigned char)">;
+ def acev1_tcvtrowd2ps : X86Builtin<"_Vector<16, float>(_Constant unsigned char, unsigned int)">;
+ def acev1_tcvtrowps2bf16h : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, unsigned int)">;
+ def acev1_tcvtrowps2bf16l : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, unsigned int)">;
+ def acev1_tcvtrowps2phh : X86Builtin<"_Vector<32, _Float16>(_Constant unsigned char, unsigned int)">;
+ def acev1_tcvtrowps2phl : X86Builtin<"_Vector<32, _Float16>(_Constant unsigned char, unsigned int)">;
+}
+
+// Shared instructions - internal (for IR-level operations).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def acev1_tilezero_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short)">;
+ def acev1_tilemovrowextract_internal : X86Builtin<"_Vector<16, int>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
+ def acev1_tcvtrowd2ps_internal : X86Builtin<"_Vector<16, float>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
+ def acev1_tcvtrowps2bf16h_internal : X86Builtin<"_Vector<32, __bf16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
+ def acev1_tcvtrowps2bf16l_internal : X86Builtin<"_Vector<32, __bf16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
+ def acev1_tcvtrowps2phh_internal : X86Builtin<"_Vector<32, _Float16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
+ def acev1_tcvtrowps2phl_internal : X86Builtin<"_Vector<32, _Float16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
}
// Tile movement internal (for IR-level operations).
diff --git a/clang/lib/Headers/acev1intrin.h b/clang/lib/Headers/acev1intrin.h
index cd5087b7d009d1..14aba6510993ee 100644
--- a/clang/lib/Headers/acev1intrin.h
+++ b/clang/lib/Headers/acev1intrin.h
@@ -76,7 +76,7 @@ _tile_ace_release(void) {
///
/// \param tile
/// Destination tile register ID (0-7).
-#define _tile_ace_zero(tile) __builtin_ia32_tilezero((tile))
+#define _tile_ace_zero(tile) __builtin_ia32_acev1_tilezero((tile))
/// Move a 64-byte ZMM vector to a tile column. The 16 doublewords from
/// the ZMM are written as a vertical column in the tile at the specified
@@ -409,7 +409,8 @@ _bsr0_extractl(void) {
/// Row index selecting the tile row to read.
/// \returns
/// ZMM vector (__m512) holding the converted FP32 elements.
-#define _tile_cvtrow_epi32_ps(tsrc, row) _tile_cvtrowd2ps((tsrc), (row))
+#define _tile_ace_cvtrow_epi32_ps(tsrc, row) \
+ __builtin_ia32_acev1_tcvtrowd2ps((tsrc), (row))
/// Read a row from a tile register and convert its FP32 elements to BF16,
/// placing the results in the high half of each destination dword.
@@ -424,7 +425,8 @@ _bsr0_extractl(void) {
/// Row index selecting the tile row to read.
/// \returns
/// ZMM vector (__m512bh) holding the converted BF16 elements.
-#define _tile_cvtrowh_ps_pbh(tsrc, row) _tile_cvtrowps2bf16h((tsrc), (row))
+#define _tile_ace_cvtrowh_ps_pbh(tsrc, row) \
+ __builtin_ia32_acev1_tcvtrowps2bf16h((tsrc), (row))
/// Read a row from a tile register and convert its FP32 elements to BF16,
/// placing the results in the low half of each destination dword.
@@ -439,7 +441,8 @@ _bsr0_extractl(void) {
/// Row index selecting the tile row to read.
/// \returns
/// ZMM vector (__m512bh) holding the converted BF16 elements.
-#define _tile_cvtrowl_ps_pbh(tsrc, row) _tile_cvtrowps2bf16l((tsrc), (row))
+#define _tile_ace_cvtrowl_ps_pbh(tsrc, row) \
+ __builtin_ia32_acev1_tcvtrowps2bf16l((tsrc), (row))
/// Read a row from a tile register and convert its FP32 elements to FP16,
/// placing the results in the high half of each destination dword.
@@ -454,7 +457,8 @@ _bsr0_extractl(void) {
/// Row index selecting the tile row to read.
/// \returns
/// ZMM vector (__m512h) holding the converted FP16 elements.
-#define _tile_cvtrowh_ps_ph(tsrc, row) _tile_cvtrowps2phh((tsrc), (row))
+#define _tile_ace_cvtrowh_ps_ph(tsrc, row) \
+ __builtin_ia32_acev1_tcvtrowps2phh((tsrc), (row))
/// Read a row from a tile register and convert its FP32 elements to FP16,
/// placing the results in the low half of each destination dword.
@@ -469,7 +473,8 @@ _bsr0_extractl(void) {
/// Row index selecting the tile row to read.
/// \returns
/// ZMM vector (__m512h) holding the converted FP16 elements.
-#define _tile_cvtrowl_ps_ph(tsrc, row) _tile_cvtrowps2phl((tsrc), (row))
+#define _tile_ace_cvtrowl_ps_ph(tsrc, row) \
+ __builtin_ia32_acev1_tcvtrowps2phl((tsrc), (row))
/// Extract one row of a tile register into a ZMM destination.
///
@@ -483,7 +488,8 @@ _bsr0_extractl(void) {
/// Row index selecting the tile row to read.
/// \returns
/// ZMM vector (__m512i) holding the extracted row.
-#define _tile_extractrow(tsrc, row) _tile_movrow((tsrc), (row))
+#define _tile_ace_extractrow(tsrc, row) \
+ ((__m512i)__builtin_ia32_acev1_tilemovrowextract((tsrc), (row)))
/// ACE tile type with fixed dimensions (16 rows x 64 bytes = 1024 bytes).
/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1.
@@ -516,7 +522,7 @@ __ace_init_config(__ace_tile_config *cfg) {
/// Pointer to __acetile variable to be zeroed.
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_zero(__acetile *dst) {
- *dst = __builtin_ia32_tilezero_internal(16, 64);
+ *dst = __builtin_ia32_acev1_tilezero_internal(16, 64);
}
/// Compute 4-way outer product of unsigned x unsigned bytes to INT32.
@@ -778,7 +784,8 @@ static __inline__ void __tile_ace_setrow(__acetile *dst, __m512i src,
/// ZMM vector (__m512i) containing the 64-byte row.
__DEFAULT_FN_ATTRS_ACE
static __inline__ __m512i __tile_ace_getrow(__acetile *src, unsigned int idx) {
- return (__m512i)__builtin_ia32_tilemovrow_internal(16, 64, *src, idx);
+ return (__m512i)__builtin_ia32_acev1_tilemovrowextract_internal(
+ 16, 64, *src, idx);
}
/// Read a row from an ACE tile and convert INT32 elements to FP32.
@@ -798,7 +805,7 @@ static __inline__ __m512i __tile_ace_getrow(__acetile *src, unsigned int idx) {
__DEFAULT_FN_ATTRS_ACE
static __inline__ __m512 __tile_ace_cvtrowd2ps(__acetile *src,
unsigned int idx) {
- return __builtin_ia32_tcvtrowd2ps_internal(16, 64, *src, idx);
+ return __builtin_ia32_acev1_tcvtrowd2ps_internal(16, 64, *src, idx);
}
/// Read a row from an ACE tile and convert FP32 elements to BF16 (high).
@@ -818,7 +825,7 @@ static __inline__ __m512 __tile_ace_cvtrowd2ps(__acetile *src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ __m512bh __tile_ace_cvtrowps2bf16h(__acetile *src,
unsigned int idx) {
- return __builtin_ia32_tcvtrowps2bf16h_internal(16, 64, *src, idx);
+ return __builtin_ia32_acev1_tcvtrowps2bf16h_internal(16, 64, *src, idx);
}
/// Read a row from an ACE tile and convert FP32 elements to BF16 (low).
@@ -838,7 +845,7 @@ static __inline__ __m512bh __tile_ace_cvtrowps2bf16h(__acetile *src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ __m512bh __tile_ace_cvtrowps2bf16l(__acetile *src,
unsigned int idx) {
- return __builtin_ia32_tcvtrowps2bf16l_internal(16, 64, *src, idx);
+ return __builtin_ia32_acev1_tcvtrowps2bf16l_internal(16, 64, *src, idx);
}
/// Read a row from an ACE tile and convert FP32 elements to FP16 (high).
@@ -858,7 +865,7 @@ static __inline__ __m512bh __tile_ace_cvtrowps2bf16l(__acetile *src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ __m512h __tile_ace_cvtrowps2phh(__acetile *src,
unsigned int idx) {
- return __builtin_ia32_tcvtrowps2phh_internal(16, 64, *src, idx);
+ return __builtin_ia32_acev1_tcvtrowps2phh_internal(16, 64, *src, idx);
}
/// Read a row from an ACE tile and convert FP32 elements to FP16 (low).
@@ -878,7 +885,7 @@ static __inline__ __m512h __tile_ace_cvtrowps2phh(__acetile *src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ __m512h __tile_ace_cvtrowps2phl(__acetile *src,
unsigned int idx) {
- return __builtin_ia32_tcvtrowps2phl_internal(16, 64, *src, idx);
+ return __builtin_ia32_acev1_tcvtrowps2phl_internal(16, 64, *src, idx);
}
// clang-format on
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index e7c7d55b2cc4fa..647284258ff565 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -554,6 +554,14 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tcvtrowps2phl:
case X86::BI__builtin_ia32_tcvtrowd2ps:
case X86::BI__builtin_ia32_tilemovrow:
+ // ACE spellings of the same instructions; the tile ID rule is unchanged.
+ case X86::BI__builtin_ia32_acev1_tilezero:
+ case X86::BI__builtin_ia32_acev1_tcvtrowps2bf16h:
+ case X86::BI__builtin_ia32_acev1_tcvtrowps2bf16l:
+ case X86::BI__builtin_ia32_acev1_tcvtrowps2phh:
+ case X86::BI__builtin_ia32_acev1_tcvtrowps2phl:
+ case X86::BI__builtin_ia32_acev1_tcvtrowd2ps:
+ case X86::BI__builtin_ia32_acev1_tilemovrowextract:
// The ACE tile inserts and outer products name only the destination tile;
// their remaining sources are ZMM values or a row/column index, so neither
// the range nor the duplicate rule applies there.
diff --git a/clang/test/CodeGen/X86/acev1.c b/clang/test/CodeGen/X86/acev1.c
index 72b54afbc63129..795bdab98bbd88 100644
--- a/clang/test/CodeGen/X86/acev1.c
+++ b/clang/test/CodeGen/X86/acev1.c
@@ -42,7 +42,7 @@ void test_acev1_tile_config(void *data) {
// CHECK: call void @llvm.x86.ldtilecfg(ptr %{{.*}})
// CHECK: call void @llvm.x86.sttilecfg(ptr %{{.*}})
// CHECK: call void @llvm.x86.tilerelease()
- // CHECK: call void @llvm.x86.tilezero(i8 0)
+ // CHECK: call void @llvm.x86.acev1.tilezero(i8 0)
_tile_ace_loadconfig(data);
_tile_ace_storeconfig(data);
_tile_ace_release();
@@ -59,6 +59,25 @@ void test_acev1_tile_movement(__m512i src) {
_tile_insertcol(1, src, 3);
}
+// The read direction is shared with AMX, so only TILEMOVROW has a separate ACE
+// name; the converting reads still lower to the AMX-spelled intrinsics.
+void test_acev1_row_reads(__m512 *ps, __m512bh *pbh, __m512h *ph,
+ __m512i *pi) {
+ // CHECK-LABEL: @test_acev1_row_reads
+ // CHECK: call <16 x float> @llvm.x86.acev1.tcvtrowd2ps(i8 0, i32 1)
+ // CHECK: call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h(i8 1, i32 2)
+ // CHECK: call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l(i8 2, i32 3)
+ // CHECK: call <32 x half> @llvm.x86.acev1.tcvtrowps2phh(i8 3, i32 4)
+ // CHECK: call <32 x half> @llvm.x86.acev1.tcvtrowps2phl(i8 4, i32 5)
+ // CHECK: call <16 x i32> @llvm.x86.acev1.tilemovrowextract(i8 5, i32 6)
+ ps[0] = _tile_ace_cvtrow_epi32_ps(0, 1);
+ pbh[0] = _tile_ace_cvtrowh_ps_pbh(1, 2);
+ pbh[1] = _tile_ace_cvtrowl_ps_pbh(2, 3);
+ ph[0] = _tile_ace_cvtrowh_ps_ph(3, 4);
+ ph[1] = _tile_ace_cvtrowl_ps_ph(4, 5);
+ pi[0] = _tile_ace_extractrow(5, 6);
+}
+
void test_acev1_bsr(void) {
// CHECK-LABEL: @test_acev1_bsr
// CHECK: call void @llvm.x86.acev1.bsr0init()
diff --git a/clang/test/CodeGen/X86/acev1_api.c b/clang/test/CodeGen/X86/acev1_api.c
index 0ffc5a58b22f7d..e5dfa7b605a1d8 100644
--- a/clang/test/CodeGen/X86/acev1_api.c
+++ b/clang/test/CodeGen/X86/acev1_api.c
@@ -10,7 +10,7 @@
// Test __acetile zero
void test_tile_ace_zero(__acetile *dst) {
// CHECK-LABEL: @test_tile_ace_zero
- // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilezero.internal
// CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_zero(dst);
}
@@ -122,49 +122,49 @@ void test_tile_ace_setcol(__acetile *dst, __m512i src) {
// Test tile getrow (ACE-specific: tile row to ZMM)
__m512i test_tile_ace_getrow(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_getrow
- // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal
return __tile_ace_getrow(src, 0);
}
// Test tile row read with INT32 to FP32 conversion (ACE-specific)
__m512 test_tile_ace_cvtrowd2ps(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowd2ps
- // CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ // CHECK: call <16 x float> @llvm.x86.acev1.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
return __tile_ace_cvtrowd2ps(src, 2);
}
// Test tile row read with FP32 to BF16 conversion, high half of each dword
__m512bh test_tile_ace_cvtrowps2bf16h(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2bf16h
- // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ // CHECK: call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
return __tile_ace_cvtrowps2bf16h(src, 4);
}
// Test tile row read with FP32 to BF16 conversion, low half of each dword
__m512bh test_tile_ace_cvtrowps2bf16l(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2bf16l
- // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ // CHECK: call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
return __tile_ace_cvtrowps2bf16l(src, 6);
}
// Test tile row read with FP32 to FP16 conversion, high half of each dword
__m512h test_tile_ace_cvtrowps2phh(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2phh
- // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ // CHECK: call <32 x half> @llvm.x86.acev1.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
return __tile_ace_cvtrowps2phh(src, 8);
}
// Test tile row read with FP32 to FP16 conversion, low half of each dword
__m512h test_tile_ace_cvtrowps2phl(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_cvtrowps2phl
- // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ // CHECK: call <32 x half> @llvm.x86.acev1.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
return __tile_ace_cvtrowps2phl(src, 10);
}
// Integration test: ACE computation workflow
void test_ace_workflow(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_ace_workflow
- // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilezero.internal
// CHECK: call x86_amx @llvm.x86.acev1.top4buud.internal
__acetile acc;
__tile_ace_zero(&acc);
diff --git a/clang/test/CodeGen/X86/acev1_errors.c b/clang/test/CodeGen/X86/acev1_errors.c
index 31f4cb083d124b..baf50dd8678728 100644
--- a/clang/test/CodeGen/X86/acev1_errors.c
+++ b/clang/test/CodeGen/X86/acev1_errors.c
@@ -32,17 +32,17 @@ void test_tile_range_errors(__m512bh bf, __m512i i) {
_tile_op4mxbss_ps(8, i, i, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
// Row read tile index errors
- _tile_cvtrow_epi32_ps(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_cvtrowh_ps_pbh(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_cvtrowl_ps_pbh(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_cvtrowh_ps_ph(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_cvtrowl_ps_ph(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_extractrow(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_cvtrow_epi32_ps(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_cvtrowh_ps_pbh(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_cvtrowl_ps_pbh(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_cvtrowh_ps_ph(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_cvtrowl_ps_ph(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_extractrow(8, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
}
void test_tile_id_constness(__m512bh bf, __m512i i) {
// The tile index must be an integer constant expression.
- _tile_ace_zero(TILE_ID); // expected-error {{argument to '__builtin_ia32_tilezero' must be a constant integer}}
+ _tile_ace_zero(TILE_ID); // expected-error {{argument to '__builtin_ia32_acev1_tilezero' must be a constant integer}}
_tile_insertcol(TILE_ID, i, 0); // expected-error {{argument to '__builtin_ia32_tilemovcolinsert' must be a constant integer}}
_tile_insertrow(TILE_ID, i, 0); // expected-error {{argument to '__builtin_ia32_tilemovrowinsert' must be a constant integer}}
@@ -59,12 +59,12 @@ void test_tile_id_constness(__m512bh bf, __m512i i) {
_tile_op4mxbf8_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxbf8ps' must be a constant integer}}
_tile_op4mxbss_ps(TILE_ID, i, i, 0); // expected-error {{argument to '__builtin_ia32_top4mxbssps' must be a constant integer}}
- _tile_cvtrow_epi32_ps(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowd2ps' must be a constant integer}}
- _tile_cvtrowh_ps_pbh(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2bf16h' must be a constant integer}}
- _tile_cvtrowl_ps_pbh(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2bf16l' must be a constant integer}}
- _tile_cvtrowh_ps_ph(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2phh' must be a constant integer}}
- _tile_cvtrowl_ps_ph(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tcvtrowps2phl' must be a constant integer}}
- _tile_extractrow(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_tilemovrow' must be a constant integer}}
+ _tile_ace_cvtrow_epi32_ps(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_acev1_tcvtrowd2ps' must be a constant integer}}
+ _tile_ace_cvtrowh_ps_pbh(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_acev1_tcvtrowps2bf16h' must be a constant integer}}
+ _tile_ace_cvtrowl_ps_pbh(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_acev1_tcvtrowps2bf16l' must be a constant integer}}
+ _tile_ace_cvtrowh_ps_ph(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_acev1_tcvtrowps2phh' must be a constant integer}}
+ _tile_ace_cvtrowl_ps_ph(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_acev1_tcvtrowps2phl' must be a constant integer}}
+ _tile_ace_extractrow(TILE_ID, 0); // expected-error {{argument to '__builtin_ia32_acev1_tilemovrowextract' must be a constant integer}}
}
void test_scale_group_errors(__m512i i) {
diff --git a/clang/test/CodeGen/X86/acev1_feature_errors.c b/clang/test/CodeGen/X86/acev1_feature_errors.c
index d3767e0ecf898a..3cb497fcee114d 100644
--- a/clang/test/CodeGen/X86/acev1_feature_errors.c
+++ b/clang/test/CodeGen/X86/acev1_feature_errors.c
@@ -16,9 +16,9 @@ void test_needs_acev1(__m512bh bf, __m512i i) {
_tile_op2bf16_ps(0, bf, bf); // nofeature-error {{'__builtin_ia32_top2bf16ps' needs target feature acev1}}
_tile_op4mxbf8_ps(0, i, i, 0); // nofeature-error {{'__builtin_ia32_top4mxbf8ps' needs target feature acev1}}
- _tile_ace_zero(0); // nofeature-error {{'__builtin_ia32_tilezero' needs target feature amx-tile|acev1}}
- _tile_cvtrow_epi32_ps(0, 0); // nofeature-error {{'__builtin_ia32_tcvtrowd2ps' needs target feature amx-avx512,avx10.2|acev1}}
- _tile_extractrow(0, 0); // nofeature-error {{'__builtin_ia32_tilemovrow' needs target feature amx-avx512,avx10.2|acev1}}
+ _tile_ace_zero(0); // nofeature-error {{'__builtin_ia32_acev1_tilezero' needs target feature acev1}}
+ _tile_ace_cvtrow_epi32_ps(0, 0); // nofeature-error {{'__builtin_ia32_acev1_tcvtrowd2ps' needs target feature acev1}}
+ _tile_ace_extractrow(0, 0); // nofeature-error {{'__builtin_ia32_acev1_tilemovrowextract' needs target feature acev1}}
_tile_ace_loadconfig(&i); // nofeature-error {{always_inline function '_tile_ace_loadconfig' requires target feature 'acev1'}}
_tile_ace_storeconfig(&i); // nofeature-error {{always_inline function '_tile_ace_storeconfig' requires target feature 'acev1'}}
diff --git a/clang/test/CodeGen/X86/acev1_tile_movement.c b/clang/test/CodeGen/X86/acev1_tile_movement.c
index 404e5e3f72f44e..86d1d16b412077 100644
--- a/clang/test/CodeGen/X86/acev1_tile_movement.c
+++ b/clang/test/CodeGen/X86/acev1_tile_movement.c
@@ -25,14 +25,14 @@ void test_tile_ace_setcol(__acetile *dst, __m512i src) {
// Test tile getrow - read tile row to ZMM (ACE-specific)
__m512i test_tile_ace_getrow(__acetile *src) {
// CHECK-LABEL: @test_tile_ace_getrow
- // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal
return __tile_ace_getrow(src, 0);
}
// Test combined tile operations
void test_tile_movement_combined(void) {
// CHECK-LABEL: @test_tile_movement_combined
- // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilezero.internal
// CHECK: call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal
// CHECK: call x86_amx @llvm.x86.acev1.tilemovcolinsert.internal
__acetile tile;
@@ -46,10 +46,10 @@ void test_tile_movement_combined(void) {
// Integration test: complete ACE tile workflow with data movement
void test_ace_data_workflow(__m512i *input, __m512i *output) {
// CHECK-LABEL: @test_ace_data_workflow
- // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.acev1.tilezero.internal
// CHECK: call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal
// CHECK: call x86_amx @llvm.x86.acev1.top4buud.internal
- // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal
__acetile tile;
__tile_ace_zero(&tile);
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index ee5dc56aae8810..3ccebdb393ed14 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5740,6 +5740,72 @@ let TargetPrefix = "x86" in {
// ACE - AMD AI Compute Extensions
let TargetPrefix = "x86" in {
+ def int_x86_acev1_tilemovrowextract : ClangBuiltin<"__builtin_ia32_acev1_tilemovrowextract">,
+ DefaultAttrsIntrinsic<[llvm_v16i32_ty], [llvm_i8_ty, llvm_i32_ty],
+ [IntrRead<[X86TileState]>, IntrReadOnly,
+ ImmArg<ArgIndex<0>>]>;
+
+ def int_x86_acev1_tilemovrowextract_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tilemovrowextract_internal">,
+ Intrinsic<[llvm_v16i32_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty],
+ []>;
+
+ // ACE - spellings of AMX-shared instructions; the palette follows the
+ // opcode.
+ def int_x86_acev1_tilezero : ClangBuiltin<"__builtin_ia32_acev1_tilezero">,
+ DefaultAttrsIntrinsic<[], [llvm_i8_ty],
+ [IntrRead<[X86TileState]>,
+ IntrWrite<[X86TileState]>, ImmArg<ArgIndex<0>>]>;
+ def int_x86_acev1_tilezero_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tilezero_internal">,
+ Intrinsic<[llvm_x86amx_ty], [llvm_i16_ty, llvm_i16_ty], []>;
+ def int_x86_acev1_tcvtrowd2ps : ClangBuiltin<"__builtin_ia32_acev1_tcvtrowd2ps">,
+ DefaultAttrsIntrinsic<[llvm_v16f32_ty], [llvm_i8_ty, llvm_i32_ty],
+ [IntrRead<[X86TileState]>, IntrReadOnly,
+ ImmArg<ArgIndex<0>>]>;
+ def int_x86_acev1_tcvtrowd2ps_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tcvtrowd2ps_internal">,
+ Intrinsic<[llvm_v16f32_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty],
+ []>;
+ def int_x86_acev1_tcvtrowps2bf16h : ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2bf16h">,
+ DefaultAttrsIntrinsic<[llvm_v32bf16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [IntrRead<[X86TileState]>, IntrReadOnly,
+ ImmArg<ArgIndex<0>>]>;
+ def int_x86_acev1_tcvtrowps2bf16h_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2bf16h_internal">,
+ Intrinsic<[llvm_v32bf16_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty],
+ []>;
+ def int_x86_acev1_tcvtrowps2bf16l : ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2bf16l">,
+ DefaultAttrsIntrinsic<[llvm_v32bf16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [IntrRead<[X86TileState]>, IntrReadOnly,
+ ImmArg<ArgIndex<0>>]>;
+ def int_x86_acev1_tcvtrowps2bf16l_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2bf16l_internal">,
+ Intrinsic<[llvm_v32bf16_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty],
+ []>;
+ def int_x86_acev1_tcvtrowps2phh : ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2phh">,
+ DefaultAttrsIntrinsic<[llvm_v32f16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [IntrRead<[X86TileState]>, IntrReadOnly,
+ ImmArg<ArgIndex<0>>]>;
+ def int_x86_acev1_tcvtrowps2phh_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2phh_internal">,
+ Intrinsic<[llvm_v32f16_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty],
+ []>;
+ def int_x86_acev1_tcvtrowps2phl : ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2phl">,
+ DefaultAttrsIntrinsic<[llvm_v32f16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [IntrRead<[X86TileState]>, IntrReadOnly,
+ ImmArg<ArgIndex<0>>]>;
+ def int_x86_acev1_tcvtrowps2phl_internal :
+ ClangBuiltin<"__builtin_ia32_acev1_tcvtrowps2phl_internal">,
+ Intrinsic<[llvm_v32f16_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty],
+ []>;
+
// ACE - block scale register
def int_x86_acev1_bsr0init : ClangBuiltin<"__builtin_ia32_bsr0init">,
Intrinsic<[], [], []>;
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index ddb40dc2d07c32..71c20407e78617 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -617,7 +617,19 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
case X86::PTCVTROWPS2PHLrteV:
case X86::PTCVTROWPS2PHLrtiV:
case X86::PTILEMOVROWrteV:
- case X86::PTILEMOVROWrtiV: {
+ case X86::PTILEMOVROWrtiV:
+ case X86::ACE_PTCVTROWD2PSrteV:
+ case X86::ACE_PTCVTROWD2PSrtiV:
+ case X86::ACE_PTCVTROWPS2BF16HrteV:
+ case X86::ACE_PTCVTROWPS2BF16HrtiV:
+ case X86::ACE_PTCVTROWPS2BF16LrteV:
+ case X86::ACE_PTCVTROWPS2BF16LrtiV:
+ case X86::ACE_PTCVTROWPS2PHHrteV:
+ case X86::ACE_PTCVTROWPS2PHHrtiV:
+ case X86::ACE_PTCVTROWPS2PHLrteV:
+ case X86::ACE_PTCVTROWPS2PHLrtiV:
+ case X86::ACE_PTILEMOVROWrteV:
+ case X86::ACE_PTILEMOVROWrtiV: {
for (unsigned i = 2; i > 0; --i)
MI.removeOperand(i);
unsigned Opc;
@@ -635,39 +647,51 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1);
break;
case X86::PTCVTROWD2PSrteV:
+ case X86::ACE_PTCVTROWD2PSrteV:
Opc = X86::TCVTROWD2PSrte;
break;
case X86::PTCVTROWD2PSrtiV:
+ case X86::ACE_PTCVTROWD2PSrtiV:
Opc = X86::TCVTROWD2PSrti;
break;
case X86::PTCVTROWPS2BF16HrteV:
+ case X86::ACE_PTCVTROWPS2BF16HrteV:
Opc = X86::TCVTROWPS2BF16Hrte;
break;
case X86::PTCVTROWPS2BF16HrtiV:
+ case X86::ACE_PTCVTROWPS2BF16HrtiV:
Opc = X86::TCVTROWPS2BF16Hrti;
break;
case X86::PTCVTROWPS2BF16LrteV:
+ case X86::ACE_PTCVTROWPS2BF16LrteV:
Opc = X86::TCVTROWPS2BF16Lrte;
break;
case X86::PTCVTROWPS2BF16LrtiV:
+ case X86::ACE_PTCVTROWPS2BF16LrtiV:
Opc = X86::TCVTROWPS2BF16Lrti;
break;
case X86::PTCVTROWPS2PHHrteV:
+ case X86::ACE_PTCVTROWPS2PHHrteV:
Opc = X86::TCVTROWPS2PHHrte;
break;
case X86::PTCVTROWPS2PHHrtiV:
+ case X86::ACE_PTCVTROWPS2PHHrtiV:
Opc = X86::TCVTROWPS2PHHrti;
break;
case X86::PTCVTROWPS2PHLrteV:
+ case X86::ACE_PTCVTROWPS2PHLrteV:
Opc = X86::TCVTROWPS2PHLrte;
break;
case X86::PTCVTROWPS2PHLrtiV:
+ case X86::ACE_PTCVTROWPS2PHLrtiV:
Opc = X86::TCVTROWPS2PHLrti;
break;
case X86::PTILEMOVROWrteV:
+ case X86::ACE_PTILEMOVROWrteV:
Opc = X86::TILEMOVROWrte;
break;
case X86::PTILEMOVROWrtiV:
+ case X86::ACE_PTILEMOVROWrtiV:
Opc = X86::TILEMOVROWrti;
break;
default:
@@ -797,7 +821,8 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
return true;
}
#undef GET_EGPR_IF_ENABLED
- case X86::PTILEZEROV: {
+ case X86::PTILEZEROV:
+ case X86::ACE_PTILEZEROV: {
for (int i = 2; i > 0; --i) // Remove row, col
MI.removeOperand(i);
MI.setDesc(TII->get(X86::TILEZERO));
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 80cadba2e51c04..0c7ba5527b118d 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -211,7 +211,8 @@ void X86FastPreTileConfigImpl::InitializeTileConfigStackSpace() {
}
// Fill in the palette first.
// Use Palette 2 for ACE v1, Palette 1 for AMX.
- int PaletteId = ST->hasACEV1() ? 2 : 1;
+ int PaletteId =
+ X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA ? 2 : 1;
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), CfgSS)
.addImm(PaletteId);
}
diff --git a/llvm/lib/Target/X86/X86FastTileConfig.cpp b/llvm/lib/Target/X86/X86FastTileConfig.cpp
index f6ee4313174bb0..be40095bf21647 100644
--- a/llvm/lib/Target/X86/X86FastTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastTileConfig.cpp
@@ -134,7 +134,7 @@ bool X86FastTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
// been initialized to zero in pre config.
// ACE Palette 2 uses fixed tile dimensions (16x64), so skip writing
// shapes - bytes 1-63 must remain zero for ACE.
- if (ST->hasACEV1()) {
+ if (X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA) {
ShapeInfos.clear();
Change = true;
continue;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d9824f4142d22a..4195d8ed57ebd5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -39131,10 +39131,10 @@ X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
namespace {
enum class TileProgModelType {
None, // Not a tile instruction
- ACE_DirectReg, // ACE-only, always ACE_DirectReg
- ACE_ManagedRA, // ACE-only, always ACE_ManagedRA
- AMX_DirectReg, // AMX/ACE, DirectReg (checks subtarget)
- AMX_ManagedRA, // AMX/ACE, ManagedRA (checks subtarget)
+ ACE_DirectReg, // ACE-spelled, sets ACEProgModel to DirectReg
+ ACE_ManagedRA, // ACE-spelled, sets ACEProgModel to ManagedRA
+ AMX_DirectReg, // AMX-spelled, sets AMXProgModel to DirectReg
+ AMX_ManagedRA, // AMX-spelled, sets AMXProgModel to ManagedRA
};
struct TileOpcodeModelEntry {
@@ -39144,7 +39144,7 @@ struct TileOpcodeModelEntry {
// Centralized table mapping tile opcodes to their program model type
static const TileOpcodeModelEntry TileOpcodeModels[] = {
- // AMX/ACE DirectReg (macro API) - checks subtarget for ACE vs AMX
+ // AMX DirectReg (macro API)
{X86::PTILEZERO, TileProgModelType::AMX_DirectReg},
{X86::PTCVTROWPS2BF16Hrti, TileProgModelType::AMX_DirectReg},
{X86::PTCVTROWPS2BF16Lrti, TileProgModelType::AMX_DirectReg},
@@ -39159,10 +39159,23 @@ static const TileOpcodeModelEntry TileOpcodeModels[] = {
{X86::PTCVTROWD2PSrte, TileProgModelType::AMX_DirectReg},
{X86::PTILEMOVROWrte, TileProgModelType::AMX_DirectReg},
- // AMX/ACE ManagedRA (struct API) - checks subtarget for ACE vs AMX
+ // AMX ManagedRA (struct API)
{X86::PTILEZEROV, TileProgModelType::AMX_ManagedRA},
- // ACE-only DirectReg (macro API)
+ // ACE DirectReg (macro API)
+ {X86::ACE_PTILEZERO, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2BF16Hrte, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2BF16Lrte, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2PHHrte, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2PHLrte, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWD2PSrte, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTILEMOVROWrte, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWD2PSrti, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2BF16Hrti, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2BF16Lrti, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2PHHrti, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTCVTROWPS2PHLrti, TileProgModelType::ACE_DirectReg},
+ {X86::ACE_PTILEMOVROWrti, TileProgModelType::ACE_DirectReg},
{X86::PTILEMOVCOLtri, TileProgModelType::ACE_DirectReg},
{X86::PTILEMOVCOLtre, TileProgModelType::ACE_DirectReg},
{X86::PTILEMOVROWtri, TileProgModelType::ACE_DirectReg},
@@ -39178,7 +39191,20 @@ static const TileOpcodeModelEntry TileOpcodeModels[] = {
{X86::PTOP4MXBF8PStrri, TileProgModelType::ACE_DirectReg},
{X86::PTOP4MXBSSPStrri, TileProgModelType::ACE_DirectReg},
- // ACE-only ManagedRA (struct API)
+ // ACE ManagedRA (struct API)
+ {X86::ACE_PTILEZEROV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2BF16HrtiV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2BF16LrtiV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2PHHrtiV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2PHLrtiV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWD2PSrtiV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTILEMOVROWrtiV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2BF16HrteV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2BF16LrteV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2PHHrteV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWPS2PHLrteV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTCVTROWD2PSrteV, TileProgModelType::ACE_ManagedRA},
+ {X86::ACE_PTILEMOVROWrteV, TileProgModelType::ACE_ManagedRA},
{X86::PTILEMOVCOLtreV, TileProgModelType::ACE_ManagedRA},
{X86::PTILEMOVROWtreV, TileProgModelType::ACE_ManagedRA},
{X86::PTOP2BF16PStrrV, TileProgModelType::ACE_ManagedRA},
@@ -39195,8 +39221,7 @@ static const TileOpcodeModelEntry TileOpcodeModels[] = {
// Set tile program model based on opcode. Returns true if model was set.
static bool setTileProgModelForOpcode(unsigned Opcode,
- X86MachineFunctionInfo *MFI,
- const X86Subtarget &Subtarget) {
+ X86MachineFunctionInfo *MFI) {
for (const auto &Entry : TileOpcodeModels) {
if (Entry.Opcode == Opcode) {
switch (Entry.ModelType) {
@@ -39209,16 +39234,10 @@ static bool setTileProgModelForOpcode(unsigned Opcode,
MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
return true;
case TileProgModelType::AMX_DirectReg:
- if (Subtarget.hasACEV1())
- MFI->setACEProgModel(ACEProgModelEnum::ACE_DirectReg);
- else
- MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
+ MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
return true;
case TileProgModelType::AMX_ManagedRA:
- if (Subtarget.hasACEV1())
- MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
- else
- MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
return true;
}
}
@@ -39236,7 +39255,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
// Centralized tile program model handling
auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
- setTileProgModelForOpcode(MI.getOpcode(), MFI, Subtarget);
+ setTileProgModelForOpcode(MI.getOpcode(), MFI);
auto TMMImmToTMMReg = [](unsigned Imm) {
assert (Imm < 8 && "Illegal tmm index");
@@ -39656,7 +39675,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MI.eraseFromParent(); // The pseudo is gone now.
return BB;
}
- case X86::PTILEZERO: {
+ case X86::PTILEZERO:
+ case X86::ACE_PTILEZERO: {
unsigned Imm = MI.getOperand(0).getImm();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
MI.eraseFromParent();
@@ -39675,6 +39695,19 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTOP4MXHBF8PStrriV:
case X86::PTOP4MXBF8PStrriV:
case X86::PTOP4MXBSSPStrriV:
+ case X86::ACE_PTILEZEROV:
+ case X86::ACE_PTCVTROWPS2BF16HrtiV:
+ case X86::ACE_PTCVTROWPS2BF16LrtiV:
+ case X86::ACE_PTCVTROWPS2PHHrtiV:
+ case X86::ACE_PTCVTROWPS2PHLrtiV:
+ case X86::ACE_PTCVTROWD2PSrtiV:
+ case X86::ACE_PTILEMOVROWrtiV:
+ case X86::ACE_PTCVTROWPS2BF16HrteV:
+ case X86::ACE_PTCVTROWPS2BF16LrteV:
+ case X86::ACE_PTCVTROWPS2PHHrteV:
+ case X86::ACE_PTCVTROWPS2PHLrteV:
+ case X86::ACE_PTCVTROWD2PSrteV:
+ case X86::ACE_PTILEMOVROWrteV:
return BB;
case X86::PTILELOADDRS:
case X86::PTILELOADDRST1:
@@ -39727,28 +39760,40 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTCVTROWPS2PHHrti:
case X86::PTCVTROWPS2PHLrti:
case X86::PTCVTROWD2PSrti:
- case X86::PTILEMOVROWrti: {
+ case X86::PTILEMOVROWrti:
+ case X86::ACE_PTCVTROWPS2BF16Hrti:
+ case X86::ACE_PTCVTROWPS2BF16Lrti:
+ case X86::ACE_PTCVTROWPS2PHHrti:
+ case X86::ACE_PTCVTROWPS2PHLrti:
+ case X86::ACE_PTCVTROWD2PSrti:
+ case X86::ACE_PTILEMOVROWrti: {
const DebugLoc &DL = MI.getDebugLoc();
unsigned Opc;
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected instruction!");
case X86::PTCVTROWD2PSrti:
+ case X86::ACE_PTCVTROWD2PSrti:
Opc = X86::TCVTROWD2PSrti;
break;
case X86::PTCVTROWPS2BF16Hrti:
+ case X86::ACE_PTCVTROWPS2BF16Hrti:
Opc = X86::TCVTROWPS2BF16Hrti;
break;
case X86::PTCVTROWPS2PHHrti:
+ case X86::ACE_PTCVTROWPS2PHHrti:
Opc = X86::TCVTROWPS2PHHrti;
break;
case X86::PTCVTROWPS2BF16Lrti:
+ case X86::ACE_PTCVTROWPS2BF16Lrti:
Opc = X86::TCVTROWPS2BF16Lrti;
break;
case X86::PTCVTROWPS2PHLrti:
+ case X86::ACE_PTCVTROWPS2PHLrti:
Opc = X86::TCVTROWPS2PHLrti;
break;
case X86::PTILEMOVROWrti:
+ case X86::ACE_PTILEMOVROWrti:
Opc = X86::TILEMOVROWrti;
break;
}
@@ -39765,28 +39810,40 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTCVTROWPS2PHHrte:
case X86::PTCVTROWPS2PHLrte:
case X86::PTCVTROWD2PSrte:
- case X86::PTILEMOVROWrte: {
+ case X86::PTILEMOVROWrte:
+ case X86::ACE_PTCVTROWPS2BF16Hrte:
+ case X86::ACE_PTCVTROWPS2BF16Lrte:
+ case X86::ACE_PTCVTROWPS2PHHrte:
+ case X86::ACE_PTCVTROWPS2PHLrte:
+ case X86::ACE_PTCVTROWD2PSrte:
+ case X86::ACE_PTILEMOVROWrte: {
const DebugLoc &DL = MI.getDebugLoc();
unsigned Opc;
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected instruction!");
case X86::PTCVTROWD2PSrte:
+ case X86::ACE_PTCVTROWD2PSrte:
Opc = X86::TCVTROWD2PSrte;
break;
case X86::PTCVTROWPS2BF16Hrte:
+ case X86::ACE_PTCVTROWPS2BF16Hrte:
Opc = X86::TCVTROWPS2BF16Hrte;
break;
case X86::PTCVTROWPS2BF16Lrte:
+ case X86::ACE_PTCVTROWPS2BF16Lrte:
Opc = X86::TCVTROWPS2BF16Lrte;
break;
case X86::PTCVTROWPS2PHHrte:
+ case X86::ACE_PTCVTROWPS2PHHrte:
Opc = X86::TCVTROWPS2PHHrte;
break;
case X86::PTCVTROWPS2PHLrte:
+ case X86::ACE_PTCVTROWPS2PHLrte:
Opc = X86::TCVTROWPS2PHLrte;
break;
case X86::PTILEMOVROWrte:
+ case X86::ACE_PTILEMOVROWrte:
Opc = X86::TILEMOVROWrte;
break;
}
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
index d64d7a053c2c66..b2914d12c05563 100644
--- a/llvm/lib/Target/X86/X86InstrACE.td
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -333,6 +333,102 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
}
+// ACE twin of the AMX tile-zero RA pseudo, so the ACE model reaches MIR.
+let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
+ canFoldAsLoad = 1, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in
+def ACE_PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
+ [(set TILE:$dst, (int_x86_acev1_tilezero_internal
+ GR16:$src1, GR16:$src2))]>;
+
+// ACE twins of the AMX row-read RA pseudos, so the ACE model reaches MIR.
+let isPseudo = true, hasNoSchedulingInfo = 1, SchedRW = [WriteSystem],
+ usesCustomInserter = 1 in {
+ def ACE_PTCVTROWD2PSrtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ i32u6imm:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowd2ps_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ imm:$src4))]>;
+ def ACE_PTCVTROWD2PSrteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowd2ps_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4))]>;
+ def ACE_PTCVTROWPS2BF16HrtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ i32u6imm:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2bf16h_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ imm:$src4))]>;
+ def ACE_PTCVTROWPS2BF16HrteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2bf16h_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4))]>;
+ def ACE_PTCVTROWPS2BF16LrtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ i32u6imm:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2bf16l_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ imm:$src4))]>;
+ def ACE_PTCVTROWPS2BF16LrteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2bf16l_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4))]>;
+ def ACE_PTCVTROWPS2PHHrtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ i32u6imm:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2phh_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ imm:$src4))]>;
+ def ACE_PTCVTROWPS2PHHrteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2phh_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4))]>;
+ def ACE_PTCVTROWPS2PHLrtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ i32u6imm:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2phl_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ imm:$src4))]>;
+ def ACE_PTCVTROWPS2PHLrteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tcvtrowps2phl_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4))]>;
+ def ACE_PTILEMOVROWrtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ i32u6imm:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tilemovrowextract_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ imm:$src4))]>;
+ def ACE_PTILEMOVROWrteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4),
+ [(set VR512:$dst,
+ (int_x86_acev1_tilemovrowextract_internal
+ GR16:$src1, GR16:$src2, TILE:$src3,
+ GR32:$src4))]>;
+}
+
// Pseudo instructions for tile spill and reload (register allocation)
let isPseudo = true, hasNoSchedulingInfo = 1 in {
let mayStore = 1 in
@@ -343,3 +439,39 @@ let isPseudo = true, hasNoSchedulingInfo = 1 in {
}
} // end HasACEV1, In64BitMode
+
+// ACE twins of the user-facing DirectReg pseudos (immediate tile IDs).
+// They expand like the AMX pseudos; the opcode is what records the ACE model.
+class ACERowReadPseudo<Intrinsic intr>
+ : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2),
+ [(set VR512:$dst, (intr timm:$src1, GR32:$src2))]>;
+
+// One ACE intrinsic covers both row index forms, so a constant index matches
+// the immediate encoding here and anything else falls back to the register one.
+class ACERowReadPseudoI<Intrinsic intr>
+ : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
+ [(set VR512:$dst, (intr timm:$src1, imm:$src2))]>;
+
+let Predicates = [HasACEV1, In64BitMode], SchedRW = [WriteSystem],
+ usesCustomInserter = 1 in {
+ def ACE_PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
+ [(int_x86_acev1_tilezero timm:$src)]>;
+
+ def ACE_PTCVTROWD2PSrte : ACERowReadPseudo<int_x86_acev1_tcvtrowd2ps>;
+ def ACE_PTCVTROWPS2BF16Hrte
+ : ACERowReadPseudo<int_x86_acev1_tcvtrowps2bf16h>;
+ def ACE_PTCVTROWPS2BF16Lrte
+ : ACERowReadPseudo<int_x86_acev1_tcvtrowps2bf16l>;
+ def ACE_PTCVTROWPS2PHHrte : ACERowReadPseudo<int_x86_acev1_tcvtrowps2phh>;
+ def ACE_PTCVTROWPS2PHLrte : ACERowReadPseudo<int_x86_acev1_tcvtrowps2phl>;
+ def ACE_PTILEMOVROWrte : ACERowReadPseudo<int_x86_acev1_tilemovrowextract>;
+
+ def ACE_PTCVTROWD2PSrti : ACERowReadPseudoI<int_x86_acev1_tcvtrowd2ps>;
+ def ACE_PTCVTROWPS2BF16Hrti
+ : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2bf16h>;
+ def ACE_PTCVTROWPS2BF16Lrti
+ : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2bf16l>;
+ def ACE_PTCVTROWPS2PHHrti : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2phh>;
+ def ACE_PTCVTROWPS2PHLrti : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2phl>;
+ def ACE_PTILEMOVROWrti : ACERowReadPseudoI<int_x86_acev1_tilemovrowextract>;
+}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 854f016ac8f902..87d557f0c65efa 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -791,6 +791,7 @@ bool X86InstrInfo::isReMaterializableImpl(
case X86::MOV64ri32:
case X86::MOV8ri:
case X86::PTILEZEROV:
+ case X86::ACE_PTILEZEROV:
return true;
case X86::MOV8rm:
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index 7c0d568ec2dca8..4e8fd73b280272 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -124,6 +124,14 @@ static bool isACEOnlyIntrinsic(Instruction *I) {
// ACE tile movement intrinsics
case Intrinsic::x86_acev1_tilemovcolinsert_internal:
case Intrinsic::x86_acev1_tilemovrowinsert_internal:
+ // ACE spellings of the AMX-shared intrinsics
+ case Intrinsic::x86_acev1_tilezero_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2bf16h_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2bf16l_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2phh_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2phl_internal:
+ case Intrinsic::x86_acev1_tcvtrowd2ps_internal:
+ case Intrinsic::x86_acev1_tilemovrowextract_internal:
return true;
default:
return false;
@@ -246,7 +254,13 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
case Intrinsic::x86_tcvtrowps2bf16l_internal:
case Intrinsic::x86_tcvtrowps2phh_internal:
case Intrinsic::x86_tcvtrowps2phl_internal:
- case Intrinsic::x86_tilemovrow_internal: {
+ case Intrinsic::x86_tilemovrow_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2bf16h_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2bf16l_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2phh_internal:
+ case Intrinsic::x86_acev1_tcvtrowps2phl_internal:
+ case Intrinsic::x86_acev1_tcvtrowd2ps_internal:
+ case Intrinsic::x86_acev1_tilemovrowextract_internal: {
assert(OpNo == 2 && "Illegal Operand Number.");
Row = II->getArgOperand(0);
Col = II->getArgOperand(1);
@@ -309,9 +323,11 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
return std::make_pair(Row, Col);
}
-static std::pair<Value *, Value *> getShape(PHINode *Phi) {
+// Walk through casts and PHIs to the intrinsic that will consume the tile,
+// reporting which operand it arrives as.
+static IntrinsicInst *getTileConsumer(PHINode *Phi, unsigned &OpNo) {
Use &U = *(Phi->use_begin());
- unsigned OpNo = U.getOperandNo();
+ OpNo = U.getOperandNo();
User *V = U.getUser();
// TODO We don't traverse all users. To make the algorithm simple, here we
// just traverse the first user. If we can find shape, then return the shape,
@@ -325,7 +341,7 @@ static std::pair<Value *, Value *> getShape(PHINode *Phi) {
OpNo = U.getOperandNo();
V = U.getUser();
} else if (isAMXIntrinsic(V)) {
- return getShape(cast<IntrinsicInst>(V), OpNo);
+ return cast<IntrinsicInst>(V);
} else if (isa<PHINode>(V)) {
if (V->use_empty())
break;
@@ -336,7 +352,16 @@ static std::pair<Value *, Value *> getShape(PHINode *Phi) {
}
}
- return std::make_pair(nullptr, nullptr);
+ return nullptr;
+}
+
+// Which tilezero spelling to synthesize follows from the intrinsic that will
+// consume the tile, not from the subtarget: a function built for an ACE target
+// can still contain plain AMX code, which needs palette 1.
+static Intrinsic::ID getTilezeroIntrinsic(IntrinsicInst *Consumer) {
+ return Consumer && isACEOnlyIntrinsic(Consumer)
+ ? Intrinsic::x86_acev1_tilezero_internal
+ : Intrinsic::x86_tilezero_internal;
}
namespace {
@@ -834,19 +859,9 @@ namespace {
class X86LowerAMXCast {
Function &Func;
std::unique_ptr<DominatorTree> DT;
- bool IsACE; // ACE configures palette 2, which has no TILELOADD/TILESTORED
public:
- X86LowerAMXCast(Function &F, const TargetMachine *TM = nullptr)
- : Func(F), DT(nullptr), IsACE(false) {
- if (TM) {
- // ACE implies AMX-TILE, but ACE code runs under palette 2 where
- // TILELOADD/TILESTORED are unavailable, so key off ACE itself.
- const X86Subtarget *ST =
- static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
- IsACE = ST && ST->hasACEV1();
- }
- }
+ X86LowerAMXCast(Function &F) : Func(F), DT(nullptr) {}
bool combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
bool combineLoadCast(IntrinsicInst *Cast, LoadInst *LD);
bool combineTilezero(IntrinsicInst *Cast);
@@ -925,7 +940,10 @@ bool X86LowerAMXCast::optimizeAMXCastFromPhi(
if (!isa<UndefValue>(IncValue) && !IncConst->isNullValue())
return false;
Value *Row = nullptr, *Col = nullptr;
- std::tie(Row, Col) = getShape(OldPN);
+ unsigned ConsumerOpNo = 0;
+ IntrinsicInst *Consumer = getTileConsumer(OldPN, ConsumerOpNo);
+ if (Consumer)
+ std::tie(Row, Col) = getShape(Consumer, ConsumerOpNo);
// TODO: If it is not constant the Row and Col must domoniate tilezero
// that we are going to create.
if (!Row || !Col || !isa<Constant>(Row) || !isa<Constant>(Col))
@@ -934,7 +952,7 @@ bool X86LowerAMXCast::optimizeAMXCastFromPhi(
auto *Block = OldPN->getIncomingBlock(I);
BasicBlock::iterator Iter = Block->getTerminator()->getIterator();
Instruction *NewInst = Builder.CreateIntrinsicWithoutFolding(
- Intrinsic::x86_tilezero_internal, {}, {Row, Col});
+ getTilezeroIntrinsic(Consumer), {}, {Row, Col});
NewInst->moveBefore(Iter);
NewInst = Builder.CreateIntrinsicWithoutFolding(
Intrinsic::x86_cast_tile_to_vector, {IncValue->getType()},
@@ -1072,9 +1090,9 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
return false;
auto *II = cast<IntrinsicInst>(Tile);
- // ACE v1 doesn't have TILESTORED instruction - skip combining for ACE targets
- // or ACE intrinsics. ACE uses struct-based API with vector load/store.
- if (IsACE || isACEOnlyIntrinsic(II))
+ // ACE v1 doesn't have TILESTORED, so an ACE tile cannot be stored this way.
+ // ACE uses struct-based API with vector load/store.
+ if (isACEOnlyIntrinsic(II))
return false;
// Tile is output from AMX intrinsic. The first operand of the
// intrinsic is row, the second operand of the intrinsic is column.
@@ -1106,9 +1124,9 @@ bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
// shape information through def-use chain.
if (!isAMXIntrinsic(II))
return false;
- // ACE v1 doesn't have TILELOADD instruction - skip combining for ACE targets
- // or ACE intrinsics. ACE uses struct-based API with vector load/store.
- if (IsACE || isACEOnlyIntrinsic(II))
+ // ACE v1 doesn't have TILELOADD, so an ACE tile cannot be loaded this way.
+ // ACE uses struct-based API with vector load/store.
+ if (isACEOnlyIntrinsic(II))
return false;
std::tie(Row, Col) = getShape(II, OpNo);
IRBuilder<> Builder(LD);
@@ -1156,7 +1174,7 @@ bool X86LowerAMXCast::combineTilezero(IntrinsicInst *Cast) {
IRBuilder<> Builder(Cast);
Value *NewInst =
- Builder.CreateIntrinsic(Intrinsic::x86_tilezero_internal, {}, {Row, Col});
+ Builder.CreateIntrinsic(getTilezeroIntrinsic(II), {}, {Row, Col});
Cast->replaceAllUsesWith(NewInst);
return true;
}
@@ -1343,14 +1361,14 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
auto *II = dyn_cast<IntrinsicInst>(U.getUser());
if (!II)
return false; // May be bitcast from x86amx to <256 x i32>.
- // ACE v1 doesn't have TILELOADD - for ACE targets or ACE intrinsics,
- // use tilezero since ACE tiles are typically zero-initialized in struct
- // API.
- if (IsACE || isACEOnlyIntrinsic(II)) {
+ // ACE v1 doesn't have TILELOADD, so an ACE tile cannot be loaded here.
+ // Use tilezero, since ACE tiles are typically zero-initialized in the
+ // struct API.
+ if (isACEOnlyIntrinsic(II)) {
Value *Row = nullptr, *Col = nullptr;
std::tie(Row, Col) = getShape(II, OpNo);
- Value *NewInst = Builder.CreateIntrinsic(Intrinsic::x86_tilezero_internal,
- {}, {Row, Col});
+ Value *NewInst =
+ Builder.CreateIntrinsic(getTilezeroIntrinsic(II), {}, {Row, Col});
AMXCast->replaceAllUsesWith(NewInst);
AMXCast->eraseFromParent();
return true;
@@ -1377,10 +1395,9 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
auto *II = dyn_cast<IntrinsicInst>(Src);
if (!II)
return false; // May be bitcast from <256 x i32> to x86amx.
- // ACE v1 doesn't have TILESTORED - for ACE targets or ACE intrinsics,
- // just create a poison value since the tile result is typically not read
- // back.
- if (IsACE || isACEOnlyIntrinsic(II)) {
+ // ACE v1 doesn't have TILESTORED, so an ACE tile cannot be stored here.
+ // Use poison, since the tile result is typically not read back.
+ if (isACEOnlyIntrinsic(II)) {
Value *Poison = PoisonValue::get(AMXCast->getType());
AMXCast->replaceAllUsesWith(Poison);
AMXCast->eraseFromParent();
@@ -1429,7 +1446,7 @@ bool lowerAmxType(Function &F, const TargetMachine *TM,
return false;
bool C = false;
- X86LowerAMXCast LAC(F, TM);
+ X86LowerAMXCast LAC(F);
C |= LAC.combineAMXcast(TLI);
// There might be remaining AMXcast after combineAMXcast and they should be
// handled elegantly.
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 765b6cb5f56d25..f60179ed236976 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -134,6 +134,18 @@ class X86PreTileConfigImpl {
case X86::PTCVTROWPS2PHLrtiV:
case X86::PTILEMOVROWrteV:
case X86::PTILEMOVROWrtiV:
+ case X86::ACE_PTCVTROWD2PSrteV:
+ case X86::ACE_PTCVTROWD2PSrtiV:
+ case X86::ACE_PTCVTROWPS2BF16HrteV:
+ case X86::ACE_PTCVTROWPS2BF16HrtiV:
+ case X86::ACE_PTCVTROWPS2BF16LrteV:
+ case X86::ACE_PTCVTROWPS2BF16LrtiV:
+ case X86::ACE_PTCVTROWPS2PHHrteV:
+ case X86::ACE_PTCVTROWPS2PHHrtiV:
+ case X86::ACE_PTCVTROWPS2PHLrteV:
+ case X86::ACE_PTCVTROWPS2PHLrtiV:
+ case X86::ACE_PTILEMOVROWrteV:
+ case X86::ACE_PTILEMOVROWrtiV:
return true;
}
@@ -454,7 +466,8 @@ bool X86PreTileConfigImpl::runOnMachineFunction(MachineFunction &MF) {
}
// Fill in the palette first.
// Use Palette 2 for ACE v1, Palette 1 for AMX.
- int PaletteId = ST.hasACEV1() ? 2 : 1;
+ int PaletteId =
+ X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA ? 2 : 1;
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), SS)
.addImm(PaletteId);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 8b66598eb6e477..2ac74a362426c2 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -1137,6 +1137,7 @@ static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
case X86::PTDPBUSDV:
case X86::PTDPBUUDV:
case X86::PTILEZEROV:
+ case X86::ACE_PTILEZEROV:
case X86::PTDPBF16PSV:
case X86::PTDPFP16PSV:
case X86::PTCMMIMFP16PSV:
diff --git a/llvm/lib/Target/X86/X86TileConfig.cpp b/llvm/lib/Target/X86/X86TileConfig.cpp
index 6667c8d1a66875..cd7c592618d38c 100644
--- a/llvm/lib/Target/X86/X86TileConfig.cpp
+++ b/llvm/lib/Target/X86/X86TileConfig.cpp
@@ -140,7 +140,7 @@ static bool tileConfig(MachineFunction &MF,
// Fill in the shape of each tile physical register.
// ACE Palette 2 uses fixed tile dimensions (16x64), so skip writing
// shapes - bytes 1-63 must remain zero for ACE.
- if (ST.hasACEV1())
+ if (X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA)
return true;
for (unsigned I = 0; I < AMXRegNum; ++I) {
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll
new file mode 100644
index 00000000000000..5a042c60d4bd45
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll
@@ -0,0 +1,35 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile,+amx-int8 \
+; RUN: -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile,+amx-int8,+acev1 \
+; RUN: -verify-machineinstrs | FileCheck %s
+
+; Test that a function holding only plain AMX intrinsics lowers the same way
+; whether or not the target also supports ACE v1.
+
+define void @amx_load_store_on_ace_target(ptr %in, ptr %out) nounwind {
+; CHECK-LABEL: amx_load_store_on_ace_target:
+; CHECK: movb $1, {{.*}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tileloadd (%rdi,%rax), %tmm0
+; CHECK-NEXT: tilestored %tmm0, (%rsi,%rax)
+; CHECK: tilerelease
+ %v = load <256 x i32>, ptr %in, align 64
+ %t = call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %v)
+ call void @llvm.x86.tilestored64.internal(i16 16, i16 64, ptr %out, i64 64, x86_amx %t)
+ ret void
+}
+
+define <256 x i32> @amx_tile_to_vector_on_ace_target() nounwind {
+; CHECK-LABEL: amx_tile_to_vector_on_ace_target:
+; CHECK: movb $1, {{.*}}(%rsp)
+; CHECK: tilezero %tmm0
+; CHECK: tilestored %tmm0, (%rdi,%rsi)
+ %t = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %v = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %t)
+ ret <256 x i32> %v
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx)
+declare x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32>)
+declare <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-direct-prog-model.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-direct-prog-model.ll
new file mode 100644
index 00000000000000..3218563c69b1a6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-direct-prog-model.ll
@@ -0,0 +1,49 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile \
+; RUN: -stop-after=finalize-isel -o - | FileCheck %s
+;
+; The programming model is recorded per function and per extension, and is read
+; off the pseudo opcode during ISel. The ACE DirectReg intrinsics used to select
+; the AMX pseudos, which recorded them against the AMX model and left a function
+; containing nothing but ACE looking like an AMX one.
+
+; Every ACE DirectReg intrinsic that shares a real instruction with AMX, in one
+; function: a single one of them selecting an AMX pseudo would show up as an AMX
+; model here.
+define void @ace_direct(i32 %i, ptr %out) {
+; CHECK-LABEL: name: ace_direct
+; CHECK: amxProgModel: None
+; CHECK: aceProgModel: DirectReg
+ call void @llvm.x86.acev1.tilezero(i8 1)
+ %a = call <16 x float> @llvm.x86.acev1.tcvtrowd2ps(i8 1, i32 %i)
+ store volatile <16 x float> %a, ptr %out, align 64
+ %b = call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h(i8 1, i32 %i)
+ store volatile <32 x bfloat> %b, ptr %out, align 64
+ %c = call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l(i8 1, i32 %i)
+ store volatile <32 x bfloat> %c, ptr %out, align 64
+ %d = call <32 x half> @llvm.x86.acev1.tcvtrowps2phh(i8 1, i32 %i)
+ store volatile <32 x half> %d, ptr %out, align 64
+ %e = call <32 x half> @llvm.x86.acev1.tcvtrowps2phl(i8 1, i32 %i)
+ store volatile <32 x half> %e, ptr %out, align 64
+ %f = call <16 x i32> @llvm.x86.acev1.tilemovrowextract(i8 1, i32 %i)
+ store volatile <16 x i32> %f, ptr %out, align 64
+ ret void
+}
+
+; The AMX spelling of a shared instruction still belongs to AMX. Same target,
+; so only the intrinsic can tell these two functions apart.
+define void @amx_direct() {
+; CHECK-LABEL: name: amx_direct
+; CHECK: amxProgModel: DirectReg
+; CHECK: aceProgModel: None
+ call void @llvm.x86.tilezero(i8 1)
+ ret void
+}
+
+declare void @llvm.x86.acev1.tilezero(i8)
+declare <16 x float> @llvm.x86.acev1.tcvtrowd2ps(i8, i32)
+declare <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h(i8, i32)
+declare <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l(i8, i32)
+declare <32 x half> @llvm.x86.acev1.tcvtrowps2phh(i8, i32)
+declare <32 x half> @llvm.x86.acev1.tcvtrowps2phl(i8, i32)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract(i8, i32)
+declare void @llvm.x86.tilezero(i8)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
index e3fc8db107404d..513e75df28b303 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
@@ -13,12 +13,12 @@ define void @test_acev1_ra(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; CHECK: PTOP2BF16PStrrV
; CHECK: PTILEMOVROWrtiV
entry:
- %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract using TILEMOVROW (ACE v1 pattern)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
@@ -32,14 +32,14 @@ define void @test_acev1_ra_multi(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %
; CHECK: PTILEZEROV
; CHECK: PTOP2BF16PStrrV
entry:
- %c1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %c2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c1 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %c2 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d1 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
%d2 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c2, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d2, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
@@ -53,13 +53,13 @@ define void @test_acev1_ra_int(ptr %out, <64 x i8> %a_i8) nounwind {
; CHECK: PTOP4BSSDtrrV
; CHECK: PTOP4BUUDtrrV
entry:
- %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d1 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
%d2 = tail call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
; Extract using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d2, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
@@ -70,7 +70,7 @@ define void @test_acev1_ra_branch(ptr %out, i32 %cond, <32 x bfloat> %zmm_a, <32
; CHECK-LABEL: name: test_acev1_ra_branch
; CHECK: PLDTILECFGV
entry:
- %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%icmp = icmp eq i32 %cond, 0
br i1 %icmp, label %then, label %else
@@ -87,14 +87,14 @@ merge:
%result = phi x86_amx [ %d1, %then ], [ %d2, %else ]
; Extract using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %result, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %result, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
-declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
declare x86_amx @llvm.x86.acev1.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
index 621b1e58fd1ba3..6983476f3ec6ec 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
@@ -336,6 +336,82 @@ define void @test_tilemovrowinsert_reg(<16 x i32> %src, i32 %idx) {
ret void
}
+; Test TILEMOVROW reading a row out of a tile. One intrinsic covers both row
+; index forms, so a constant index picks up the immediate encoding.
+define <16 x i32> @test_tilemovrowextract(i32 %idx) {
+; CHECK-LABEL: test_tilemovrowextract:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow %edi, %tmm2, %zmm0 # encoding: [0x62,0xf2,0x45,0x48,0x4a,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <16 x i32> @llvm.x86.acev1.tilemovrowextract(i8 2, i32 %idx)
+ ret <16 x i32> %res
+}
+
+define <16 x i32> @test_tilemovrowextract_imm() {
+; CHECK-LABEL: test_tilemovrowextract_imm:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow $5, %tmm2, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x07,0xc2,0x05]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <16 x i32> @llvm.x86.acev1.tilemovrowextract(i8 2, i32 5)
+ ret <16 x i32> %res
+}
+
+; Test TCVTROWD2PS reading a row and converting int32 to FP32. The ACE spelling
+; selects its own pseudo, so the palette follows from the opcode.
+define <16 x float> @test_acev1_tcvtrowd2ps(i32 %idx) {
+; CHECK-LABEL: test_acev1_tcvtrowd2ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tcvtrowd2ps %edi, %tmm2, %zmm0 # encoding: [0x62,0xf2,0x46,0x48,0x4a,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <16 x float> @llvm.x86.acev1.tcvtrowd2ps(i8 2, i32 %idx)
+ ret <16 x float> %res
+}
+
+; The other shared row reads follow the same path: an ACE-spelled intrinsic
+; selecting the shared instruction through an ACE-only pattern.
+define <32 x bfloat> @test_acev1_tcvtrowps2bf16h(i32 %idx) {
+; CHECK-LABEL: test_acev1_tcvtrowps2bf16h:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tcvtrowps2bf16h %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x47,0x48,0x6d,0xc1]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h(i8 1, i32 %idx)
+ ret <32 x bfloat> %res
+}
+
+; The other shared row reads follow the same path: an ACE-spelled intrinsic
+; selecting the shared instruction through an ACE-only pattern.
+define <32 x bfloat> @test_acev1_tcvtrowps2bf16l(i32 %idx) {
+; CHECK-LABEL: test_acev1_tcvtrowps2bf16l:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tcvtrowps2bf16l %edi, %tmm2, %zmm0 # encoding: [0x62,0xf2,0x46,0x48,0x6d,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l(i8 2, i32 %idx)
+ ret <32 x bfloat> %res
+}
+
+; The other shared row reads follow the same path: an ACE-spelled intrinsic
+; selecting the shared instruction through an ACE-only pattern.
+define <32 x half> @test_acev1_tcvtrowps2phh(i32 %idx) {
+; CHECK-LABEL: test_acev1_tcvtrowps2phh:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tcvtrowps2phh %edi, %tmm3, %zmm0 # encoding: [0x62,0xf2,0x44,0x48,0x6d,0xc3]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <32 x half> @llvm.x86.acev1.tcvtrowps2phh(i8 3, i32 %idx)
+ ret <32 x half> %res
+}
+
+; The other shared row reads follow the same path: an ACE-spelled intrinsic
+; selecting the shared instruction through an ACE-only pattern.
+define <32 x half> @test_acev1_tcvtrowps2phl(i32 %idx) {
+; CHECK-LABEL: test_acev1_tcvtrowps2phl:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tcvtrowps2phl %edi, %tmm4, %zmm0 # encoding: [0x62,0xf2,0x45,0x48,0x6d,0xc4]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %res = call <32 x half> @llvm.x86.acev1.tcvtrowps2phl(i8 4, i32 %idx)
+ ret <32 x half> %res
+}
+
+
; Intrinsic declarations
declare void @llvm.x86.acev1.bsr0init()
declare void @llvm.x86.acev1.bsr0movf(<64 x i8>, <64 x i8>)
@@ -345,6 +421,12 @@ declare void @llvm.x86.acev1.bsr0movlinsert(<64 x i8>)
declare <64 x i8> @llvm.x86.acev1.bsr0movlextract()
declare void @llvm.x86.acev1.tilemovcolinsert(i8, <16 x i32>, i32)
declare void @llvm.x86.acev1.tilemovrowinsert(i8, <16 x i32>, i32)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract(i8, i32)
+declare <16 x float> @llvm.x86.acev1.tcvtrowd2ps(i8, i32)
+declare <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h(i8, i32)
+declare <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l(i8, i32)
+declare <32 x half> @llvm.x86.acev1.tcvtrowps2phh(i8, i32)
+declare <32 x half> @llvm.x86.acev1.tcvtrowps2phl(i8, i32)
declare void @llvm.x86.acev1.top2bf16ps(i8, <32 x bfloat>, <32 x bfloat>)
declare void @llvm.x86.acev1.top4buud(i8, <64 x i8>, <64 x i8>)
declare void @llvm.x86.acev1.top4busd(i8, <64 x i8>, <64 x i8>)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
index 353c68d95c1c25..4846ce51d98242 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
@@ -14,11 +14,11 @@ define void @test_top2bf16ps(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_
; CHECK: top2bf16ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract using TILEMOVROW (ACE v1 pattern)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -31,10 +31,10 @@ define void @test_top4buud(ptr %out, <64 x i8> %a_i8) {
; CHECK: top4buud
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -47,10 +47,10 @@ define void @test_top4bssd(ptr %out, <64 x i8> %a_i8) {
; CHECK: top4bssd
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -63,10 +63,10 @@ define void @test_top4busd(ptr %out, <64 x i8> %a_i8) {
; CHECK: top4busd
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -79,10 +79,10 @@ define void @test_top4bsud(ptr %out, <64 x i8> %a_i8) {
; CHECK: top4bsud
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -95,10 +95,10 @@ define void @test_top4mxhf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zm
; CHECK: top4mxhf8ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -111,10 +111,10 @@ define void @test_top4mxbhf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %z
; CHECK: top4mxbhf8ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -127,10 +127,10 @@ define void @test_top4mxhbf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %z
; CHECK: top4mxhbf8ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -143,10 +143,10 @@ define void @test_top4mxbf8ps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zm
; CHECK: top4mxbf8ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -159,10 +159,10 @@ define void @test_top4mxbssps_internal(ptr %out, <64 x i8> %zmm_a, <64 x i8> %zm
; CHECK: top4mxbssps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d = call x86_amx @llvm.x86.acev1.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %c, <64 x i8> %zmm_a, <64 x i8> %zmm_b)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
@@ -177,7 +177,7 @@ define void @test_chained_outer_products(ptr %out, <32 x bfloat> %zmm_bf16, <64
; CHECK: top4buud
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
; BF16 outer product
%d1 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_bf16, <32 x bfloat> %zmm_bf16)
@@ -187,12 +187,12 @@ define void @test_chained_outer_products(ptr %out, <32 x bfloat> %zmm_bf16, <64
%d3 = call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d2, <64 x i8> %a_i8, <64 x i8> %a_i8)
; Extract using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d3, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
-declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
declare x86_amx @llvm.x86.acev1.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
@@ -203,4 +203,4 @@ declare x86_amx @llvm.x86.acev1.top4mxbhf8ps.internal(i16, i16, i16, i8, x86_amx
declare x86_amx @llvm.x86.acev1.top4mxhbf8ps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
declare x86_amx @llvm.x86.acev1.top4mxbf8ps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
declare x86_amx @llvm.x86.acev1.top4mxbssps.internal(i16, i16, i16, i8, x86_amx, <64 x i8>, <64 x i8>)
-declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
index a97adffe41db8e..c66e5110f1d939 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
@@ -25,8 +25,8 @@ define void @extract_one_row(ptr %out) nounwind {
; CHECK: tilezero %tmm{{[0-7]}}
; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
entry:
- %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %r = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ %t = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %r = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %t, i32 0)
store volatile <16 x i32> %r, ptr %out, align 64
ret void
}
@@ -40,10 +40,10 @@ define void @extract_several_rows(ptr %out) nounwind {
; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
entry:
- %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ %t = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %r0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %t, i32 0)
store volatile <16 x i32> %r0, ptr %out, align 64
- %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 7)
+ %r7 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %t, i32 7)
store volatile <16 x i32> %r7, ptr %out, align 64
ret void
}
@@ -56,12 +56,12 @@ define void @extract_row_convert(ptr %out) nounwind {
; CHECK: tilezero %tmm{{[0-7]}}
; CHECK: tcvtrowd2ps ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
entry:
- %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %r = call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ %t = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %r = call <16 x float> @llvm.x86.acev1.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %t, i32 0)
store volatile <16 x float> %r, ptr %out, align 64
ret void
}
-declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
-declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
-declare <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16, i16, x86_amx, i32)
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
+declare <16 x float> @llvm.x86.acev1.tcvtrowd2ps.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
index 32f89f4592356e..95ef68537adafc 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
@@ -13,18 +13,70 @@ define void @test_acev1_basic(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm
; CHECK: top2bf16ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
; ACE BF16 outer product
%d = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract result using TILEMOVROW (ACE v1 pattern - no TILESTORED)
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
+; The ACE spelling of the shared row-read intrinsic selects its own pseudo, so
+; the palette follows from the opcode rather than from the subtarget.
+define void @test_acev1_cvtrowd2ps_internal(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b, i32 %idx) {
+; CHECK-LABEL: test_acev1_cvtrowd2ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: top2bf16ps
+; CHECK: tcvtrowd2ps
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %ps = call <16 x float> @llvm.x86.acev1.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %d, i32 %idx)
+ store volatile <16 x float> %ps, ptr %out, align 64
+ ret void
+}
+
+; A tile zero on its own must still establish the palette: it is the only tile
+; instruction here, so nothing else can set the program model for it.
+define void @test_acev1_tilezero_only(ptr %out) {
+; CHECK-LABEL: test_acev1_tilezero_only:
+; CHECK: movb $2,
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %v = call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %c)
+ store volatile <256 x i32> %v, ptr %out, align 64
+ ret void
+}
+
+; The remaining shared row reads take the same ACE-spelled struct-API path.
+define void @test_acev1_cvtrowps2_internal(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b, i32 %idx) {
+; CHECK-LABEL: test_acev1_cvtrowps2_internal:
+; CHECK: ldtilecfg
+; CHECK: tcvtrowps2bf16h
+; CHECK: tcvtrowps2bf16l
+; CHECK: tcvtrowps2phh
+; CHECK: tcvtrowps2phl
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %r0 = call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx %d, i32 %idx)
+ store volatile <32 x bfloat> %r0, ptr %out, align 64
+ %r1 = call <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx %d, i32 %idx)
+ store volatile <32 x bfloat> %r1, ptr %out, align 64
+ %r2 = call <32 x half> @llvm.x86.acev1.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx %d, i32 %idx)
+ store volatile <32 x half> %r2, ptr %out, align 64
+ %r3 = call <32 x half> @llvm.x86.acev1.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx %d, i32 %idx)
+ store volatile <32 x half> %r3, ptr %out, align 64
+ ret void
+}
+
; Test ACE integer outer products
define void @test_acev1_int_outer_products(ptr %out, <64 x i8> %a_i8, <64 x i8> %b_i8) {
; CHECK-LABEL: test_acev1_int_outer_products:
@@ -34,7 +86,7 @@ define void @test_acev1_int_outer_products(ptr %out, <64 x i8> %a_i8, <64 x i8>
; CHECK-DAG: top4bssd
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
; Unsigned x Unsigned -> DWORD
%d0 = call x86_amx @llvm.x86.acev1.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %b_i8)
@@ -43,7 +95,7 @@ define void @test_acev1_int_outer_products(ptr %out, <64 x i8> %a_i8, <64 x i8>
%d1 = call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d0, <64 x i8> %a_i8, <64 x i8> %b_i8)
; Extract result using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d1, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
@@ -59,7 +111,7 @@ define void @test_acev1_accumulation(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloa
; CHECK: top2bf16ps
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
; Multiple accumulations
%d0 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
@@ -67,7 +119,7 @@ define void @test_acev1_accumulation(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloa
%d2 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
; Extract result using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d2, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
@@ -80,17 +132,23 @@ define void @test_acev1_tilezero(ptr %out) {
; CHECK: tilezero
; CHECK: tilemovrow
; CHECK: tilerelease
- %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
; Extract using TILEMOVROW
- %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %c, i32 0)
+ %row0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %c, i32 0)
store volatile <16 x i32> %row0, ptr %out, align 64
ret void
}
-declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
declare x86_amx @llvm.x86.acev1.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
+declare <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx)
+declare <16 x float> @llvm.x86.acev1.tcvtrowd2ps.internal(i16, i16, x86_amx, i32)
+declare <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16h.internal(i16, i16, x86_amx, i32)
+declare <32 x bfloat> @llvm.x86.acev1.tcvtrowps2bf16l.internal(i16, i16, x86_amx, i32)
+declare <32 x half> @llvm.x86.acev1.tcvtrowps2phh.internal(i16, i16, x86_amx, i32)
+declare <32 x half> @llvm.x86.acev1.tcvtrowps2phl.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
index 58887c9623cca0..ffe6d64a69e886 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
@@ -25,15 +25,15 @@ define void @spill_nine_tiles_bf16(ptr %out, <32 x bfloat> %a, <32 x bfloat> %b)
; Reload loads each row back and writes it into the tile.
; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
entry:
- %t0 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t3 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t4 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t5 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t0 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t1 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t2 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t3 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t4 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t5 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t6 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t7 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t8 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d0 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t0, <32 x bfloat> %a, <32 x bfloat> %b)
%d1 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t1, <32 x bfloat> %a, <32 x bfloat> %b)
@@ -46,23 +46,23 @@ entry:
%d8 = tail call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t8, <32 x bfloat> %a, <32 x bfloat> %b)
; Keep all nine results live to the end so the allocator has to spill.
- %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ %r0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d0, i32 0)
store volatile <16 x i32> %r0, ptr %out, align 64
- %r1 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ %r1 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d1, i32 0)
store volatile <16 x i32> %r1, ptr %out, align 64
- %r2 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ %r2 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d2, i32 0)
store volatile <16 x i32> %r2, ptr %out, align 64
- %r3 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ %r3 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d3, i32 0)
store volatile <16 x i32> %r3, ptr %out, align 64
- %r4 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ %r4 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d4, i32 0)
store volatile <16 x i32> %r4, ptr %out, align 64
- %r5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ %r5 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d5, i32 0)
store volatile <16 x i32> %r5, ptr %out, align 64
- %r6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ %r6 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d6, i32 0)
store volatile <16 x i32> %r6, ptr %out, align 64
- %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ %r7 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d7, i32 0)
store volatile <16 x i32> %r7, ptr %out, align 64
- %r8 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+ %r8 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d8, i32 0)
store volatile <16 x i32> %r8, ptr %out, align 64
ret void
@@ -76,15 +76,15 @@ define void @spill_nine_tiles_int(ptr %out, <64 x i8> %a) nounwind {
; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
entry:
- %t0 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t3 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t4 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t5 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
- %t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t0 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t1 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t2 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t3 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t4 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t5 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t6 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t7 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t8 = tail call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
%d0 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t0, <64 x i8> %a, <64 x i8> %a)
%d1 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t1, <64 x i8> %a, <64 x i8> %a)
@@ -96,29 +96,29 @@ entry:
%d7 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t7, <64 x i8> %a, <64 x i8> %a)
%d8 = tail call x86_amx @llvm.x86.acev1.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t8, <64 x i8> %a, <64 x i8> %a)
- %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ %r0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d0, i32 0)
store volatile <16 x i32> %r0, ptr %out, align 64
- %r1 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ %r1 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d1, i32 0)
store volatile <16 x i32> %r1, ptr %out, align 64
- %r2 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ %r2 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d2, i32 0)
store volatile <16 x i32> %r2, ptr %out, align 64
- %r3 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ %r3 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d3, i32 0)
store volatile <16 x i32> %r3, ptr %out, align 64
- %r4 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ %r4 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d4, i32 0)
store volatile <16 x i32> %r4, ptr %out, align 64
- %r5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ %r5 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d5, i32 0)
store volatile <16 x i32> %r5, ptr %out, align 64
- %r6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ %r6 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d6, i32 0)
store volatile <16 x i32> %r6, ptr %out, align 64
- %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ %r7 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d7, i32 0)
store volatile <16 x i32> %r7, ptr %out, align 64
- %r8 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+ %r8 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d8, i32 0)
store volatile <16 x i32> %r8, ptr %out, align 64
ret void
}
-declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
declare x86_amx @llvm.x86.acev1.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
-declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
>From 6ce88631884980013687ba56cacc27955eeff3f2 Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Mon, 16 Nov 2026 07:51:22 +0530
Subject: [PATCH 5/6] [X86][ACEV1] Model the tile row and column writes as
read-modify-write
TILEMOVROW and TILEMOVCOL writing into a tile update one row or column and
keep the rest, so the destination is a source as well; the ACE v1
specification encodes operand 1 of all four write forms as REG(r/w). Tie
it. Palette 2 has no TILELOADD, so a reload rebuilds the tile a row at a
time, and without the tie every write but the last looked dead and the
sequence was not valid SSA before register allocation.
Moving a tile also stops asking the subtarget which palette it belongs to.
An ACE target can hold palette 1 tiles, which took the row-by-row route
shaped for palette 2's fixed 16x64 tile. Key it on the programming model,
as the palette byte in the same function already is.
---
llvm/lib/Target/X86/X86ExpandPseudo.cpp | 7 ++
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 36 +++++---
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 19 ----
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 +
llvm/lib/Target/X86/X86InstrACE.td | 14 +--
llvm/lib/Target/X86/X86InstrInfo.cpp | 28 ++++--
llvm/lib/Target/X86/X86LowerTileCopy.cpp | 13 +--
.../CodeGen/X86/ACEV1/acev1-amx-interop.ll | 41 ++++++++-
llvm/test/CodeGen/X86/ACEV1/acev1-tile-rmw.ll | 89 +++++++++++++++++++
.../X86/ACEV1/acev1-tile-spill-fallthrough.ll | 38 ++++++++
10 files changed, 239 insertions(+), 50 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-tile-rmw.ll
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill-fallthrough.ll
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 71c20407e78617..67413f4def0b12 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -812,6 +812,13 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
unsigned Opc = (Opcode == X86::PTILEMOVCOLtreV) ? X86::TILEMOVCOLtre
: X86::TILEMOVROWtre;
MI.setDesc(TII->get(Opc));
+ // The write merges into the destination tile, which these intrinsics don't
+ // provide, so merge with undef. addOperand ties it, per the descriptor.
+ MI.insert(MI.operands_begin() + 1,
+ MachineOperand::CreateReg(MI.getOperand(0).getReg(),
+ /*isDef=*/false, /*isImp=*/false,
+ /*isKill=*/false, /*isDead=*/false,
+ /*isUndef=*/true));
return true;
}
case X86::PTILESTOREDV: {
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 0c7ba5527b118d..2c3919a6c6962f 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -227,19 +227,20 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
- // ACE configures palette 2, which has no TILESTORED - use TILEMOVROW
- // row-by-row through a scratch ZMM.
- // FIXME: This defines the tile and the scratch register once per row, which
- // is not valid SSA. X86VolatileTileData also still emits the AMX load/store
- // intrinsics at -O0, so the ACE -O0 path needs both fixed together.
- if (ST->hasACEV1()) {
- const DebugLoc &DL = Before->getDebugLoc();
- Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+ // Palette 2 has no TILESTORED - use TILEMOVROW row-by-row through a scratch
+ // ZMM. Only palette 2 tiles, since palette 1 has its own shape.
+ if (X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA) {
+ // Before is MBB.end() when the tile def ends the block, so it can't be
+ // dereferenced for a debug location.
+ DebugLoc DL;
const unsigned NumRows = 16;
const unsigned RowSize = 64;
for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // The read form writes the whole ZMM, so each row needs its own.
+ Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+
// tilemovrow $row, %tmm, %zmm
BuildMI(*MBB, Before, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
.addReg(VirtReg)
@@ -284,15 +285,24 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
// ACE configures palette 2, which has no TILELOADD - use TILEMOVROW
// row-by-row through a scratch ZMM.
- // FIXME: See the corresponding comment in spill().
- if (ST->hasACEV1()) {
+ if (X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA) {
const DebugLoc &DL = UseMI->getDebugLoc();
- Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
const unsigned NumRows = 16;
const unsigned RowSize = 64;
+ // Each row write returns the tile with one more row in place, so thread a
+ // register through the rows and end at TileReg. Row 0 has nothing to
+ // merge with.
+ Register PartialTile = MRI->createVirtualRegister(&RC);
+ BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::IMPLICIT_DEF),
+ PartialTile);
+
for (unsigned Row = 0; Row < NumRows; ++Row) {
+ Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+ Register Filled =
+ Row + 1 == NumRows ? TileReg : MRI->createVirtualRegister(&RC);
+
// vmovups row*64(%sp), %zmm
MachineInstrBuilder MIB = BuildMI(*UseMI->getParent(), UseMI, DL,
TII->get(X86::VMOVUPSZrm), ScratchZMM);
@@ -300,9 +310,11 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
// tilemovrow $row, %zmm, %tmm
BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWtri),
- TileReg)
+ Filled)
+ .addReg(PartialTile)
.addReg(ScratchZMM)
.addImm(Row);
+ PartialTile = Filled;
}
} else {
// AMX: Use PTILELOADDV
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index d1cc15598e4044..8b995885d23a28 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -5778,25 +5778,6 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
ReplaceNode(Node, CNode);
return;
}
- // AMX-AVX512 internal intrinsics - set program model for struct-based API
- case Intrinsic::x86_tcvtrowd2ps_internal:
- case Intrinsic::x86_tcvtrowps2bf16h_internal:
- case Intrinsic::x86_tcvtrowps2bf16l_internal:
- case Intrinsic::x86_tcvtrowps2phh_internal:
- case Intrinsic::x86_tcvtrowps2phl_internal:
- case Intrinsic::x86_tilemovrow_internal: {
- if (!Subtarget->hasAMXAVX512() && !Subtarget->hasACEV1())
- break;
- auto *MFI =
- CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
- // Set program model for struct-based API (ManagedRA)
- if (Subtarget->hasACEV1())
- MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
- else
- MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
- // Let pattern matching handle the rest
- break;
- }
}
break;
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 4195d8ed57ebd5..8cb2387d9bb1fe 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -39868,6 +39868,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
Register SrcReg = MI.getOperand(1).getReg();
unsigned Idx = MI.getOperand(2).getImm();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLtri), DstReg)
+ .addReg(DstReg, RegState::Undef)
.addReg(SrcReg)
.addImm(Idx);
MI.eraseFromParent();
@@ -39877,6 +39878,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
Register SrcReg = MI.getOperand(1).getReg();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLtre), DstReg)
+ .addReg(DstReg, RegState::Undef)
.addReg(SrcReg)
.addReg(MI.getOperand(2).getReg());
MI.eraseFromParent();
@@ -39887,6 +39889,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
Register SrcReg = MI.getOperand(1).getReg();
unsigned Idx = MI.getOperand(2).getImm();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWtri), DstReg)
+ .addReg(DstReg, RegState::Undef)
.addReg(SrcReg)
.addImm(Idx);
MI.eraseFromParent();
@@ -39896,6 +39899,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
Register SrcReg = MI.getOperand(1).getReg();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWtre), DstReg)
+ .addReg(DstReg, RegState::Undef)
.addReg(SrcReg)
.addReg(MI.getOperand(2).getReg());
MI.eraseFromParent();
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
index b2914d12c05563..5a281f137698d9 100644
--- a/llvm/lib/Target/X86/X86InstrACE.td
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -103,27 +103,29 @@ let SchedRW = [WriteSystem] in {
}
// TILEMOVCOL - Move Column from Vector to Tile
-let SchedRW = [WriteSystem] in {
+// Writes one column and keeps the rest, so $dst is a source too.
+let Constraints = "$src0 = $dst", SchedRW = [WriteSystem] in {
def TILEMOVCOLtri : Ii8<0x2F, MRMSrcReg, (outs TILE:$dst),
- (ins VR512:$src, u8imm:$idx),
+ (ins TILE:$src0, VR512:$src, u8imm:$idx),
"tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
EVEX, PD, TA, REX_W, EVEX_V512;
def TILEMOVCOLtre : I<0x4B, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins VR512:$src, GR32:$idx),
+ (ins TILE:$src0, VR512:$src, GR32:$idx),
"tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
}
// TILEMOVROW - Move Row from Vector to Tile
-let SchedRW = [WriteSystem] in {
+// Writes one row and keeps the rest, so $dst is a source too.
+let Constraints = "$src0 = $dst", SchedRW = [WriteSystem] in {
def TILEMOVROWtri : Ii8<0x07, MRMSrcReg, (outs TILE:$dst),
- (ins VR512:$src, u8imm:$idx),
+ (ins TILE:$src0, VR512:$src, u8imm:$idx),
"tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
EVEX, PD, TA, REX_W, EVEX_V512;
def TILEMOVROWtre : I<0x4A, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins VR512:$src, GR32:$idx),
+ (ins TILE:$src0, VR512:$src, GR32:$idx),
"tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 87d557f0c65efa..1e32be773ba72b 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4520,12 +4520,6 @@ static unsigned getLoadStoreRegOpcode(Register Reg,
assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
assert((STI.hasAMXTILE() || STI.hasACEV1()) &&
"Using 8*1024-bit register requires AMX-TILE or ACE");
- // ACE configures palette 2, which has no TILELOADD/TILESTORED. Spill and
- // reload through pseudos that X86LowerTileCopy expands into TILEMOVROW.
- // This is the only place these pseudos are created, so it is what keeps
- // the row-by-row sequences confined to ACE targets.
- if (STI.hasACEV1())
- return Load ? X86::ACE_TILERELOAD : X86::ACE_TILESPILL;
#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
: GET_EGPR_IF_ENABLED(X86::TILESTORED);
@@ -4716,6 +4710,26 @@ static bool isAMXOpcode(unsigned Opc) {
}
}
+// Palette 2 has no TILELOADD/TILESTORED, so those tiles spill and reload
+// through pseudos that X86LowerTileCopy expands into TILEMOVROW. An ACE target
+// can still hold palette 1 tiles, which spill the AMX way, so the programming
+// model decides this rather than the subtarget.
+static unsigned getACETileSpillOpcode(unsigned Opc, const MachineFunction &MF) {
+ if (MF.getInfo<X86MachineFunctionInfo>()->getACEProgModel() !=
+ ACEProgModelEnum::ACE_ManagedRA)
+ return Opc;
+ switch (Opc) {
+ default:
+ return Opc;
+ case X86::TILELOADD:
+ case X86::TILELOADD_EVEX:
+ return X86::ACE_TILERELOAD;
+ case X86::TILESTORED:
+ case X86::TILESTORED_EVEX:
+ return X86::ACE_TILESPILL;
+ }
+}
+
void X86InstrInfo::loadStoreTileReg(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
unsigned Opc, Register Reg, int FrameIdx,
@@ -4767,6 +4781,7 @@ void X86InstrInfo::storeRegToStackSlot(
(RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
+ Opc = getACETileSpillOpcode(Opc, MF);
if (isAMXOpcode(Opc))
loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill);
else
@@ -4791,6 +4806,7 @@ void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
(RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
+ Opc = getACETileSpillOpcode(Opc, MF);
if (isAMXOpcode(Opc))
loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx);
else
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 0c29d84c15d81d..0a9add593eb9fc 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -75,7 +75,9 @@ static void emitACETileReload(MachineBasicBlock &MBB,
addFrameReference(MIB, TileSS, Row * ACETileRowSize);
// tilemovrow $row, %zmm, %tmm
+ // Each write merges into the tile; row 0 has nothing to merge with.
BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWtri), DstReg)
+ .addReg(DstReg, Row == 0 ? RegState::Undef : RegState::NoFlags)
.addReg(ScratchZMM, RegState::Kill)
.addImm(Row);
}
@@ -152,10 +154,11 @@ static bool lowerTileCopy(MachineFunction &MF) {
// Expand the tile spill/reload pseudos inserted by the register
// allocator. getLoadStoreRegOpcode is the only place these are created,
- // and it only does so for ACE targets.
+ // and it only does so for palette 2 tiles.
unsigned Opcode = MI.getOpcode();
if (Opcode == X86::ACE_TILESPILL || Opcode == X86::ACE_TILERELOAD) {
- assert(ST.hasACEV1() && "ACE tile spill pseudo on non-ACE target");
+ assert(FuncInfo->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA &&
+ "ACE tile spill pseudo outside an ACE function");
bool IsSpill = Opcode == X86::ACE_TILESPILL;
// ACE_TILESPILL is (mem, tile), ACE_TILERELOAD is (tile, mem).
MachineOperand &TileMO =
@@ -207,9 +210,9 @@ static bool lowerTileCopy(MachineFunction &MF) {
const DebugLoc &DL = MI.getDebugLoc();
- // ACE configures palette 2, which has no TILELOADD/TILESTORED, so copy
- // the tile row-by-row through a scratch ZMM.
- if (ST.hasACEV1()) {
+ // Palette 2 has no TILELOADD/TILESTORED, so copy the tile row-by-row
+ // through a scratch ZMM.
+ if (FuncInfo->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA) {
Register ScratchZMM = findScratchZMM(MF, TRI, UsedRegs);
int ZmmSS = -1;
if (!ScratchZMM) {
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll
index 5a042c60d4bd45..d5fcd8f87e4836 100644
--- a/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-amx-interop.ll
@@ -1,7 +1,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile,+amx-int8 \
-; RUN: -verify-machineinstrs | FileCheck %s
+; RUN: -verify-machineinstrs | FileCheck %s --implicit-check-not=tilemovrow
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile,+amx-int8,+acev1 \
-; RUN: -verify-machineinstrs | FileCheck %s
+; RUN: -verify-machineinstrs | FileCheck %s --implicit-check-not=tilemovrow
; Test that a function holding only plain AMX intrinsics lowers the same way
; whether or not the target also supports ACE v1.
@@ -29,7 +29,44 @@ define <256 x i32> @amx_tile_to_vector_on_ace_target() nounwind {
ret <256 x i32> %v
}
+; Nine live tiles force a spill. These palette 1 tiles are 8 rows by 32 bytes,
+; so they spill with TILESTORED rather than the row-by-row TILEMOVROW sequence.
+define void @amx_spill_on_ace_target(ptr %out) nounwind {
+; CHECK-LABEL: amx_spill_on_ace_target:
+; CHECK: tilestored %tmm{{[0-7]}}, {{.*}}(%r{{[a-z0-9]+}}
+; CHECK: tileloadd {{.*}}(%r{{[a-z0-9]+}}{{.*}}, %tmm{{[0-7]}}
+ %t0 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t1 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t2 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t3 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t4 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t5 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t6 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t7 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %t8 = call x86_amx @llvm.x86.tilezero.internal(i16 8, i16 32)
+ %d0 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t0, x86_amx %t0, x86_amx %t0)
+ %d1 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t1, x86_amx %t1, x86_amx %t1)
+ %d2 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t2, x86_amx %t2, x86_amx %t2)
+ %d3 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t3, x86_amx %t3, x86_amx %t3)
+ %d4 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t4, x86_amx %t4, x86_amx %t4)
+ %d5 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t5, x86_amx %t5, x86_amx %t5)
+ %d6 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t6, x86_amx %t6, x86_amx %t6)
+ %d7 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t7, x86_amx %t7, x86_amx %t7)
+ %d8 = call x86_amx @llvm.x86.tdpbssd.internal(i16 8, i16 32, i16 32, x86_amx %t8, x86_amx %t8, x86_amx %t8)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d0)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d1)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d2)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d3)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d4)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d5)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d6)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d7)
+ call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr %out, i64 32, x86_amx %d8)
+ ret void
+}
+
declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx)
declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx)
declare x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32>)
declare <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-rmw.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-rmw.ll
new file mode 100644
index 00000000000000..9db8eae24aecac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-rmw.ll
@@ -0,0 +1,89 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -O0 \
+; RUN: -verify-machineinstrs -stop-after=x86-fast-pre-tile-config -o - \
+; RUN: | FileCheck %s --check-prefix=MIR
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -O0 \
+; RUN: -verify-machineinstrs | FileCheck %s --check-prefix=ASM
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 \
+; RUN: -verify-machineinstrs | FileCheck %s --check-prefix=ASM
+
+; Test the tile row and column writes modelled as read-modify-write: they write
+; one row or column and keep the rest, so the destination is a source as well.
+
+; Nine live tiles exceed the eight physical TMM registers, forcing a spill.
+define void @reload_is_a_merge_chain(ptr %out, <32 x bfloat> %a, <32 x bfloat> %b) nounwind {
+; The reload threads one register through the rows; row 0 starts from a def.
+; MIR-LABEL: name: reload_is_a_merge_chain
+; MIR: [[SEED:%[0-9]+]]:tile = IMPLICIT_DEF
+; MIR: [[R0:%[0-9]+]]:tile = TILEMOVROWtri [[SEED]], %{{[0-9]+}}, 0
+; MIR-NEXT: {{.*}}VMOVUPSZrm
+; MIR-NEXT: [[R1:%[0-9]+]]:tile = TILEMOVROWtri [[R0]], %{{[0-9]+}}, 1
+; MIR-NEXT: {{.*}}VMOVUPSZrm
+; MIR-NEXT: %{{[0-9]+}}:tile = TILEMOVROWtri [[R1]], %{{[0-9]+}}, 2
+;
+; The read direction writes a whole ZMM, so those defs stay independent.
+; MIR: %{{[0-9]+}}:vr512 = TILEMOVROWrti %{{[0-9]+}}, 0
+entry:
+ %t0 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t1 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t2 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t3 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t4 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t5 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t6 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t7 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %t8 = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+
+ %d0 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t0, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d1 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t1, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d2 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t2, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d3 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t3, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d4 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t4, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d5 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t5, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d6 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t6, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d7 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t7, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d8 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t8, <32 x bfloat> %a, <32 x bfloat> %b)
+
+ ; Read a row back out of every tile to keep them all live across the region.
+ %r0 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ %r1 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ %r2 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ %r3 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ %r4 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ %r5 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ %r6 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ %r7 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ %r8 = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+
+ %s0 = add <16 x i32> %r0, %r1
+ %s1 = add <16 x i32> %s0, %r2
+ %s2 = add <16 x i32> %s1, %r3
+ %s3 = add <16 x i32> %s2, %r4
+ %s4 = add <16 x i32> %s3, %r5
+ %s5 = add <16 x i32> %s4, %r6
+ %s6 = add <16 x i32> %s5, %r7
+ %s7 = add <16 x i32> %s6, %r8
+ store <16 x i32> %s7, ptr %out
+ ret void
+}
+
+; These intrinsics take no incoming tile, so the tied source is undef.
+define void @write_without_an_incoming_tile(ptr %out, <16 x i32> %v, i32 %idx) nounwind {
+; ASM-LABEL: write_without_an_incoming_tile:
+; ASM: ldtilecfg
+; ASM: tilemovrow %{{[a-z0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
+; ASM: tilemovcol %{{[a-z0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
+entry:
+ %row = call x86_amx @llvm.x86.acev1.tilemovrowinsert.internal(i16 16, i16 64, <16 x i32> %v, i32 %idx)
+ %col = call x86_amx @llvm.x86.acev1.tilemovcolinsert.internal(i16 16, i16 64, <16 x i32> %v, i32 %idx)
+ %a = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %row, i32 0)
+ %b = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %col, i32 0)
+ %s = add <16 x i32> %a, %b
+ store <16 x i32> %s, ptr %out
+ ret void
+}
+
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
+declare x86_amx @llvm.x86.acev1.tilemovrowinsert.internal(i16, i16, <16 x i32>, i32)
+declare x86_amx @llvm.x86.acev1.tilemovcolinsert.internal(i16, i16, <16 x i32>, i32)
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill-fallthrough.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill-fallthrough.ll
new file mode 100644
index 00000000000000..e9dfbb0ba8a5cb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill-fallthrough.ll
@@ -0,0 +1,38 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -O0 \
+; RUN: -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -O0 \
+; RUN: -verify-machineinstrs | FileCheck %s
+
+; Test spilling a tile that is defined by the last instruction of a block, where
+; the insertion point one past the def is MBB.end().
+
+; The tile in each arm is live out through the PHI, and the branch to the
+; fall-through successor is gone by then, so the tile def ends the block.
+define void @spill_tile_defined_last_in_block(ptr %out, i32 %cond, <32 x bfloat> %a, <32 x bfloat> %b) nounwind {
+; CHECK-LABEL: spill_tile_defined_last_in_block:
+; CHECK: ldtilecfg
+; CHECK-COUNT-16: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; CHECK: tilerelease
+entry:
+ %z = call x86_amx @llvm.x86.acev1.tilezero.internal(i16 16, i16 64)
+ %c = icmp eq i32 %cond, 0
+ br i1 %c, label %then, label %else
+
+then:
+ %d1 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %z, <32 x bfloat> %a, <32 x bfloat> %b)
+ br label %merge
+
+else:
+ %d2 = call x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %z, <32 x bfloat> %b, <32 x bfloat> %a)
+ br label %merge
+
+merge:
+ %t = phi x86_amx [ %d1, %then ], [ %d2, %else ]
+ %row = call <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x i32> %row, ptr %out, align 64
+ ret void
+}
+
+declare x86_amx @llvm.x86.acev1.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.acev1.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare <16 x i32> @llvm.x86.acev1.tilemovrowextract.internal(i16, i16, x86_amx, i32)
>From c32bcf355fd0819180500181ac491989a67aabdc Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Mon, 16 Nov 2026 08:39:33 +0530
Subject: [PATCH 6/6] [X86][ACEV1][NFC] Emit the ACE instruction definitions
from multiclasses
Most ACE operations need three definitions: the instruction itself, a pseudo
for the user-facing intrinsic that takes an immediate tile ID, and a pseudo
for the internal intrinsic that takes a shape and a tile register so the
register allocator can assign TMMs. All three were written out by hand for
every operation, so the four integer outer products differed only in
mnemonic and prefix, and the twelve row-read pseudos only in the intrinsic.
Emit them from five multiclasses, one per operation shape, deriving the
intrinsic names from the mnemonic as X86InstrAMX.td already does.
---
llvm/lib/Target/X86/X86InstrACE.td | 606 ++++++++++-------------------
1 file changed, 207 insertions(+), 399 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
index 5a281f137698d9..f678c994f5a15b 100644
--- a/llvm/lib/Target/X86/X86InstrACE.td
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -11,6 +11,184 @@
//
//===----------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+// ACE multiclasses
+//
+// Most ACE operations need three definitions: the instruction itself, a pseudo
+// for the user-facing intrinsic that takes an immediate tile ID, and a pseudo
+// for the internal intrinsic that takes a shape and a tile register so the
+// register allocator can assign TMMs. The multiclasses below emit all of them
+// from one mnemonic, which also names the intrinsics.
+//===----------------------------------------------------------------------===//
+
+// BSRMOVH and BSRMOVL move half of the block scale register, which is wider
+// than a vector register, in either direction.
+multiclass ACE_BSRMOV_HALF<string Half, Prefix P> {
+ defvar Insert = !cast<Intrinsic>("int_x86_acev1_bsr0mov"#Half#"insert");
+ defvar Extract = !cast<Intrinsic>("int_x86_acev1_bsr0mov"#Half#"extract");
+
+ let SchedRW = [WriteSystem], OpPrefix = P in {
+ let Defs = [BSR0], Uses = [BSR0] in {
+ def rr_set : I<0x95, MRM0r, (outs),
+ (ins VR512:$src),
+ !strconcat("bsrmov", Half, "\t{$src, %bsr0|bsr0, $src}"),
+ [(Insert (v64i8 VR512:$src))]>,
+ EVEX, T_MAP6, REX_W, EVEX_V512;
+
+ let mayLoad = 1 in
+ def rm_set : I<0x95, MRM0m, (outs),
+ (ins f512mem:$src),
+ !strconcat("bsrmov", Half, "\t{$src, %bsr0|bsr0, $src}"),
+ []>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+
+ let Uses = [BSR0] in {
+ def rr_get : I<0x95, MRM0r, (outs VR512:$dst),
+ (ins),
+ !strconcat("bsrmov", Half, "\t{%bsr0, $dst|$dst, bsr0}"),
+ [(set (v64i8 VR512:$dst), (Extract))]>,
+ EVEX, T_MAP6, EVEX_V512;
+
+ let mayStore = 1 in
+ def mr_get : I<0x95, MRM0m, (outs),
+ (ins f512mem:$dst),
+ !strconcat("bsrmov", Half, "\t{%bsr0, $dst|$dst, bsr0}"),
+ [(store (v64i8 (Extract)), addr:$dst)]>,
+ EVEX, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+ }
+}
+
+// TILEMOVCOL and TILEMOVROW in the tile-destination direction write one column
+// or row and keep the rest, so $dst is a source too. Only the register index
+// form has an internal twin; the shape operands carry the index.
+multiclass ACE_TILEMOVE<bits<8> OpcImm, bits<8> OpcReg, string Opstr,
+ string IntName> {
+ let Constraints = "$src0 = $dst", SchedRW = [WriteSystem] in {
+ def tri : Ii8<OpcImm, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src0, VR512:$src, u8imm:$idx),
+ !strconcat(Opstr, "\t{$idx, $src, $dst|$dst, $src, $idx}"),
+ []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def tre : I<OpcReg, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src0, VR512:$src, GR32:$idx),
+ !strconcat(Opstr, "\t{$idx, $src, $dst|$dst, $src, $idx}"),
+ []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+ }
+
+ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
+ def "P"#NAME#"tri" : PseudoI<(outs),
+ (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+
+ def "P"#NAME#"tre" : PseudoI<(outs),
+ (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+ }
+
+ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in
+ def "P"#NAME#"treV"
+ : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#IntName#"_internal")
+ GR16:$row, GR16:$col, VR512:$src, GR32:$idx))]>;
+}
+
+// Outer products accumulate into the destination tile, so $src1 is tied to it.
+multiclass ACE_TOP<bits<8> Opc, string Opstr, Prefix P> {
+ let Constraints = "$src1 = $dst", SchedRW = [WriteSystem], OpPrefix = P in
+ def trr : I<Opc, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ !strconcat(Opstr, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+ []>,
+ EVEX, VVVV, T8, EVEX_V512;
+
+ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in
+ def "P"#NAME#"trr"
+ : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(!cast<Intrinsic>("int_x86_acev1_"#Opstr)
+ timm:$dst, VR512:$src1, VR512:$src2)]>;
+
+ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1,
+ Constraints = "$src1 = $dst" in
+ def "P"#NAME#"trrV"
+ : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#Opstr#"_internal")
+ GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+}
+
+// The mixed-precision outer products take an immediate selecting the element
+// formats and read the scales from the block scale register.
+multiclass ACE_TOP4MX<bits<8> Opc, string Opstr, Prefix P> {
+ let Constraints = "$src1 = $dst", SchedRW = [WriteSystem], Uses = [BSR0],
+ OpPrefix = P in
+ def trri : Ii8<Opc, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ !strconcat(Opstr,
+ "\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}"),
+ []>,
+ EVEX, VVVV, TA, EVEX_V512;
+
+ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in
+ def "P"#NAME#"trri"
+ : PseudoI<(outs),
+ (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(!cast<Intrinsic>("int_x86_acev1_"#Opstr)
+ timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
+
+ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1,
+ Constraints = "$src1 = $dst", Uses = [BSR0] in
+ def "P"#NAME#"trriV"
+ : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#Opstr#"_internal")
+ GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+}
+
+// The row reads have no ACE-specific encoding - they are the AMX instructions
+// under palette 2 - so only pseudos are needed here. The opcode is what
+// records the ACE programming model.
+multiclass ACE_ROWREAD<string Opstr> {
+ let SchedRW = [WriteSystem], usesCustomInserter = 1 in {
+ def rte : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2),
+ [(set VR512:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#Opstr)
+ timm:$src1, GR32:$src2))]>;
+
+ // One ACE intrinsic covers both index forms, so a constant index matches
+ // the immediate encoding here and anything else falls back to the register
+ // one.
+ def rti : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
+ [(set VR512:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#Opstr)
+ timm:$src1, imm:$src2))]>;
+ }
+
+ let isPseudo = true, hasNoSchedulingInfo = 1, SchedRW = [WriteSystem],
+ usesCustomInserter = 1 in {
+ def rtiV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
+ [(set VR512:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#Opstr#"_internal")
+ GR16:$src1, GR16:$src2, TILE:$src3, imm:$src4))]>;
+
+ def rteV : PseudoI<(outs VR512:$dst),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4),
+ [(set VR512:$dst,
+ (!cast<Intrinsic>("int_x86_acev1_"#Opstr#"_internal")
+ GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4))]>;
+ }
+}
+
//===----------------------------------------------------------------------===//
// ACE instructions
@@ -39,301 +217,56 @@ let SchedRW = [WriteSystem], Defs = [BSR0] in {
}
// BSRMOVH - Move Half to/from BSR
-let SchedRW = [WriteSystem] in {
- let Defs = [BSR0], Uses = [BSR0] in {
- def BSRMOVHrr_set : I<0x95, MRM0r, (outs),
- (ins VR512:$src),
- "bsrmovh\t{$src, %bsr0|bsr0, $src}",
- [(int_x86_acev1_bsr0movhinsert (v64i8 VR512:$src))]>,
- EVEX, XD, T_MAP6, REX_W, EVEX_V512;
-
- let mayLoad = 1 in
- def BSRMOVHrm_set : I<0x95, MRM0m, (outs),
- (ins f512mem:$src),
- "bsrmovh\t{$src, %bsr0|bsr0, $src}", []>,
- EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
- }
-
- let Uses = [BSR0] in {
- def BSRMOVHrr_get : I<0x95, MRM0r, (outs VR512:$dst),
- (ins),
- "bsrmovh\t{%bsr0, $dst|$dst, bsr0}",
- [(set (v64i8 VR512:$dst), (int_x86_acev1_bsr0movhextract))]>,
- EVEX, XD, T_MAP6, EVEX_V512;
-
- let mayStore = 1 in
- def BSRMOVHmr_get : I<0x95, MRM0m, (outs),
- (ins f512mem:$dst),
- "bsrmovh\t{%bsr0, $dst|$dst, bsr0}",
- [(store (v64i8 (int_x86_acev1_bsr0movhextract)), addr:$dst)]>,
- EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
- }
-}
+defm BSRMOVH : ACE_BSRMOV_HALF<"h", XD>;
// BSRMOVL - Move Low to/from BSR
-let SchedRW = [WriteSystem] in {
- let Defs = [BSR0], Uses = [BSR0] in {
- def BSRMOVLrr_set : I<0x95, MRM0r, (outs),
- (ins VR512:$src),
- "bsrmovl\t{$src, %bsr0|bsr0, $src}",
- [(int_x86_acev1_bsr0movlinsert (v64i8 VR512:$src))]>,
- EVEX, XS, T_MAP6, REX_W, EVEX_V512;
-
- let mayLoad = 1 in
- def BSRMOVLrm_set : I<0x95, MRM0m, (outs),
- (ins f512mem:$src),
- "bsrmovl\t{$src, %bsr0|bsr0, $src}", []>,
- EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
- }
-
- let Uses = [BSR0] in {
- def BSRMOVLrr_get : I<0x95, MRM0r, (outs VR512:$dst),
- (ins),
- "bsrmovl\t{%bsr0, $dst|$dst, bsr0}",
- [(set (v64i8 VR512:$dst), (int_x86_acev1_bsr0movlextract))]>,
- EVEX, XS, T_MAP6, EVEX_V512;
-
- let mayStore = 1 in
- def BSRMOVLmr_get : I<0x95, MRM0m, (outs),
- (ins f512mem:$dst),
- "bsrmovl\t{%bsr0, $dst|$dst, bsr0}",
- [(store (v64i8 (int_x86_acev1_bsr0movlextract)), addr:$dst)]>,
- EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
- }
-}
+defm BSRMOVL : ACE_BSRMOV_HALF<"l", XS>;
// TILEMOVCOL - Move Column from Vector to Tile
-// Writes one column and keeps the rest, so $dst is a source too.
-let Constraints = "$src0 = $dst", SchedRW = [WriteSystem] in {
- def TILEMOVCOLtri : Ii8<0x2F, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src0, VR512:$src, u8imm:$idx),
- "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, TA, REX_W, EVEX_V512;
-
- def TILEMOVCOLtre : I<0x4B, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src0, VR512:$src, GR32:$idx),
- "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
-}
+defm TILEMOVCOL : ACE_TILEMOVE<0x2F, 0x4B, "tilemovcol", "tilemovcolinsert">;
// TILEMOVROW - Move Row from Vector to Tile
-// Writes one row and keeps the rest, so $dst is a source too.
-let Constraints = "$src0 = $dst", SchedRW = [WriteSystem] in {
- def TILEMOVROWtri : Ii8<0x07, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src0, VR512:$src, u8imm:$idx),
- "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, TA, REX_W, EVEX_V512;
-
- def TILEMOVROWtre : I<0x4A, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src0, VR512:$src, GR32:$idx),
- "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
- EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
-}
+defm TILEMOVROW : ACE_TILEMOVE<0x07, 0x4A, "tilemovrow", "tilemovrowinsert">;
// Outer Product Instructions
// TOP2BF16PS - Outer Product BF16 to Single Precision
-let Constraints = "$src1 = $dst", SchedRW = [WriteSystem] in {
- def TOP2BF16PStrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src2, VR512:$src3),
- "top2bf16ps\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, XS, T8, EVEX_V512;
-}
+defm TOP2BF16PS : ACE_TOP<0x5C, "top2bf16ps", XS>;
// TOP4 Instructions - integer outer products
-let Constraints = "$src1 = $dst", SchedRW = [WriteSystem] in {
- def TOP4BUUDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src2, VR512:$src3),
- "top4buud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, T8, EVEX_V512;
-
- def TOP4BUSDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src2, VR512:$src3),
- "top4busd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, PD, T8, EVEX_V512;
-
- def TOP4BSSDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src2, VR512:$src3),
- "top4bssd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, XD, T8, EVEX_V512;
-
- def TOP4BSUDtrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src2, VR512:$src3),
- "top4bsud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
- EVEX, VVVV, XS, T8, EVEX_V512;
-}
+defm TOP4BUUD : ACE_TOP<0x5E, "top4buud", PS>;
+defm TOP4BUSD : ACE_TOP<0x5E, "top4busd", PD>;
+defm TOP4BSSD : ACE_TOP<0x5E, "top4bssd", XD>;
+defm TOP4BSUD : ACE_TOP<0x5E, "top4bsud", XS>;
// TOP4MX Instructions - Mixed Precision Outer Products with Immediate
-let Constraints = "$src1 = $dst", SchedRW = [WriteSystem], Uses = [BSR0] in {
- def TOP4MXBF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
- "top4mxbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, TA, EVEX_V512;
-
- def TOP4MXHF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
- "top4mxhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, PD, TA, EVEX_V512;
-
- def TOP4MXBHF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
- "top4mxbhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, XD, TA, EVEX_V512;
-
- def TOP4MXHBF8PStrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
- "top4mxhbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, XS, TA, EVEX_V512;
-
- def TOP4MXBSSPStrri : Ii8<0x8F, MRMSrcReg, (outs TILE:$dst),
- (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
- "top4mxbssps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
- EVEX, VVVV, XD, TA, EVEX_V512;
-}
+defm TOP4MXHF8PS : ACE_TOP4MX<0x8D, "top4mxhf8ps", PD>;
+defm TOP4MXBHF8PS : ACE_TOP4MX<0x8D, "top4mxbhf8ps", XD>;
+defm TOP4MXHBF8PS : ACE_TOP4MX<0x8D, "top4mxhbf8ps", XS>;
+defm TOP4MXBF8PS : ACE_TOP4MX<0x8D, "top4mxbf8ps", PS>;
+defm TOP4MXBSSPS : ACE_TOP4MX<0x8F, "top4mxbssps", XD>;
//===----------------------------------------------------------------------===//
// ACE Pseudo Instructions
// These are used by intrinsics and expanded via custom inserter
//===----------------------------------------------------------------------===//
-// Pseudo instructions for user-facing intrinsics (immediate tile IDs).
-// usesCustomInserter expands these to real instructions in X86ISelLowering.
-let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
- def PBSRINIT : PseudoI<(outs), (ins),
- [(int_x86_acev1_bsr0init)]>;
-
- def PTILEMOVCOLtri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
-
- def PTILEMOVCOLtre : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
-
- def PTILEMOVROWtri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
-
- def PTILEMOVROWtre : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
-
- def PTOP2BF16PStrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_acev1_top2bf16ps timm:$dst, VR512:$src1, VR512:$src2)]>;
-
- def PTOP4BUUDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_acev1_top4buud timm:$dst, VR512:$src1, VR512:$src2)]>;
+// Pseudo instruction for a user-facing intrinsic with no instruction operands.
+let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in
+def PBSRINIT : PseudoI<(outs), (ins), [(int_x86_acev1_bsr0init)]>;
- def PTOP4BUSDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_acev1_top4busd timm:$dst, VR512:$src1, VR512:$src2)]>;
+// ACE twins of the AMX row reads, so the ACE model reaches MIR. The ACE read
+// direction of TILEMOVROW has an intrinsic name of its own.
+defm ACE_PTCVTROWD2PS : ACE_ROWREAD<"tcvtrowd2ps">;
+defm ACE_PTCVTROWPS2BF16H : ACE_ROWREAD<"tcvtrowps2bf16h">;
+defm ACE_PTCVTROWPS2BF16L : ACE_ROWREAD<"tcvtrowps2bf16l">;
+defm ACE_PTCVTROWPS2PHH : ACE_ROWREAD<"tcvtrowps2phh">;
+defm ACE_PTCVTROWPS2PHL : ACE_ROWREAD<"tcvtrowps2phl">;
+defm ACE_PTILEMOVROW : ACE_ROWREAD<"tilemovrowextract">;
- def PTOP4BSSDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_acev1_top4bssd timm:$dst, VR512:$src1, VR512:$src2)]>;
-
- def PTOP4BSUDtrr : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
- [(int_x86_acev1_top4bsud timm:$dst, VR512:$src1, VR512:$src2)]>;
-
- def PTOP4MXHF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_acev1_top4mxhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
-
- def PTOP4MXBHF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_acev1_top4mxbhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
-
- def PTOP4MXHBF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_acev1_top4mxhbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
-
- def PTOP4MXBF8PStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_acev1_top4mxbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
-
- def PTOP4MXBSSPStrri : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
- [(int_x86_acev1_top4mxbssps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
-}
-
-// Pseudo instructions for internal intrinsics (register allocation).
-let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
- def PTILEMOVCOLtreV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
- [(set TILE:$dst,
- (int_x86_acev1_tilemovcolinsert_internal GR16:$row, GR16:$col,
- VR512:$src, GR32:$idx))]>;
-
- def PTILEMOVROWtreV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
- [(set TILE:$dst,
- (int_x86_acev1_tilemovrowinsert_internal GR16:$row, GR16:$col,
- VR512:$src, GR32:$idx))]>;
-
- let Constraints = "$src1 = $dst" in
- def PTOP2BF16PStrrV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top2bf16ps_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst" in
- def PTOP4BUUDtrrV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4buud_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst" in
- def PTOP4BUSDtrrV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4busd_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst" in
- def PTOP4BSSDtrrV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4bssd_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst" in
- def PTOP4BSUDtrrV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4bsud_internal GR16:$row, GR16:$col, GR16:$k,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXHF8PStrriV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4mxhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXBHF8PStrriV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4mxbhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXHBF8PStrriV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4mxhbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXBF8PStrriV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4mxbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-
- let Constraints = "$src1 = $dst", Uses = [BSR0] in
- def PTOP4MXBSSPStrriV : PseudoI<(outs TILE:$dst),
- (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3),
- [(set TILE:$dst,
- (int_x86_acev1_top4mxbssps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
- TILE:$src1, VR512:$src2, VR512:$src3))]>;
-}
+// ACE twin of the AMX tile-zero user-facing pseudo.
+let SchedRW = [WriteSystem], usesCustomInserter = 1 in
+def ACE_PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
+ [(int_x86_acev1_tilezero timm:$src)]>;
// ACE twin of the AMX tile-zero RA pseudo, so the ACE model reaches MIR.
let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
@@ -342,95 +275,6 @@ def ACE_PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
[(set TILE:$dst, (int_x86_acev1_tilezero_internal
GR16:$src1, GR16:$src2))]>;
-// ACE twins of the AMX row-read RA pseudos, so the ACE model reaches MIR.
-let isPseudo = true, hasNoSchedulingInfo = 1, SchedRW = [WriteSystem],
- usesCustomInserter = 1 in {
- def ACE_PTCVTROWD2PSrtiV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- i32u6imm:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowd2ps_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- imm:$src4))]>;
- def ACE_PTCVTROWD2PSrteV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowd2ps_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4))]>;
- def ACE_PTCVTROWPS2BF16HrtiV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- i32u6imm:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2bf16h_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- imm:$src4))]>;
- def ACE_PTCVTROWPS2BF16HrteV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2bf16h_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4))]>;
- def ACE_PTCVTROWPS2BF16LrtiV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- i32u6imm:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2bf16l_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- imm:$src4))]>;
- def ACE_PTCVTROWPS2BF16LrteV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2bf16l_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4))]>;
- def ACE_PTCVTROWPS2PHHrtiV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- i32u6imm:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2phh_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- imm:$src4))]>;
- def ACE_PTCVTROWPS2PHHrteV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2phh_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4))]>;
- def ACE_PTCVTROWPS2PHLrtiV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- i32u6imm:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2phl_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- imm:$src4))]>;
- def ACE_PTCVTROWPS2PHLrteV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tcvtrowps2phl_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4))]>;
- def ACE_PTILEMOVROWrtiV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- i32u6imm:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tilemovrowextract_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- imm:$src4))]>;
- def ACE_PTILEMOVROWrteV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4),
- [(set VR512:$dst,
- (int_x86_acev1_tilemovrowextract_internal
- GR16:$src1, GR16:$src2, TILE:$src3,
- GR32:$src4))]>;
-}
-
// Pseudo instructions for tile spill and reload (register allocation)
let isPseudo = true, hasNoSchedulingInfo = 1 in {
let mayStore = 1 in
@@ -441,39 +285,3 @@ let isPseudo = true, hasNoSchedulingInfo = 1 in {
}
} // end HasACEV1, In64BitMode
-
-// ACE twins of the user-facing DirectReg pseudos (immediate tile IDs).
-// They expand like the AMX pseudos; the opcode is what records the ACE model.
-class ACERowReadPseudo<Intrinsic intr>
- : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2),
- [(set VR512:$dst, (intr timm:$src1, GR32:$src2))]>;
-
-// One ACE intrinsic covers both row index forms, so a constant index matches
-// the immediate encoding here and anything else falls back to the register one.
-class ACERowReadPseudoI<Intrinsic intr>
- : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
- [(set VR512:$dst, (intr timm:$src1, imm:$src2))]>;
-
-let Predicates = [HasACEV1, In64BitMode], SchedRW = [WriteSystem],
- usesCustomInserter = 1 in {
- def ACE_PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
- [(int_x86_acev1_tilezero timm:$src)]>;
-
- def ACE_PTCVTROWD2PSrte : ACERowReadPseudo<int_x86_acev1_tcvtrowd2ps>;
- def ACE_PTCVTROWPS2BF16Hrte
- : ACERowReadPseudo<int_x86_acev1_tcvtrowps2bf16h>;
- def ACE_PTCVTROWPS2BF16Lrte
- : ACERowReadPseudo<int_x86_acev1_tcvtrowps2bf16l>;
- def ACE_PTCVTROWPS2PHHrte : ACERowReadPseudo<int_x86_acev1_tcvtrowps2phh>;
- def ACE_PTCVTROWPS2PHLrte : ACERowReadPseudo<int_x86_acev1_tcvtrowps2phl>;
- def ACE_PTILEMOVROWrte : ACERowReadPseudo<int_x86_acev1_tilemovrowextract>;
-
- def ACE_PTCVTROWD2PSrti : ACERowReadPseudoI<int_x86_acev1_tcvtrowd2ps>;
- def ACE_PTCVTROWPS2BF16Hrti
- : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2bf16h>;
- def ACE_PTCVTROWPS2BF16Lrti
- : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2bf16l>;
- def ACE_PTCVTROWPS2PHHrti : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2phh>;
- def ACE_PTCVTROWPS2PHLrti : ACERowReadPseudoI<int_x86_acev1_tcvtrowps2phl>;
- def ACE_PTILEMOVROWrti : ACERowReadPseudoI<int_x86_acev1_tilemovrowextract>;
-}
More information about the cfe-commits
mailing list