[clang] [llvm] [X86] Add ACE v1 (AI Compute Extensions) support (PR #208408)
via cfe-commits
cfe-commits at lists.llvm.org
Sat Sep 5 10:01:29 PDT 2026
https://github.com/ganeshgit updated https://github.com/llvm/llvm-project/pull/208408
>From 1194f3364e5ace066dd69c8c6e1c59b78c0125ba Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Wed, 8 Jul 2026 00:41:01 +0530
Subject: [PATCH 1/2] [X86] Add ACE v1 (AI Compute Extensions) support
ACE v1 introduces new tile-based matrix operations with fixed 16x64
tile dimensions (Palette 2), unlike AMX's configurable dimensions.
Key features:
- New __acetile type with fixed 16x64 dimensions
- __tile_ace_* intrinsics for ACE tile operations
- Outer product instructions: TOP4BUUD, TOP4BUSD, TOP4BSSD, TOP4BSUD, TOP2BF16PS
- Mixed precision FP8 instructions: TOP4MX variants
- Tile movement: TILEMOVROW, TILEMOVCOL for ZMM<->tile transfers
- BSR (Block Scale Register) operations for scaling factors
- ACE tile spill/reload using TILEMOVROW + VMOVUPS row-by-row
(ACE doesn't have TILELOADD/TILESTORED instructions)
Backend changes:
- X86LowerTileCopy handles ACE tile copies
- X86PreTileConfig/X86FastPreTileConfig for ACE register allocation
- Palette 2 tile configuration support
Reused from AMX:
- The x86_amx type
- AMX-AVX512f definitions and TILE instruction definitions
- Passes for lowering and handling the x86_amx type
- Wrappers internally use rows and cols to use amx definitions
This PR builds on PR #206888 and should be reviewed after/with it.
---
clang/include/clang/Basic/BuiltinsX86_64.td | 71 +-
clang/include/clang/Options/Options.td | 2 +
clang/lib/Basic/Targets/X86.cpp | 6 +
clang/lib/Basic/Targets/X86.h | 1 +
clang/lib/Headers/CMakeLists.txt | 1 +
clang/lib/Headers/acev1intrin.h | 766 ++++++++++++++++++
clang/lib/Headers/cpuid.h | 2 +
clang/lib/Headers/immintrin.h | 2 +
clang/lib/Sema/SemaX86.cpp | 5 +
clang/test/CodeGen/X86/ace-api.c | 145 ++++
clang/test/CodeGen/X86/ace-builtins.c | 119 +++
clang/test/CodeGen/X86/ace-inline-asm.c | 85 ++
clang/test/CodeGen/X86/acev1.c | 38 +
clang/test/CodeGen/X86/acev1_api.c | 138 ++++
clang/test/CodeGen/X86/acev1_bsr.c | 63 ++
clang/test/CodeGen/X86/acev1_errors.c | 17 +
clang/test/CodeGen/X86/acev1_inline_asm.c | 76 ++
clang/test/CodeGen/X86/acev1_mxfp8.c | 51 ++
clang/test/CodeGen/X86/acev1_tile_movement.c | 113 +++
clang/test/CodeGen/attr-target-x86.c | 4 +-
clang/test/Preprocessor/x86_target_features.c | 7 +
llvm/include/llvm/IR/IntrinsicsX86.td | 198 +++++
.../llvm/TargetParser/X86TargetParser.def | 1 +
.../lib/Target/X86/AsmParser/X86AsmParser.cpp | 50 ++
.../X86/MCTargetDesc/X86ATTInstPrinter.cpp | 92 ++-
.../X86/MCTargetDesc/X86ATTInstPrinter.h | 1 +
.../X86/MCTargetDesc/X86IntelInstPrinter.cpp | 88 +-
.../X86/MCTargetDesc/X86IntelInstPrinter.h | 1 +
llvm/lib/Target/X86/X86.td | 3 +
llvm/lib/Target/X86/X86ExpandPseudo.cpp | 89 ++
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 105 ++-
llvm/lib/Target/X86/X86FastTileConfig.cpp | 17 +-
llvm/lib/Target/X86/X86FrameLowering.cpp | 5 +-
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 19 +
llvm/lib/Target/X86/X86ISelLowering.cpp | 290 ++++++-
llvm/lib/Target/X86/X86InstrACE.td | 417 ++++++++++
llvm/lib/Target/X86/X86InstrAMX.td | 67 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 8 +-
llvm/lib/Target/X86/X86InstrInfo.td | 3 +
llvm/lib/Target/X86/X86InstrPredicates.td | 3 +
llvm/lib/Target/X86/X86LowerAMXType.cpp | 144 +++-
llvm/lib/Target/X86/X86LowerTileCopy.cpp | 200 +++--
.../lib/Target/X86/X86MachineFunctionInfo.cpp | 4 +-
llvm/lib/Target/X86/X86MachineFunctionInfo.h | 22 +
llvm/lib/Target/X86/X86PreTileConfig.cpp | 10 +-
llvm/lib/Target/X86/X86RegisterInfo.cpp | 17 +-
llvm/lib/Target/X86/X86RegisterInfo.td | 9 +
llvm/lib/Target/X86/X86TileConfig.cpp | 10 +-
llvm/lib/TargetParser/X86TargetParser.cpp | 1 +
llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll | 244 ++++++
llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll | 99 +++
.../CodeGen/X86/ACE/acev1-outer-product.ll | 204 +++++
llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll | 94 +++
llvm/test/CodeGen/X86/ipra-reg-usage.ll | 4 +-
llvm/test/MC/X86/ACE/ace-att.s | 109 +++
llvm/test/MC/X86/ACE/ace-error.s | 39 +
llvm/test/MC/X86/ACE/ace-intel.s | 109 +++
llvm/test/TableGen/x86-fold-tables.inc | 5 +
58 files changed, 4362 insertions(+), 131 deletions(-)
create mode 100644 clang/lib/Headers/acev1intrin.h
create mode 100644 clang/test/CodeGen/X86/ace-api.c
create mode 100644 clang/test/CodeGen/X86/ace-builtins.c
create mode 100644 clang/test/CodeGen/X86/ace-inline-asm.c
create mode 100644 clang/test/CodeGen/X86/acev1.c
create mode 100644 clang/test/CodeGen/X86/acev1_api.c
create mode 100644 clang/test/CodeGen/X86/acev1_bsr.c
create mode 100644 clang/test/CodeGen/X86/acev1_errors.c
create mode 100644 clang/test/CodeGen/X86/acev1_inline_asm.c
create mode 100644 clang/test/CodeGen/X86/acev1_mxfp8.c
create mode 100644 clang/test/CodeGen/X86/acev1_tile_movement.c
create mode 100644 llvm/lib/Target/X86/X86InstrACE.td
create mode 100644 llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
create mode 100644 llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
create mode 100644 llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
create mode 100644 llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
create mode 100644 llvm/test/MC/X86/ACE/ace-att.s
create mode 100644 llvm/test/MC/X86/ACE/ace-error.s
create mode 100644 llvm/test/MC/X86/ACE/ace-intel.s
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index cf76d9a257614..f2f52f029455f 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -223,6 +223,9 @@ let Features = "amx-int8", Attributes = [NoThrow] in {
let Features = "amx-tile", Attributes = [NoThrow] in {
def tilestored64_internal : X86Builtin<"void(unsigned short, unsigned short, void *, size_t, _Vector<256, int>)">;
+}
+
+let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
def tilezero_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short)">;
}
@@ -239,7 +242,7 @@ let Features = "amx-complex", Attributes = [NoThrow] in {
def tcmmrlfp16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-avx512,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+let Features = "amx-avx512,avx10.2|acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def tcvtrowd2ps_internal : X86Builtin<"_Vector<16, float>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
def tcvtrowps2bf16h_internal : X86Builtin<"_Vector<32, __bf16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
def tcvtrowps2bf16l_internal : X86Builtin<"_Vector<32, __bf16>(unsigned short, unsigned short, _Vector<256, int>, unsigned int)">;
@@ -255,7 +258,7 @@ let Features = "amx-fp8", Attributes = [NoThrow] in {
def tdphf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-tile", Attributes = [NoThrow] in {
+let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
def tile_loadconfig : X86Builtin<"void(void const *)">;
def tile_storeconfig : X86Builtin<"void(void const *)">;
def tilerelease : X86Builtin<"void()">;
@@ -293,7 +296,7 @@ let Features = "amx-complex", Attributes = [NoThrow] in {
def tcmmrlfp16ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
}
-let Features = "amx-avx512,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+let Features = "amx-avx512,avx10.2|acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def tcvtrowd2ps : X86Builtin<"_Vector<16, float>(_Constant unsigned char, unsigned int)">;
def tcvtrowps2bf16h : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, unsigned int)">;
def tcvtrowps2bf16l : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, unsigned int)">;
@@ -389,3 +392,65 @@ let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<256>]
let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def vmovrsw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short const *>)">;
}
+
+// ACE (AI Compute Extensions) Builtins
+// Tile movement - single builtin handles both immediate and register index forms
+// The lowering code checks if index is constant and selects appropriate instruction
+let Features = "acev1", Attributes = [NoThrow] in {
+ def tilesetcol : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow writes TO tile (AMX tilemovrow reads FROM tile)
+ def tilesetrow : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+}
+
+// Tile movement internal (for IR-level operations)
+let Features = "acev1", Attributes = [NoThrow] in {
+ def tilesetcol_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow_to_tile writes TO tile (AMX tilemovrow reads FROM tile)
+ def tilesetrow_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+}
+
+// BSR management
+let Features = "acev1", Attributes = [NoThrow] in {
+ def bsrinit : X86Builtin<"void()">;
+ def bsrmovf : X86Builtin<"void(_Vector<16, int>, _Vector<16, int>)">;
+ def bsrmovh_set : X86Builtin<"void(_Vector<16, int>)">;
+ def bsrmovh_get : X86Builtin<"_Vector<16, int>()">;
+ def bsrmovl_set : X86Builtin<"void(_Vector<16, int>)">;
+ def bsrmovl_get : X86Builtin<"_Vector<16, int>()">;
+}
+
+// Outer products - user-facing (immediate tile IDs)
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4buud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4busd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4bssd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4bsud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+}
+
+// Mixed precision outer products - user-facing
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+ def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+}
+
+// Outer products - internal (for IR-level operations with tile values)
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top2bf16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
+ def top4buud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4busd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bssd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bsud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
+}
+
+// Mixed precision internal
+let Features = "acev1", Attributes = [NoThrow] in {
+ def top4mxhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxbhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxhbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+ def top4mxbssps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
+}
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index 37e5c3199a003..51a3acde9fddb 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -7365,6 +7365,8 @@ def mamx_tile : Flag<["-"], "mamx-tile">, Group<m_x86_Features_Group>;
def mno_amx_tile : Flag<["-"], "mno-amx-tile">, Group<m_x86_Features_Group>;
def mamx_movrs: Flag<["-"], "mamx-movrs">, Group<m_x86_Features_Group>;
def mno_amx_movrs: Flag<["-"], "mno-amx-movrs">, Group<m_x86_Features_Group>;
+def macev1 : Flag<["-"], "macev1">, Group<m_x86_Features_Group>;
+def mno_acev1 : Flag<["-"], "mno-acev1">, Group<m_x86_Features_Group>;
def mcmpccxadd : Flag<["-"], "mcmpccxadd">, Group<m_x86_Features_Group>;
def mno_cmpccxadd : Flag<["-"], "mno-cmpccxadd">, Group<m_x86_Features_Group>;
def msse : Flag<["-"], "msse">, Group<m_x86_Features_Group>;
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 8ab39b750dc99..1d15a7c0f619e 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -408,6 +408,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
HasAMXMOVRS = true;
} else if (Feature == "+amx-avx512") {
HasAMXAVX512 = true;
+ } else if (Feature == "+acev1") {
+ HasACEV1 = true;
} else if (Feature == "+cmpccxadd") {
HasCMPCCXADD = true;
} else if (Feature == "+raoint") {
@@ -956,6 +958,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__AMX_MOVRS__");
if (HasAMXAVX512)
Builder.defineMacro("__AMX_AVX512__");
+ if (HasACEV1)
+ Builder.defineMacro("__ACEV1__");
if (HasCMPCCXADD)
Builder.defineMacro("__CMPCCXADD__");
if (HasRAOINT)
@@ -1095,6 +1099,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
.Case("amx-int8", true)
.Case("amx-movrs", true)
.Case("amx-tile", true)
+ .Case("acev1", true)
.Case("avx", true)
.Case("avx10.1", true)
.Case("avx10.2", true)
@@ -1217,6 +1222,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
.Case("amx-int8", HasAMXINT8)
.Case("amx-movrs", HasAMXMOVRS)
.Case("amx-tile", HasAMXTILE)
+ .Case("acev1", HasACEV1)
.Case("avx", SSELevel >= AVX)
.Case("avx10.1", HasAVX10_1)
.Case("avx10.2", HasAVX10_2)
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 658730ae88186..9d683c3ac6132 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -139,6 +139,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
bool HasAMXFP8 = false;
bool HasAMXMOVRS = false;
bool HasAMXAVX512 = false;
+ bool HasACEV1 = false;
bool HasSERIALIZE = false;
bool HasTSXLDTRK = false;
bool HasUSERMSR = false;
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 3d845423759ac..ff04b528dac2a 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -167,6 +167,7 @@ set(webassembly_files
set(x86_files
# Intrinsics
+ acev1intrin.h
adcintrin.h
adxintrin.h
ammintrin.h
diff --git a/clang/lib/Headers/acev1intrin.h b/clang/lib/Headers/acev1intrin.h
new file mode 100644
index 0000000000000..0491c7f0b383d
--- /dev/null
+++ b/clang/lib/Headers/acev1intrin.h
@@ -0,0 +1,766 @@
+/*===-------------- acev1intrin.h - ACEV1 intrinsics -*- C/C++ -*------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===------------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <acev1intrin.h> directly; include <immintrin.h> instead."
+#endif /* __IMMINTRIN_H */
+
+#ifndef __ACEV1INTRIN_H
+#define __ACEV1INTRIN_H
+#ifdef __x86_64__
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS_ACE \
+ __attribute__((__always_inline__, __nodebug__, __target__("acev1")))
+
+/// Load tile configuration from a 64-byte memory location. For ACE
+/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1),
+/// ACE tiles have fixed dimensions of 16 rows × 64 bytes, so per-tile
+/// row/column configuration bytes are ignored. If palette_id is zero,
+/// tiles return to init state and are zeroed. Invalid configurations
+/// result in #GP fault.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> LDTILECFG </c> instruction.
+///
+/// \param __config
+/// A pointer to 64-byte tile configuration (use __ace_tile_config).
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_tile_ace_loadconfig(const void *__config) {
+ __builtin_ia32_tile_loadconfig(__config);
+}
+
+/// Store the current tile configuration to a 64-byte memory location.
+/// For ACE (Palette 2), the stored palette_id will be 2 and per-tile
+/// configuration bytes reflect the fixed 16×64 dimensions. If tiles
+/// are not configured, all zeroes are stored.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> STTILECFG </c> instruction.
+///
+/// \param __config
+/// A pointer to 64-byte tile configuration buffer.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+_tile_ace_storeconfig(void *__config) {
+ __builtin_ia32_tile_storeconfig(__config);
+}
+
+/// Release the tile configuration to return to init state, releasing
+/// all tile storage. After this, tiles must be reconfigured with
+/// _tile_ace_loadconfig before use.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILERELEASE </c> instruction.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
+ __builtin_ia32_tilerelease();
+}
+
+/// Zero the ACE tile specified by "tile". Sets all 1024 bytes
+/// (16 rows × 64 bytes) of the tile register to zero.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEZERO </c> instruction.
+///
+/// \param tile
+/// Destination tile register ID (0-7).
+#define _tile_ace_zero(tile) __builtin_ia32_tilezero((tile))
+
+/// Move a 64-byte ZMM vector to a tile column. The 16 doublewords from
+/// the ZMM are written as a vertical column in the tile at the specified
+/// index.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVCOL </c> instruction.
+///
+/// \param dst
+/// Destination tile register ID (0-7).
+/// \param src
+/// Source ZMM register ID (0-31).
+/// \param idx
+/// Column index (0-15). Immediate or register form selected automatically.
+#define _tile_setcol(dst, src, idx) \
+ __builtin_ia32_tilesetcol((dst), (src), (idx))
+
+/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written
+/// as a horizontal row in the tile at the specified index.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param dst
+/// Destination tile register ID (0-7).
+/// \param src
+/// Source ZMM register ID (0-31).
+/// \param idx
+/// Row index (0-15). Immediate or register form selected automatically.
+#define _tile_setrow(dst, src, idx) \
+ __builtin_ia32_tilesetrow((dst), (src), (idx))
+
+/// Initialize the Block Scale Register (BSR) to zero. The BSR holds
+/// 32 scaling factors used by mixed-precision outer product instructions
+/// (TOP4MX* variants). Must be called before using BSR-scaled operations.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRINIT </c> instruction.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) {
+ __builtin_ia32_bsrinit();
+}
+
+/// Load the full BSR (32 scale factors) from two ZMM registers. The low
+/// half (16 factors) comes from __src1, high half from __src2. Each
+/// doubleword contains one scale factor.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVF </c> instruction.
+///
+/// \param __src1
+/// ZMM with scale factors for BSR low half (factors 0-15).
+/// \param __src2
+/// ZMM with scale factors for BSR high half (factors 16-31).
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1,
+ __m512i __src2) {
+ __builtin_ia32_bsrmovf((__v16si)__src1, (__v16si)__src2);
+}
+
+/// Load the high half of BSR (scale factors 16-31) from a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
+///
+/// \param __src
+/// ZMM with 16 scale factors to write to BSR high half.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) {
+ __builtin_ia32_bsrmovh_set((__v16si)__src);
+}
+
+/// Read the high half of BSR (scale factors 16-31) to a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
+///
+/// \returns
+/// ZMM containing 16 scale factors from BSR high half.
+static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) {
+ return (__m512i)__builtin_ia32_bsrmovh_get();
+}
+
+/// Load the low half of BSR (scale factors 0-15) from a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
+///
+/// \param __src
+/// ZMM with 16 scale factors to write to BSR low half.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) {
+ __builtin_ia32_bsrmovl_set((__v16si)__src);
+}
+
+/// Read the low half of BSR (scale factors 0-15) to a ZMM register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
+///
+/// \returns
+/// ZMM containing 16 scale factors from BSR low half.
+static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
+ return (__m512i)__builtin_ia32_bsrmovl_get();
+}
+
+/// Compute 2-way outer product of BF16 pairs, accumulating to FP32.
+/// Each BF16 pair from src1 and src2 produces two FP32 products that
+/// are accumulated into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP2BF16PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 32 BF16 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 32 BF16 values.
+#define _tile_top2bf16ps(dst, src1, src2) \
+ __builtin_ia32_top2bf16ps((dst), (src1), (src2))
+
+/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Each group of 4 unsigned byte pairs from src1 and src2 produces
+/// 4 products accumulated into the destination tile as 32-bit integers.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUUD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+#define _tile_top4buud(dst, src1, src2) \
+ __builtin_ia32_top4buud((dst), (src1), (src2))
+
+/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Each group of 4 byte pairs (unsigned from src1, signed from src2)
+/// produces 4 products accumulated into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUSD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+#define _tile_top4busd(dst, src1, src2) \
+ __builtin_ia32_top4busd((dst), (src1), (src2))
+
+/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Each group of 4 signed byte pairs produces 4 products accumulated
+/// into the destination tile as 32-bit integers.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSSD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+#define _tile_top4bssd(dst, src1, src2) \
+ __builtin_ia32_top4bssd((dst), (src1), (src2))
+
+/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Each group of 4 byte pairs (signed from src1, unsigned from src2)
+/// produces 4 products accumulated into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSUD </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+#define _tile_top4bsud(dst, src1, src2) \
+ __builtin_ia32_top4bsud((dst), (src1), (src2))
+
+/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
+/// Multiplies HF8 values from src1 with BF8 values from src2, converting
+/// to FP32 and accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing HF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing BF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxhf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxhf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product with BF8/HF8 format.
+/// Multiplies BF8 values from src1 with HF8 (E4M3) values from src2,
+/// converting to FP32 and accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBHF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing BF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing HF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxbhf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbhf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product with HF8/BF8 format.
+/// Multiplies HF8 (E4M3) values from src1 with BF8 values from src2,
+/// converting to FP32 and accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHBF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing HF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing BF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxhbf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxhbf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
+/// Multiplies BF8 values from both sources, converting to FP32 and
+/// accumulating into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBF8PS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing BF8 values.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing BF8 values.
+/// \param imm
+/// 8-bit control immediate for scaling/rounding options.
+#define _tile_top4mxbf8ps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbf8ps((dst), (src1), (src2), (imm))
+
+/// Compute 4-way mixed precision outer product of signed INT8 with BSR
+/// scaling. Multiplies signed bytes, applies scale factors from the BSR,
+/// and accumulates FP32 results into the destination tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBSSPS </c> instruction.
+///
+/// \param dst
+/// Destination/accumulator tile register ID (0-7).
+/// \param src1
+/// First source ZMM register ID (0-31) containing signed bytes.
+/// \param src2
+/// Second source ZMM register ID (0-31) containing signed bytes.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define _tile_top4mxbssps(dst, src1, src2, imm) \
+ __builtin_ia32_top4mxbssps((dst), (src1), (src2), (imm))
+
+/// ACE tile type with fixed dimensions (16 rows × 64 bytes = 1024 bytes).
+/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1.
+typedef int __acetile __attribute__((__vector_size__(1024), __aligned__(64)));
+
+/// ACE Palette 2 tile configuration structure (64 bytes).
+/// For ACE, only byte 0 (palette_id = 2) is significant; bytes 1-63
+/// must be zero. Unlike AMX Palette 1, tile dimensions are fixed.
+typedef struct __attribute__((__packed__, __aligned__(64))) {
+ unsigned char palette_id;
+ unsigned char reserved[63];
+} __ace_tile_config;
+
+/// Initialize an ACE tile configuration structure for Palette 2.
+/// Sets palette_id to 2 and clears all reserved bytes to zero.
+/// Use with _tile_ace_loadconfig to configure tiles for ACE operations.
+static __inline__ void __DEFAULT_FN_ATTRS_ACE
+__ace_init_config(__ace_tile_config *cfg) {
+ for (int i = 0; i < 64; i++)
+ ((unsigned char *)cfg)[i] = 0;
+ cfg->palette_id = 2;
+}
+
+/// Zero an ACE tile variable. Sets all 1024 bytes to zero.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEZERO </c> instruction.
+///
+/// \param dst
+/// Pointer to __acetile variable to be zeroed.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_zero(__acetile *dst) {
+ *dst = __builtin_ia32_tilezero_internal(16, 64);
+}
+
+/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Multiplies 64 unsigned bytes from each ZMM source, producing a
+/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUUD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 unsigned bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4buud(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4buud_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Multiplies 64 bytes from each ZMM source (unsigned from src1,
+/// signed from src2), producing a 16×16 grid of 32-bit sums.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BUSD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 unsigned bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 signed bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4busd(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4busd_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Multiplies 64 signed bytes from each ZMM source, producing a
+/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSSD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 signed bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4bssd(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4bssd_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Multiplies 64 bytes from each ZMM source (signed from src1,
+/// unsigned from src2), producing a 16×16 grid of 32-bit sums.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4BSUD </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing 64 signed bytes.
+/// \param src2
+/// Second source ZMM vector containing 64 unsigned bytes.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top4bsud(__acetile *dst, __m512i src1,
+ __m512i src2) {
+ *dst = __builtin_ia32_top4bsud_internal(16, 64, 64, *dst, (__v64qs)src1,
+ (__v64qs)src2);
+}
+
+/// Compute 2-way outer product of BF16 to FP32.
+/// Multiplies 32 BF16 values from each source, producing a 16×16 grid
+/// of FP32 sums accumulated into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP2BF16PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector (__m512bh) containing 32 BF16 values.
+/// \param src2
+/// Second source ZMM vector (__m512bh) containing 32 BF16 values.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
+ __m512bh src2) {
+ *dst = __builtin_ia32_top2bf16ps_internal(16, 64, 64, *dst, (__v32bf)src1,
+ (__v32bf)src2);
+}
+
+/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
+/// Multiplies HF8 values from src1 with HF8 values from src2, applies
+/// BSR scaling, converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing HF8 (E4M3) values.
+/// \param src2
+/// Second source ZMM vector containing HF8 (E4M3) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxhf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxhf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product with BF8/HF8 format.
+/// Multiplies BF8 (E5M2) values from src1 with HF8 (E4M3) values from src2,
+/// applies BSR scaling, converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBHF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing BF8 (E5M2) values.
+/// \param src2
+/// Second source ZMM vector containing HF8 (E4M3) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxbhf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxbhf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product with HF8/BF8 format.
+/// Multiplies HF8 (E4M3) values from src1 with BF8 (E5M2) values from src2,
+/// applies BSR scaling, converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXHBF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing HF8 (E4M3) values.
+/// \param src2
+/// Second source ZMM vector containing BF8 (E5M2) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxhbf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxhbf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
+/// Multiplies BF8 values from both sources, applies BSR scaling,
+/// converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBF8PS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing BF8 (E5M2) values.
+/// \param src2
+/// Second source ZMM vector containing BF8 (E5M2) values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxbf8ps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxbf8ps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Compute 4-way mixed precision outer product of MX INT8 with BSR scaling.
+/// Multiplies signed MX INT8 bytes from both sources, applies BSR scaling,
+/// converts to FP32 and accumulates into the ACE tile.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TOP4MXBSSPS </c> instruction.
+///
+/// \param dst
+/// Pointer to destination/accumulator __acetile.
+/// \param src1
+/// First source ZMM vector containing signed MX INT8 values.
+/// \param src2
+/// Second source ZMM vector containing signed MX INT8 values.
+/// \param imm
+/// 8-bit immediate selecting BSR scale factors to apply.
+#define __tile_ace_top4mxbssps(dst, src1, src2, imm) \
+ (*(dst) = __builtin_ia32_top4mxbssps_internal( \
+ 16, 64, 64, (imm), *(dst), (__v16si)(src1), (__v16si)(src2)))
+
+/// Write a ZMM vector as a column in an ACE tile.
+/// The 16 doublewords from src are written vertically at column idx.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVCOL </c> instruction.
+///
+/// \param dst
+/// Pointer to destination __acetile.
+/// \param src
+/// Source ZMM vector (__m512i) with 16 doublewords.
+/// \param idx
+/// Column index (0-15).
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
+ unsigned int idx) {
+ *dst = __builtin_ia32_tilesetcol_internal(16, 64, (__v16si)src, idx);
+}
+
+/// Write a ZMM vector as a row in an ACE tile.
+/// The 64 bytes from src are written horizontally at row idx.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param dst
+/// Pointer to destination __acetile.
+/// \param src
+/// Source ZMM vector (__m512i) with 64 bytes.
+/// \param idx
+/// Row index (0-15).
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ void __tile_ace_setrow(__acetile *dst, __m512i src,
+ unsigned int idx) {
+ *dst = __builtin_ia32_tilesetrow_internal(16, 64, (__v16si)src, idx);
+}
+
+/// Read a row from an ACE tile to a ZMM vector.
+/// Returns the 64 bytes at row idx as a ZMM vector.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512i) containing the 64-byte row.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512i __tile_ace_getrow(__acetile *src, unsigned int idx) {
+ return (__m512i)__builtin_ia32_tilemovrow_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert INT32 elements to FP32.
+/// Each of the 16 INT32 elements in the row is converted to FP32.
+/// Rounding uses RTNE (round to nearest even).
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWD2PS </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing INT32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512) containing 16 FP32 values.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512 __tile_ace_cvtrowd2ps(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowd2ps_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to BF16 (high).
+/// Each FP32 element is converted to BF16 and placed in the high 16 bits
+/// of each destination dword; low 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2BF16H </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512bh) with BF16 values in high half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512bh __tile_ace_cvtrowps2bf16h(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2bf16h_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to BF16 (low).
+/// Each FP32 element is converted to BF16 and placed in the low 16 bits
+/// of each destination dword; high 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2BF16L </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512bh) with BF16 values in low half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512bh __tile_ace_cvtrowps2bf16l(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2bf16l_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to FP16 (high).
+/// Each FP32 element is converted to FP16 and placed in the high 16 bits
+/// of each destination dword; low 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2PHH </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512h) with FP16 values in high half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512h __tile_ace_cvtrowps2phh(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2phh_internal(16, 64, *src, idx);
+}
+
+/// Read a row from an ACE tile and convert FP32 elements to FP16 (low).
+/// Each FP32 element is converted to FP16 and placed in the low 16 bits
+/// of each destination dword; high 16 bits are zeroed.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCVTROWPS2PHL </c> instruction.
+///
+/// \param src
+/// Pointer to source __acetile containing FP32 elements.
+/// \param idx
+/// Row index (0-15).
+/// \returns
+/// ZMM vector (__m512h) with FP16 values in low half of each dword.
+__DEFAULT_FN_ATTRS_ACE
+static __inline__ __m512h __tile_ace_cvtrowps2phl(__acetile *src,
+ unsigned int idx) {
+ return __builtin_ia32_tcvtrowps2phl_internal(16, 64, *src, idx);
+}
+
+#undef __DEFAULT_FN_ATTRS_ACE
+
+#endif /* __x86_64__ */
+#endif /* __ACEV1INTRIN_H */
diff --git a/clang/lib/Headers/cpuid.h b/clang/lib/Headers/cpuid.h
index 8017db9f6a080..d5ffc7babdf2c 100644
--- a/clang/lib/Headers/cpuid.h
+++ b/clang/lib/Headers/cpuid.h
@@ -222,6 +222,8 @@
#define bit_AVX10 0x00080000
#define bit_APXF 0x00200000
+/* Features in %ecx for leaf 7 sub-leaf 1 */
+#define bit_ACEV1 0x00000800
/* Features in %eax for leaf 13 sub-leaf 1 */
#define bit_XSAVEOPT 0x00000001
#define bit_XSAVEC 0x00000002
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index c9be46af22c29..35c3ea0d64d3a 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -483,6 +483,8 @@ _storebe_i64(void * __P, long long __D) {
#include <amxavx512intrin.h>
+#include <acev1intrin.h>
+
#include <avx512vp2intersectintrin.h>
#include <avx512vlvp2intersectintrin.h>
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index cfd71d366e6fa..94c2257b970aa 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -508,6 +508,11 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tdpbhf8ps:
case X86::BI__builtin_ia32_tdphbf8ps:
case X86::BI__builtin_ia32_tdphf8ps:
+ case X86::BI__builtin_ia32_top2bf16ps:
+ case X86::BI__builtin_ia32_top4buud:
+ case X86::BI__builtin_ia32_top4busd:
+ case X86::BI__builtin_ia32_top4bssd:
+ case X86::BI__builtin_ia32_top4bsud:
return CheckBuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
case X86::BI__builtin_ia32_tcvtrowps2bf16hi:
case X86::BI__builtin_ia32_tcvtrowps2bf16li:
diff --git a/clang/test/CodeGen/X86/ace-api.c b/clang/test/CodeGen/X86/ace-api.c
new file mode 100644
index 0000000000000..8f82fbf66e39a
--- /dev/null
+++ b/clang/test/CodeGen/X86/ace-api.c
@@ -0,0 +1,145 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
+// RUN: -emit-llvm -o - -Werror | FileCheck %s
+
+// Test ACE struct-based API functions using __acetile type
+// ACE v1 uses fixed 16x64 tile dimensions (Palette 2)
+
+#include <immintrin.h>
+
+// CHECK-LABEL: @test_ace_top4buud
+// CHECK: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %{{.*}})
+// CHECK: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+// CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %{{.*}})
+void test_ace_top4buud(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4buud(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4busd
+// CHECK: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+void test_ace_top4busd(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4busd(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4bssd
+// CHECK: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+void test_ace_top4bssd(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4bssd(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4bsud
+// CHECK: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+void test_ace_top4bsud(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4bsud(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top2bf16ps
+// CHECK: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
+void test_ace_top2bf16ps(__acetile *dst, __m512bh a, __m512bh b) {
+ __tile_ace_top2bf16ps(dst, a, b);
+}
+
+// CHECK-LABEL: @test_ace_top4mxhf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxhf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxhf8ps(dst, a, b, 5);
+}
+
+// CHECK-LABEL: @test_ace_top4mxbhf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxbhf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxbhf8ps(dst, a, b, 3);
+}
+
+// CHECK-LABEL: @test_ace_top4mxhbf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxhbf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxhbf8ps(dst, a, b, 7);
+}
+
+// CHECK-LABEL: @test_ace_top4mxbf8ps
+// CHECK: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxbf8ps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxbf8ps(dst, a, b, 1);
+}
+
+// CHECK-LABEL: @test_ace_top4mxbssps
+// CHECK: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_ace_top4mxbssps(__acetile *dst, __m512i a, __m512i b) {
+ __tile_ace_top4mxbssps(dst, a, b, 9);
+}
+
+// CHECK-LABEL: @test_ace_setcol
+// CHECK: call x86_amx @llvm.x86.tilesetcol.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
+void test_ace_setcol(__acetile *dst, __m512i src) {
+ __tile_ace_setcol(dst, src, 3);
+}
+
+// CHECK-LABEL: @test_ace_setrow
+// CHECK: call x86_amx @llvm.x86.tilesetrow.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
+void test_ace_setrow(__acetile *dst, __m512i src) {
+ __tile_ace_setrow(dst, src, 5);
+}
+
+// CHECK-LABEL: @test_ace_getrow
+// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512i test_ace_getrow(__acetile *src) {
+ return __tile_ace_getrow(src, 7);
+}
+
+// CHECK-LABEL: @test_ace_zero
+// CHECK: call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+void test_ace_zero(__acetile *dst) {
+ __tile_ace_zero(dst);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowd2ps
+// CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512 test_ace_cvtrowd2ps(__acetile *src) {
+ return __tile_ace_cvtrowd2ps(src, 2);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2bf16h
+// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512bh test_ace_cvtrowps2bf16h(__acetile *src) {
+ return __tile_ace_cvtrowps2bf16h(src, 4);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2bf16l
+// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512bh test_ace_cvtrowps2bf16l(__acetile *src) {
+ return __tile_ace_cvtrowps2bf16l(src, 6);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2phh
+// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512h test_ace_cvtrowps2phh(__acetile *src) {
+ return __tile_ace_cvtrowps2phh(src, 8);
+}
+
+// CHECK-LABEL: @test_ace_cvtrowps2phl
+// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
+__m512h test_ace_cvtrowps2phl(__acetile *src) {
+ return __tile_ace_cvtrowps2phl(src, 10);
+}
+
+// CHECK-LABEL: @test_ace_workflow
+// CHECK: call x86_amx @llvm.x86.tilezero.internal
+// CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+// CHECK: call x86_amx @llvm.x86.top4buud.internal
+// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+void test_ace_workflow(__m512i *input, __m512i *output) {
+ __acetile acc;
+
+ // Zero the tile
+ __tile_ace_zero(&acc);
+
+ // Initialize with data via setrow
+ __tile_ace_setrow(&acc, input[0], 0);
+
+ // Perform outer product computation
+ __tile_ace_top4buud(&acc, input[1], input[2]);
+
+ // Extract result via getrow
+ output[0] = __tile_ace_getrow(&acc, 0);
+}
diff --git a/clang/test/CodeGen/X86/ace-builtins.c b/clang/test/CodeGen/X86/ace-builtins.c
new file mode 100644
index 0000000000000..edaa80c78486f
--- /dev/null
+++ b/clang/test/CodeGen/X86/ace-builtins.c
@@ -0,0 +1,119 @@
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +acev1 -target-feature +avx512f -emit-llvm -o - | FileCheck %s
+
+#include <immintrin.h>
+
+// BSR Management Tests
+
+// CHECK-LABEL: @test_bsrinit
+// CHECK: call void @llvm.x86.bsrinit()
+void test_bsrinit(void) {
+ _bsr0_init();
+}
+
+// CHECK-LABEL: @test_bsrmovf
+// CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+void test_bsrmovf(__m512i a, __m512i b) {
+ _bsr0_movf(a, b);
+}
+
+// CHECK-LABEL: @test_bsrmovh_set
+// CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
+void test_bsrmovh_set(__m512i a) {
+ _bsr0_movh_set(a);
+}
+
+// CHECK-LABEL: @test_bsrmovh_get
+// CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+__m512i test_bsrmovh_get(void) {
+ return _bsr0_movh_get();
+}
+
+// CHECK-LABEL: @test_bsrmovl_set
+// CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
+void test_bsrmovl_set(__m512i a) {
+ _bsr0_movl_set(a);
+}
+
+// CHECK-LABEL: @test_bsrmovl_get
+// CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+__m512i test_bsrmovl_get(void) {
+ return _bsr0_movl_get();
+}
+
+// Tile Movement Tests
+
+// CHECK-LABEL: @test_tilemovcol
+// CHECK: call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %{{.*}}, i32 5)
+void test_tilemovcol(__m512i src) {
+ _tile_setcol(0, src, 5);
+}
+
+// CHECK-LABEL: @test_tilemovrow_to_tile
+// CHECK: call void @llvm.x86.tilesetrow(i8 0, <16 x i32> %{{.*}}, i32 5)
+void test_tilemovrow_to_tile(__m512i src) {
+ _tile_setrow(0, src, 5);
+}
+
+// Outer Product Tests
+
+// CHECK-LABEL: @test_top2bf16ps
+// CHECK: call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
+void test_top2bf16ps(void) {
+ _tile_top2bf16ps(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4buud
+// CHECK: call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
+void test_top4buud(void) {
+ _tile_top4buud(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4busd
+// CHECK: call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
+void test_top4busd(void) {
+ _tile_top4busd(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4bssd
+// CHECK: call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
+void test_top4bssd(void) {
+ _tile_top4bssd(0, 1, 2);
+}
+
+// CHECK-LABEL: @test_top4bsud
+// CHECK: call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
+void test_top4bsud(void) {
+ _tile_top4bsud(0, 1, 2);
+}
+
+// Mixed Precision Outer Product Tests
+
+// CHECK-LABEL: @test_top4mxhf8ps
+// CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxhf8ps(void) {
+ _tile_top4mxhf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxbhf8ps
+// CHECK: call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxbhf8ps(void) {
+ _tile_top4mxbhf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxhbf8ps
+// CHECK: call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxhbf8ps(void) {
+ _tile_top4mxhbf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxbf8ps
+// CHECK: call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxbf8ps(void) {
+ _tile_top4mxbf8ps(0, 1, 2, 5);
+}
+
+// CHECK-LABEL: @test_top4mxbssps
+// CHECK: call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
+void test_top4mxbssps(void) {
+ _tile_top4mxbssps(0, 1, 2, 5);
+}
diff --git a/clang/test/CodeGen/X86/ace-inline-asm.c b/clang/test/CodeGen/X86/ace-inline-asm.c
new file mode 100644
index 0000000000000..27e36d84149fc
--- /dev/null
+++ b/clang/test/CodeGen/X86/ace-inline-asm.c
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f \
+// RUN: -emit-llvm -o - -Wall -Werror | FileCheck %s
+
+// Test ACE instructions in inline assembly
+
+// BSR Management Instructions
+
+// CHECK-LABEL: @test_bsrinit_asm
+// CHECK: call void asm sideeffect "bsrinit", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrinit_asm(void) {
+ __asm__ volatile("bsrinit");
+}
+
+// CHECK-LABEL: @test_bsrmovf_asm
+// CHECK: call void asm sideeffect "bsrmovf %zmm1, %zmm0", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrmovf_asm(void) {
+ __asm__ volatile("bsrmovf %%zmm1, %%zmm0" :::);
+}
+
+// CHECK-LABEL: @test_bsrmovh_asm
+// CHECK: call void asm sideeffect "bsrmovh %zmm2", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrmovh_asm(void) {
+ __asm__ volatile("bsrmovh %%zmm2" :::);
+}
+
+// CHECK-LABEL: @test_bsrmovl_asm
+// CHECK: call void asm sideeffect "bsrmovl %zmm3", "~{dirflag},~{fpsr},~{flags}"()
+void test_bsrmovl_asm(void) {
+ __asm__ volatile("bsrmovl %%zmm3" :::);
+}
+
+// Tile Movement Instructions
+
+// CHECK-LABEL: @test_tilemovcol_asm
+// CHECK: call void asm sideeffect "tilemovcol $$$$5, %zmm0, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
+void test_tilemovcol_asm(void) {
+ __asm__ volatile("tilemovcol $$5, %%zmm0, %%tmm1" ::: "tmm1");
+}
+
+// CHECK-LABEL: @test_tilemovrow_asm
+// CHECK: call void asm sideeffect "tilemovrow $$$$3, %zmm0, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
+void test_tilemovrow_asm(void) {
+ __asm__ volatile("tilemovrow $$3, %%zmm0, %%tmm2" ::: "tmm2");
+}
+
+// Outer Product Instructions
+
+// CHECK-LABEL: @test_top4buud_asm
+// CHECK: call void asm sideeffect "top4buud %zmm1, %zmm0, %tmm0", "~{tmm0},~{dirflag},~{fpsr},~{flags}"()
+void test_top4buud_asm(void) {
+ __asm__ volatile("top4buud %%zmm1, %%zmm0, %%tmm0" ::: "tmm0");
+}
+
+// CHECK-LABEL: @test_top4bssd_asm
+// CHECK: call void asm sideeffect "top4bssd %zmm2, %zmm1, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
+void test_top4bssd_asm(void) {
+ __asm__ volatile("top4bssd %%zmm2, %%zmm1, %%tmm1" ::: "tmm1");
+}
+
+// CHECK-LABEL: @test_top2bf16ps_asm
+// CHECK: call void asm sideeffect "top2bf16ps %zmm3, %zmm2, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
+void test_top2bf16ps_asm(void) {
+ __asm__ volatile("top2bf16ps %%zmm3, %%zmm2, %%tmm2" ::: "tmm2");
+}
+
+// Mixed Precision Outer Product Instructions
+
+// CHECK-LABEL: @test_top4mxhfbps_asm
+// CHECK: call void asm sideeffect "top4mxhfbps $$$$5, %zmm1, %zmm0, %tmm3", "~{tmm3},~{dirflag},~{fpsr},~{flags}"()
+void test_top4mxhfbps_asm(void) {
+ __asm__ volatile("top4mxhfbps $$5, %%zmm1, %%zmm0, %%tmm3" ::: "tmm3");
+}
+
+// Full ACE operation sequence in inline assembly
+// CHECK-LABEL: @test_ace_sequence_asm
+// CHECK: call void asm sideeffect
+void test_ace_sequence_asm(void) {
+ __asm__ volatile(
+ "bsrinit \n\t"
+ "top4buud %%zmm1, %%zmm0, %%tmm0 \n\t"
+ "top4bssd %%zmm3, %%zmm2, %%tmm1 \n\t"
+ ::: "memory", "tmm0", "tmm1"
+ );
+}
diff --git a/clang/test/CodeGen/X86/acev1.c b/clang/test/CodeGen/X86/acev1.c
new file mode 100644
index 0000000000000..8f487b8ea2f51
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1.c
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests macro-based ACE v1 intrinsics emit correct LLVM IR
+
+#include <immintrin.h>
+
+void test_acev1_outer_products(void) {
+ // CHECK-LABEL: @test_acev1_outer_products
+ // CHECK: call void @llvm.x86.top2bf16ps(i8 0,
+ // CHECK: call void @llvm.x86.top4buud(i8 1,
+ // CHECK: call void @llvm.x86.top4busd(i8 2,
+ // CHECK: call void @llvm.x86.top4bssd(i8 3,
+ // CHECK: call void @llvm.x86.top4bsud(i8 4,
+ _tile_top2bf16ps(0, 1, 2);
+ _tile_top4buud(1, 2, 3);
+ _tile_top4busd(2, 3, 4);
+ _tile_top4bssd(3, 4, 5);
+ _tile_top4bsud(4, 5, 6);
+}
+
+void test_acev1_tile_config(void *data) {
+ // CHECK-LABEL: @test_acev1_tile_config
+ // CHECK: call void @llvm.x86.ldtilecfg(ptr %{{.*}})
+ // CHECK: call void @llvm.x86.sttilecfg(ptr %{{.*}})
+ // CHECK: call void @llvm.x86.tilerelease()
+ // CHECK: call void @llvm.x86.tilezero(i8 0)
+ _tile_ace_loadconfig(data);
+ _tile_ace_storeconfig(data);
+ _tile_ace_release();
+ _tile_ace_zero(0);
+}
+
+void test_acev1_bsr(void) {
+ // CHECK-LABEL: @test_acev1_bsr
+ // CHECK: call void @llvm.x86.bsrinit()
+ _bsr0_init();
+}
diff --git a/clang/test/CodeGen/X86/acev1_api.c b/clang/test/CodeGen/X86/acev1_api.c
new file mode 100644
index 0000000000000..acb12dae30cc4
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_api.c
@@ -0,0 +1,138 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
+// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 APIs using __acetile type (fixed 16x64 dimensions)
+// Note: ACE v1 does not have TILELOADD/TILESTORED - uses ACE-specific tile operations
+
+#include <immintrin.h>
+
+// Test __acetile zero
+void test_tile_ace_zero(void) {
+ // CHECK-LABEL: @test_tile_ace_zero
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __acetile c;
+ __tile_ace_zero(&c);
+}
+
+// Test TOP4BUUD ACE API
+void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4buud
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4buud(&dst, src1, src2);
+}
+
+// Test TOP4BUSD ACE API
+void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4busd
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4busd(&dst, src1, src2);
+}
+
+// Test TOP4BSSD ACE API
+void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4bssd
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4bssd(&dst, src1, src2);
+}
+
+// Test TOP4BSUD ACE API
+void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4bsud
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4bsud(&dst, src1, src2);
+}
+
+// Test TOP2BF16PS ACE API
+void test_tile_ace_top2bf16ps(__acetile dst, __m512bh src1, __m512bh src2) {
+ // CHECK-LABEL: @test_tile_ace_top2bf16ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top2bf16ps(&dst, src1, src2);
+}
+
+// Test TOP4MXHF8PS ACE API (HF8 x HF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxhf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxhf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxhf8ps(&dst, src1, src2, 5);
+}
+
+// Test TOP4MXBHF8PS ACE API (BF8 x HF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxbhf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxbhf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxbhf8ps(&dst, src1, src2, 3);
+}
+
+// Test TOP4MXHBF8PS ACE API (HF8 x BF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxhbf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxhbf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxhbf8ps(&dst, src1, src2, 7);
+}
+
+// Test TOP4MXBF8PS ACE API (BF8 x BF8 -> FP32 with BSR scaling)
+void test_tile_ace_top4mxbf8ps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxbf8ps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxbf8ps(&dst, src1, src2, 1);
+}
+
+// Test TOP4MXBSSPS ACE API (MX INT8 SxS -> FP32 with BSR scaling)
+void test_tile_ace_top4mxbssps(__acetile dst, __m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_tile_ace_top4mxbssps
+ // CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
+ // CHECK-DAG: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9
+ // CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
+ __tile_ace_top4mxbssps(&dst, src1, src2, 9);
+}
+
+// Test tile setrow (ACE-specific: ZMM to tile row)
+void test_tile_ace_setrow(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setrow
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __tile_ace_setrow(&dst, src, 0);
+}
+
+// Test tile setcol (ACE-specific: ZMM to tile column)
+void test_tile_ace_setcol(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setcol
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __tile_ace_setcol(&dst, src, 0);
+}
+
+// Test tile getrow (ACE-specific: tile row to ZMM)
+__m512i test_tile_ace_getrow(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_getrow
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ return __tile_ace_getrow(&src, 0);
+}
+
+// Integration test: ACE computation workflow
+void test_ace_workflow(__m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_ace_workflow
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.top4buud.internal
+ __acetile acc;
+ __tile_ace_zero(&acc);
+ __tile_ace_top4buud(&acc, src1, src2);
+}
diff --git a/clang/test/CodeGen/X86/acev1_bsr.c b/clang/test/CodeGen/X86/acev1_bsr.c
new file mode 100644
index 0000000000000..d68ecdcb59f7d
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_bsr.c
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 BSR (Block Sparse Register) operations emit correct LLVM IR
+
+#include <immintrin.h>
+#include <stdint.h>
+
+void test_bsr_init(void) {
+ // CHECK-LABEL: @test_bsr_init
+ // CHECK: call void @llvm.x86.bsrinit()
+ _bsr0_init();
+}
+
+void test_bsr_movf(__m512i src1, __m512i src2) {
+ // CHECK-LABEL: @test_bsr_movf
+ // CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+ _bsr0_movf(src1, src2);
+}
+
+void test_bsr_movh_set(__m512i src) {
+ // CHECK-LABEL: @test_bsr_movh_set
+ // CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
+ _bsr0_movh_set(src);
+}
+
+__m512i test_bsr_movh_get(void) {
+ // CHECK-LABEL: @test_bsr_movh_get
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+ return _bsr0_movh_get();
+}
+
+void test_bsr_movl_set(__m512i src) {
+ // CHECK-LABEL: @test_bsr_movl_set
+ // CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
+ _bsr0_movl_set(src);
+}
+
+__m512i test_bsr_movl_get(void) {
+ // CHECK-LABEL: @test_bsr_movl_get
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ return _bsr0_movl_get();
+}
+
+// Test BSR operations sequence
+void test_bsr_sequence(void) {
+ // CHECK-LABEL: @test_bsr_sequence
+ // CHECK: call void @llvm.x86.bsrinit()
+ // CHECK: call void @llvm.x86.bsrmovf
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+ // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ _bsr0_init();
+
+ __m512i zmm1 = _mm512_set1_epi32(1);
+ __m512i zmm2 = _mm512_set1_epi32(2);
+ _bsr0_movf(zmm1, zmm2);
+
+ __m512i high = _bsr0_movh_get();
+ __m512i low = _bsr0_movl_get();
+
+ (void)high;
+ (void)low;
+}
diff --git a/clang/test/CodeGen/X86/acev1_errors.c b/clang/test/CodeGen/X86/acev1_errors.c
new file mode 100644
index 0000000000000..c928882b62f4a
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_errors.c
@@ -0,0 +1,17 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -fsyntax-only -verify
+
+// Tests compile-time semantic errors for ACE v1 intrinsics
+
+#include <immintrin.h>
+
+void test_tile_range_errors(void) {
+ // Tile index must be in range [0, 7]
+ _tile_ace_zero(8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_ace_zero(16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+
+ // Outer product tile index errors
+ _tile_top4buud(8, 0, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_top4bssd(16, 0, 0); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ _tile_top2bf16ps(9, 0, 0); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
+}
diff --git a/clang/test/CodeGen/X86/acev1_inline_asm.c b/clang/test/CodeGen/X86/acev1_inline_asm.c
new file mode 100644
index 0000000000000..1dbf7d76a5355
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_inline_asm.c
@@ -0,0 +1,76 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
+
+// Tests inline assembly with ACE v1 tile registers
+
+#include <immintrin.h>
+
+void test_acev1_inline_asm_outer_product(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_outer_product
+ // CHECK: call void asm sideeffect "tilezero %tmm0
+ // CHECK: top4buud %zmm0, %zmm1, %tmm0
+ // CHECK: tilestored %tmm0
+ __asm__ volatile (
+ "tilezero %%tmm0 \n\t"
+ "top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
+ "tilestored %%tmm0, 0(%%rdi) \n\t"
+ ::: "memory", "tmm0", "zmm0", "zmm1"
+ );
+}
+
+void test_acev1_inline_asm_bf16(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_bf16
+ // CHECK: call void asm sideeffect "tilezero %tmm1
+ // CHECK: top2bf16ps %zmm2, %zmm3, %tmm1
+ __asm__ volatile (
+ "tilezero %%tmm1 \n\t"
+ "top2bf16ps %%zmm2, %%zmm3, %%tmm1 \n\t"
+ ::: "memory", "tmm1", "zmm2", "zmm3"
+ );
+}
+
+void test_acev1_inline_asm_bsr(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_bsr
+ // CHECK: call void asm sideeffect "bsrinit %bsr0
+ // CHECK: bsrmovf %zmm0, %zmm1
+ __asm__ volatile (
+ "bsrinit %%bsr0 \n\t"
+ "bsrmovf %%zmm0, %%zmm1 \n\t"
+ ::: "memory", "zmm0", "zmm1"
+ );
+}
+
+void test_acev1_inline_asm_tile_config(void *cfg) {
+ // CHECK-LABEL: @test_acev1_inline_asm_tile_config
+ // CHECK: call void asm sideeffect "ldtilecfg ($0)
+ // CHECK: tilerelease
+ __asm__ volatile (
+ "ldtilecfg (%0) \n\t"
+ "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm1 \n\t"
+ "tilerelease \n\t"
+ :: "r"(cfg) : "memory", "tmm0", "tmm1"
+ );
+}
+
+void test_acev1_inline_asm_all_outer_products(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_all_outer_products
+ // CHECK: top4buud
+ // CHECK: top4busd
+ // CHECK: top4bssd
+ // CHECK: top4bsud
+ // CHECK: top2bf16ps
+ __asm__ volatile (
+ "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm1 \n\t"
+ "tilezero %%tmm2 \n\t"
+ "tilezero %%tmm3 \n\t"
+ "tilezero %%tmm4 \n\t"
+ "top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
+ "top4busd %%zmm0, %%zmm1, %%tmm1 \n\t"
+ "top4bssd %%zmm0, %%zmm1, %%tmm2 \n\t"
+ "top4bsud %%zmm0, %%zmm1, %%tmm3 \n\t"
+ "top2bf16ps %%zmm0, %%zmm1, %%tmm4 \n\t"
+ ::: "memory", "tmm0", "tmm1", "tmm2", "tmm3", "tmm4", "zmm0", "zmm1"
+ );
+}
diff --git a/clang/test/CodeGen/X86/acev1_mxfp8.c b/clang/test/CodeGen/X86/acev1_mxfp8.c
new file mode 100644
index 0000000000000..e99ca9dda4067
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_mxfp8.c
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
+// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 MX FP8 outer product macro-based intrinsics
+
+#include <immintrin.h>
+
+void test_top4mxhf8ps(void) {
+ // CHECK-LABEL: @test_top4mxhf8ps
+ // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 0)
+ _tile_top4mxhf8ps(0, 1, 2, 0);
+}
+
+void test_top4mxbhf8ps(void) {
+ // CHECK-LABEL: @test_top4mxbhf8ps
+ // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, i8 2, i8 3, i8 1)
+ _tile_top4mxbhf8ps(1, 2, 3, 1);
+}
+
+void test_top4mxhbf8ps(void) {
+ // CHECK-LABEL: @test_top4mxhbf8ps
+ // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, i8 3, i8 4, i8 0)
+ _tile_top4mxhbf8ps(2, 3, 4, 0);
+}
+
+void test_top4mxbf8ps(void) {
+ // CHECK-LABEL: @test_top4mxbf8ps
+ // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, i8 4, i8 5, i8 1)
+ _tile_top4mxbf8ps(3, 4, 5, 1);
+}
+
+void test_top4mxbssps(void) {
+ // CHECK-LABEL: @test_top4mxbssps
+ // CHECK: call void @llvm.x86.top4mxbssps(i8 4, i8 5, i8 6, i8 0)
+ _tile_top4mxbssps(4, 5, 6, 0);
+}
+
+// Test all MX FP8 variants in sequence
+void test_all_mxfp8_variants(void) {
+ // CHECK-LABEL: @test_all_mxfp8_variants
+ // CHECK: call void @llvm.x86.top4mxhf8ps
+ // CHECK: call void @llvm.x86.top4mxbhf8ps
+ // CHECK: call void @llvm.x86.top4mxhbf8ps
+ // CHECK: call void @llvm.x86.top4mxbf8ps
+ // CHECK: call void @llvm.x86.top4mxbssps
+ _tile_top4mxhf8ps(0, 1, 2, 0);
+ _tile_top4mxbhf8ps(3, 4, 5, 0);
+ _tile_top4mxhbf8ps(6, 7, 0, 0);
+ _tile_top4mxbf8ps(1, 2, 3, 1);
+ _tile_top4mxbssps(4, 5, 6, 1);
+}
diff --git a/clang/test/CodeGen/X86/acev1_tile_movement.c b/clang/test/CodeGen/X86/acev1_tile_movement.c
new file mode 100644
index 0000000000000..f1383104f5db2
--- /dev/null
+++ b/clang/test/CodeGen/X86/acev1_tile_movement.c
@@ -0,0 +1,113 @@
+// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +acev1 -target-feature +avx512f \
+// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
+
+// Tests ACE v1 tile movement operations (ZMM <-> Tile row/col)
+// Note: ACE v1 does not have TILELOADD/TILESTORED - uses TILEMOVROW/TILEMOVCOL for data movement
+
+#include <immintrin.h>
+
+// Test tile setrow - write ZMM to tile row (ACE-specific)
+void test_tile_ace_setrow(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setrow
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __tile_ace_setrow(&dst, src, 0);
+}
+
+// Test tile setrow with different row indices
+void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setrow_indices
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __tile_ace_setrow(&dst, src, 0);
+ __tile_ace_setrow(&dst, src, 8);
+ __tile_ace_setrow(&dst, src, 15);
+}
+
+// Test tile setcol - write ZMM to tile column (ACE-specific)
+void test_tile_ace_setcol(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setcol
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __tile_ace_setcol(&dst, src, 0);
+}
+
+// Test tile setcol with different column indices
+void test_tile_ace_setcol_indices(__acetile dst, __m512i src) {
+ // CHECK-LABEL: @test_tile_ace_setcol_indices
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __tile_ace_setcol(&dst, src, 0);
+ __tile_ace_setcol(&dst, src, 8);
+}
+
+// Test tile getrow - read tile row to ZMM (ACE-specific)
+__m512i test_tile_ace_getrow(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_getrow
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ return __tile_ace_getrow(&src, 0);
+}
+
+// Test tile getrow with different row indices
+void test_tile_ace_getrow_indices(__acetile src, __m512i *out) {
+ // CHECK-LABEL: @test_tile_ace_getrow_indices
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ out[0] = __tile_ace_getrow(&src, 0);
+ out[1] = __tile_ace_getrow(&src, 8);
+ out[2] = __tile_ace_getrow(&src, 15);
+}
+
+// Integration test: initialize tile using ACE tile zero and setrow
+void test_tile_init_via_setrow(void) {
+ // CHECK-LABEL: @test_tile_init_via_setrow
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ __acetile tile;
+ __tile_ace_zero(&tile);
+
+ __m512i zmm = _mm512_set1_epi32(0x12345678);
+ __tile_ace_setrow(&tile, zmm, 0);
+}
+
+// Test combined tile operations
+void test_tile_movement_combined(void) {
+ // CHECK-LABEL: @test_tile_movement_combined
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ __acetile tile;
+ __tile_ace_zero(&tile);
+
+ __m512i zmm1 = _mm512_set1_epi32(1);
+ __m512i zmm2 = _mm512_set1_epi32(2);
+
+ // Set multiple rows
+ __tile_ace_setrow(&tile, zmm1, 0);
+ __tile_ace_setrow(&tile, zmm2, 1);
+
+ // Set columns
+ __tile_ace_setcol(&tile, zmm1, 0);
+}
+
+// Integration test: complete ACE tile workflow with data movement
+void test_ace_data_workflow(__m512i *input, __m512i *output) {
+ // CHECK-LABEL: @test_ace_data_workflow
+ // CHECK: call x86_amx @llvm.x86.tilezero.internal
+ // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
+ __acetile tile;
+ __tile_ace_zero(&tile);
+
+ // Initialize first row with input
+ __tile_ace_setrow(&tile, input[0], 0);
+
+ // Perform computation
+ __tile_ace_top4buud(&tile, input[1], input[2]);
+
+ // Extract result row
+ output[0] = __tile_ace_getrow(&tile, 0);
+}
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index f280f3b8b6557..fa917aaccbadd 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -145,7 +145,7 @@ __attribute__((target("fpmath=387")))
void f_fpmath_387(void) {}
// CHECK-NOT: tune-cpu
-// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
+// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-acev1,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse2")))
void f_no_sse2(void) {}
@@ -153,7 +153,7 @@ void f_no_sse2(void) {}
__attribute__((target("sse4")))
void f_sse4(void) {}
-// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
+// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-acev1,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse4")))
void f_no_sse4(void) {}
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index c7d006171fddb..3bc63f8031df6 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -547,6 +547,13 @@
// NO-AMX-AVX512-NOT: #define __AMX_AVX512__ 1
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 -x c \
+// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 %s
+// ACEV1: #define __ACEV1__ 1
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mno-acev1 -x c \
+// RUN: -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+// NO-ACEV1-NOT: #define __ACEV1__ 1
+
// RUN: %clang -target i386-unknown-unknown -march=atom -mavxvnni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVXVNNI %s
// AVXVNNI: #define __AVX2__ 1
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index ad79a8dce3660..1f3a8160e606e 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5678,6 +5678,204 @@ let TargetPrefix = "x86" in {
[]>;
}
+//===----------------------------------------------------------------------===//
+// ACE (AI Compute Extensions) Intrinsics
+//===----------------------------------------------------------------------===//
+let TargetPrefix = "x86" in {
+ // BSRINIT - Initialize Block Scale Register
+ def int_x86_bsrinit : ClangBuiltin<"__builtin_ia32_bsrinit">,
+ Intrinsic<[], [], []>;
+
+ // BSRMOVF - Move Full to BSR
+ // Takes two ZMM registers as input (doubleword integers), writes to BSR (implicit)
+ def int_x86_bsrmovf : ClangBuiltin<"__builtin_ia32_bsrmovf">,
+ Intrinsic<[], [llvm_v16i32_ty, llvm_v16i32_ty], []>;
+
+ // BSRMOVH - Move Half to/from BSR
+ // Load variant: read from ZMM (doubleword integers), update BSR
+ def int_x86_bsrmovh_set : ClangBuiltin<"__builtin_ia32_bsrmovh_set">,
+ Intrinsic<[], [llvm_v16i32_ty], []>;
+ // Store variant: read from BSR, return ZMM (doubleword integers)
+ def int_x86_bsrmovh_get : ClangBuiltin<"__builtin_ia32_bsrmovh_get">,
+ Intrinsic<[llvm_v16i32_ty], [], []>;
+
+ // BSRMOVL - Move Low to/from BSR
+ // Load variant: read from ZMM (doubleword integers), write to BSR
+ def int_x86_bsrmovl_set : ClangBuiltin<"__builtin_ia32_bsrmovl_set">,
+ Intrinsic<[], [llvm_v16i32_ty], []>;
+ // Store variant: read from BSR, return ZMM (doubleword integers)
+ def int_x86_bsrmovl_get : ClangBuiltin<"__builtin_ia32_bsrmovl_get">,
+ Intrinsic<[llvm_v16i32_ty], [], []>;
+
+ // TILEMOVCOL - Move Column from Vector to Tile
+ // Single intrinsic handles both immediate and register index forms.
+ // Lowering checks if index is constant and selects appropriate instruction.
+ def int_x86_tilesetcol : ClangBuiltin<"__builtin_ia32_tilesetcol">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>]>;
+
+ // ACE TILEMOVROW - Move Row from Vector TO Tile (extends AMX TILEMOVROW)
+ // Note: AMX has int_x86_tilemovrow which reads FROM tile to vector
+ // ACE adds the reverse direction: writing TO tile from vector
+ // Single intrinsic handles both immediate and register index forms.
+ // Lowering checks if index is constant and selects appropriate instruction.
+ def int_x86_tilesetrow : ClangBuiltin<"__builtin_ia32_tilesetrow">,
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>]>;
+
+ // TOP2BF16PS - Outer Product BF16 to Single Precision
+ def int_x86_top2bf16ps : ClangBuiltin<"__builtin_ia32_top2bf16ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4 - Tile Outer Product (4x4 block operations)
+ // TOP4BUUD - Unsigned/Unsigned
+ def int_x86_top4buud : ClangBuiltin<"__builtin_ia32_top4buud">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4BUSD - Unsigned/Signed
+ def int_x86_top4busd : ClangBuiltin<"__builtin_ia32_top4busd">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4BSSD - Signed/Signed
+ def int_x86_top4bssd : ClangBuiltin<"__builtin_ia32_top4bssd">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4BSUD - Signed/Unsigned
+ def int_x86_top4bsud : ClangBuiltin<"__builtin_ia32_top4bsud">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>]>;
+
+ // TOP4MX - Mixed Precision Outer Products
+ // TOP4MXHFBPS - FP16/BF16 to FP32
+ def int_x86_top4mxhf8ps : ClangBuiltin<"__builtin_ia32_top4mxhf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBHFBPS - BF16/FP16 to FP32
+ def int_x86_top4mxbhf8ps : ClangBuiltin<"__builtin_ia32_top4mxbhf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXHBFBPS - FP16/BF16/BF16 to FP32
+ def int_x86_top4mxhbf8ps : ClangBuiltin<"__builtin_ia32_top4mxhbf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBF8PS - MX FP8 BF8xBF8 to FP32
+ def int_x86_top4mxbf8ps : ClangBuiltin<"__builtin_ia32_top4mxbf8ps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBSSPS - MX INT8 SxS to FP32 (BSR scaled)
+ def int_x86_top4mxbssps : ClangBuiltin<"__builtin_ia32_top4mxbssps">,
+ Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
+ ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+
+ // ACE - internal intrinsics (for IR-level optimization)
+ // These take tile/ZMM values directly instead of immediate tile IDs
+
+ // TILEMOVCOL internal - takes dimensions and returns tile
+ def int_x86_tilesetcol_internal :
+ ClangBuiltin<"__builtin_ia32_tilesetcol_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
+ []>;
+
+ // ACE TILEMOVROW internal - takes dimensions, vector, index and returns tile
+ // Note: AMX has int_x86_tilemovrow_internal which extracts from tile to vector
+ // ACE adds the reverse: write vector to tile row
+ def int_x86_tilesetrow_internal :
+ ClangBuiltin<"__builtin_ia32_tilesetrow_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
+ []>;
+
+ // TOP2BF16PS internal - operates on BF16 data (32 BF16 elements per ZMM)
+ def int_x86_top2bf16ps_internal :
+ ClangBuiltin<"__builtin_ia32_top2bf16ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v32bf16_ty, llvm_v32bf16_ty], []>;
+
+ // TOP4BUUD internal - unsigned bytes (64 elements per ZMM)
+ def int_x86_top4buud_internal :
+ ClangBuiltin<"__builtin_ia32_top4buud_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4BUSD internal - unsigned/signed bytes (64 elements per ZMM)
+ def int_x86_top4busd_internal :
+ ClangBuiltin<"__builtin_ia32_top4busd_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4BSSD internal - signed bytes (64 elements per ZMM)
+ def int_x86_top4bssd_internal :
+ ClangBuiltin<"__builtin_ia32_top4bssd_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4BSUD internal - signed/unsigned bytes (64 elements per ZMM)
+ def int_x86_top4bsud_internal :
+ ClangBuiltin<"__builtin_ia32_top4bsud_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+ llvm_x86amx_ty, llvm_v64i8_ty, llvm_v64i8_ty], []>;
+
+ // TOP4MXHFBPS internal - 4th arg is BSR index (i8 immediate)
+ // FP8 data packed in dwords (16 dwords per ZMM)
+ def int_x86_top4mxhf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxhf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBHFBPS internal - FP8 data packed in dwords
+ def int_x86_top4mxbhf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxbhf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXHBFBPS internal - FP8 data packed in dwords
+ def int_x86_top4mxhbf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxhbf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBF8PS internal - MX FP8 BF8xBF8 data packed in dwords
+ def int_x86_top4mxbf8ps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxbf8ps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+
+ // TOP4MXBSSPS internal - MX INT8 SxS to FP32 (BSR scaled)
+ def int_x86_top4mxbssps_internal :
+ ClangBuiltin<"__builtin_ia32_top4mxbssps_internal">,
+ Intrinsic<[llvm_x86amx_ty],
+ [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_i8_ty,
+ llvm_x86amx_ty, llvm_v16i32_ty, llvm_v16i32_ty], [ImmArg<ArgIndex<3>>]>;
+}
+
//===----------------------------------------------------------------------===//
let TargetPrefix = "x86" in {
// CMPCCXADD
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index 46ab187289799..e88d7de0f1c2b 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -263,6 +263,7 @@ X86_FEATURE (NDD, "ndd")
X86_FEATURE (EGPR, "egpr")
X86_FEATURE (ZU, "zu")
X86_FEATURE (JMPABS, "jmpabs")
+X86_FEATURE (ACEV1, "acev1")
// These features aren't really CPU features, but the frontend can set them.
X86_FEATURE (RETPOLINE_EXTERNAL_THUNK, "retpoline-external-thunk")
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 06532096a142c..f8fbe99529494 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -3973,6 +3973,56 @@ bool X86AsmParser::parseInstruction(ParseInstructionInfo &Info, StringRef Name,
}
}
+ // ACE BSR instructions: Strip %bsr0 operands and distinguish load/store.
+ // BSR (Block Scale Register) is implicit in the encoding (always BSR0),
+ // but the spec requires explicit %bsr0 in assembly syntax for disambiguation.
+ //
+ // For BSRMOVH and BSRMOVL, load vs store is determined by operand order:
+ // - ATT load: zmm/mem, bsr0 (ZMM first, BSR last) - move TO bsr0
+ // - ATT store: bsr0, zmm/mem (BSR first, ZMM last) - move FROM bsr0
+ // - Intel syntax reverses source/dest from ATT
+ //
+ // Note: Check for specific ACE BSR instructions to avoid matching standard
+ // x86 "bsrl"/"bsrq" (bit scan reverse).
+ if (Name == "bsrinit" || Name == "bsrmovf" || Name == "bsrmovh" ||
+ Name == "bsrmovl") {
+ // Single pass: track positions and filter operands simultaneously
+ int FirstBsrPos = -1;
+ int FirstNonBsrPos = -1;
+ SmallVector<std::unique_ptr<MCParsedAsmOperand>, 4> NewOperands;
+ NewOperands.push_back(std::move(Operands[0])); // Keep the mnemonic token
+
+ for (size_t I = 1; I < Operands.size(); ++I) {
+ X86Operand &Op = static_cast<X86Operand &>(*Operands[I]);
+ if (Op.isReg() && Op.getReg() == X86::BSR0) {
+ if (FirstBsrPos < 0)
+ FirstBsrPos = I;
+ // Skip BSR operands (don't add to NewOperands)
+ } else {
+ if (FirstNonBsrPos < 0)
+ FirstNonBsrPos = I;
+ NewOperands.push_back(std::move(Operands[I]));
+ }
+ }
+
+ // BSR instructions require explicit %bsr0 operand
+ if (FirstBsrPos < 0)
+ return Error(NameLoc, "BSR instruction requires explicit %bsr0 operand");
+
+ Operands = std::move(NewOperands);
+
+ // For BSRMOVH and BSRMOVL, detect set vs get by operand order
+ if (Name == "bsrmovh" || Name == "bsrmovl") {
+ // Intel: BSR first = set (ZMM→BSR); ATT: non-BSR first = set
+ bool IsLoad = isParsingIntelSyntax() ? (FirstBsrPos < FirstNonBsrPos)
+ : (FirstNonBsrPos < FirstBsrPos);
+ StringRef NewMnemonic =
+ IsLoad ? (Name == "bsrmovh" ? "bsrmovh_set" : "bsrmovl_set")
+ : (Name == "bsrmovh" ? "bsrmovh_get" : "bsrmovl_get");
+ static_cast<X86Operand &>(*Operands[0]).setTokenValue(NewMnemonic);
+ }
+ }
+
if (Flags)
Operands.push_back(X86Operand::CreatePrefix(Flags, NameLoc, NameLoc));
return false;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
index 564636959f00f..e310276f4d75f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp
@@ -84,8 +84,9 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, uint64_t Address,
STI.hasFeature(X86::Is16Bit)) {
OS << "\tdata32";
}
- // Try to print any aliases first.
- else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
+ // Try to print any aliases first, then BSR instructions, then compare instrs.
+ else if (!printAliasInstr(MI, Address, OS) && !printBSRInstr(MI, OS) &&
+ !printVecCompareInstr(MI, OS))
printInstruction(MI, Address, OS);
// Next always print the annotation.
@@ -402,6 +403,93 @@ bool X86ATTInstPrinter::printVecCompareInstr(const MCInst *MI,
return false;
}
+bool X86ATTInstPrinter::printBSRInstr(const MCInst *MI, raw_ostream &OS) {
+ // ACE BSR instructions: Print with explicit %bsr0 operand as per spec.
+ // BSR is implicit in the encoding, but the spec requires explicit syntax.
+ if (!MI)
+ return false;
+ unsigned Opcode = MI->getOpcode();
+
+ switch (Opcode) {
+ default:
+ return false;
+
+ case X86::BSRINIT:
+ OS << "\tbsrinit\t%bsr0";
+ return true;
+
+ case X86::BSRMOVFrr:
+ // Syntax: bsrmovf %zmm2, %zmm1, %bsr0
+ OS << "\tbsrmovf\t";
+ printOperand(MI, 1, OS); // src2
+ OS << ", ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVFrm:
+ // Syntax: bsrmovf mem, %zmm1, %bsr0
+ OS << "\tbsrmovf\t";
+ printMemReference(MI, 1, OS); // src2 (memory)
+ OS << ", ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVHrr_set:
+ // Syntax: bsrmovh %zmm1, %bsr0 (set BSR high from zmm)
+ OS << "\tbsrmovh\t";
+ printOperand(MI, 0, OS); // src
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVHrm_set:
+ // Syntax: bsrmovh mem, %bsr0 (set BSR high from memory)
+ OS << "\tbsrmovh\t";
+ printMemReference(MI, 0, OS); // src (memory)
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVHrr_get:
+ // Syntax: bsrmovh %bsr0, %zmm1 (get BSR high to zmm)
+ OS << "\tbsrmovh\t%bsr0, ";
+ printOperand(MI, 0, OS); // dst
+ return true;
+
+ case X86::BSRMOVHmr_get:
+ // Syntax: bsrmovh %bsr0, mem
+ OS << "\tbsrmovh\t%bsr0, ";
+ printMemReference(MI, 0, OS); // dst (memory)
+ return true;
+
+ case X86::BSRMOVLrr_set:
+ // Syntax: bsrmovl %zmm2, %bsr0 (set BSR low from zmm)
+ OS << "\tbsrmovl\t";
+ printOperand(MI, 0, OS); // src
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVLrm_set:
+ // Syntax: bsrmovl mem, %bsr0
+ OS << "\tbsrmovl\t";
+ printMemReference(MI, 0, OS); // src (memory)
+ OS << ", %bsr0";
+ return true;
+
+ case X86::BSRMOVLrr_get:
+ // Syntax: bsrmovl %bsr0, %zmm3 (get BSR low to zmm)
+ OS << "\tbsrmovl\t%bsr0, ";
+ printOperand(MI, 0, OS); // dst
+ return true;
+
+ case X86::BSRMOVLmr_get:
+ // Syntax: bsrmovl %bsr0, mem
+ OS << "\tbsrmovl\t%bsr0, ";
+ printMemReference(MI, 0, OS); // dst (memory)
+ return true;
+ }
+}
+
void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
index 1452622ebcea8..f28024a50a78c 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h
@@ -28,6 +28,7 @@ class X86ATTInstPrinter final : public X86InstPrinterCommon {
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
+ bool printBSRInstr(const MCInst *MI, raw_ostream &OS);
// Autogenerated by tblgen, returns true if we successfully printed an
// alias.
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
index ff27005f52ea8..def7784ea7b5f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp
@@ -46,7 +46,8 @@ void X86IntelInstPrinter::printInst(const MCInst *MI, uint64_t Address,
if (MI->getOpcode() == X86::DATA16_PREFIX &&
STI.hasFeature(X86::Is16Bit)) {
OS << "\tdata32";
- } else if (!printAliasInstr(MI, Address, OS) && !printVecCompareInstr(MI, OS))
+ } else if (!printAliasInstr(MI, Address, OS) && !printBSRInstr(MI, OS) &&
+ !printVecCompareInstr(MI, OS))
printInstruction(MI, Address, OS);
// Next always print the annotation.
@@ -363,6 +364,91 @@ bool X86IntelInstPrinter::printVecCompareInstr(const MCInst *MI, raw_ostream &OS
return false;
}
+bool X86IntelInstPrinter::printBSRInstr(const MCInst *MI, raw_ostream &OS) {
+ // ACE BSR instructions: Print with explicit bsr0 operand as per spec.
+ // BSR is implicit in the encoding, but the spec requires explicit syntax.
+ if (!MI)
+ return false;
+ unsigned Opcode = MI->getOpcode();
+
+ switch (Opcode) {
+ default:
+ return false;
+
+ case X86::BSRINIT:
+ OS << "\tbsrinit\tbsr0";
+ return true;
+
+ case X86::BSRMOVFrr:
+ // Intel Syntax: bsrmovf bsr0, zmm1, zmm2
+ OS << "\tbsrmovf\tbsr0, ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", ";
+ printOperand(MI, 1, OS); // src2
+ return true;
+
+ case X86::BSRMOVFrm:
+ // Intel Syntax: bsrmovf bsr0, zmm1, zmmword ptr [mem]
+ OS << "\tbsrmovf\tbsr0, ";
+ printOperand(MI, 0, OS); // src1
+ OS << ", ";
+ printzmmwordmem(MI, 1, OS); // src2 (memory)
+ return true;
+
+ case X86::BSRMOVHrr_set:
+ // Intel Syntax: bsrmovh bsr0, zmm1 (set BSR high from zmm)
+ OS << "\tbsrmovh\tbsr0, ";
+ printOperand(MI, 0, OS); // src
+ return true;
+
+ case X86::BSRMOVHrm_set:
+ // Intel Syntax: bsrmovh bsr0, zmmword ptr [mem] (set BSR high from memory)
+ OS << "\tbsrmovh\tbsr0, ";
+ printzmmwordmem(MI, 0, OS); // src (memory)
+ return true;
+
+ case X86::BSRMOVHrr_get:
+ // Intel Syntax: bsrmovh zmm1, bsr0 (get BSR high to zmm)
+ OS << "\tbsrmovh\t";
+ printOperand(MI, 0, OS); // dst
+ OS << ", bsr0";
+ return true;
+
+ case X86::BSRMOVHmr_get:
+ // Intel Syntax: bsrmovh zmmword ptr [mem], bsr0
+ OS << "\tbsrmovh\t";
+ printzmmwordmem(MI, 0, OS); // dst (memory)
+ OS << ", bsr0";
+ return true;
+
+ case X86::BSRMOVLrr_set:
+ // Intel Syntax: bsrmovl bsr0, zmm2 (set BSR low from zmm)
+ OS << "\tbsrmovl\tbsr0, ";
+ printOperand(MI, 0, OS); // src
+ return true;
+
+ case X86::BSRMOVLrm_set:
+ // Intel Syntax: bsrmovl bsr0, zmmword ptr [mem]
+ OS << "\tbsrmovl\tbsr0, ";
+ printzmmwordmem(MI, 0, OS); // src (memory)
+ return true;
+
+ case X86::BSRMOVLrr_get:
+ // Intel Syntax: bsrmovl zmm3, bsr0 (get BSR low to zmm)
+ OS << "\tbsrmovl\t";
+ printOperand(MI, 0, OS); // dst
+ OS << ", bsr0";
+ return true;
+
+ case X86::BSRMOVLmr_get:
+ // Intel Syntax: bsrmovl zmmword ptr [mem], bsr0
+ OS << "\tbsrmovl\t";
+ printzmmwordmem(MI, 0, OS); // dst (memory)
+ OS << ", bsr0";
+ return true;
+ }
+}
+
void X86IntelInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
raw_ostream &O) {
const MCOperand &Op = MI->getOperand(OpNo);
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
index 324c56c1329da..85830cc0c02cb 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h
@@ -28,6 +28,7 @@ class X86IntelInstPrinter final : public X86InstPrinterCommon {
void printInst(const MCInst *MI, uint64_t Address, StringRef Annot,
const MCSubtargetInfo &STI, raw_ostream &OS) override;
bool printVecCompareInstr(const MCInst *MI, raw_ostream &OS);
+ bool printBSRInstr(const MCInst *MI, raw_ostream &OS);
// Autogenerated by tblgen, returns true if we successfully printed an
// alias.
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 5786c659b0f98..2b21219a29f02 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -296,6 +296,9 @@ def FeatureAMXAVX512 : SubtargetFeature<"amx-avx512",
"HasAMXAVX512", "true",
"Support AMX-AVX512 instructions",
[FeatureAMXTILE]>;
+def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
+ "Support ACEV1 (AI Compute Extensions v1) instructions",
+ [FeatureAMXAVX512]>;
def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true",
"Support CMPCCXADD instructions">;
def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true",
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 4a8e8bef897f3..4e00be9d9b028 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -699,6 +699,95 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
MI.tieOperands(0, 1);
return true;
}
+ // ACE internal pseudo instructions
+ // Pattern: Remove dimension operands (1-3), change opcode, re-tie operands
+ case X86::PTOP2BF16PSV:
+ case X86::PTOP4BUUDV:
+ case X86::PTOP4BUSDV:
+ case X86::PTOP4BSSDV:
+ case X86::PTOP4BSUDV: {
+ MI.untieRegOperand(4);
+ for (unsigned i = 3; i > 0; --i)
+ MI.removeOperand(i);
+ unsigned Opc = 0;
+ switch (Opcode) {
+ case X86::PTOP2BF16PSV:
+ Opc = X86::TOP2BF16PSrrr;
+ break;
+ case X86::PTOP4BUUDV:
+ Opc = X86::TOP4BUUDrrr;
+ break;
+ case X86::PTOP4BUSDV:
+ Opc = X86::TOP4BUSDrrr;
+ break;
+ case X86::PTOP4BSSDV:
+ Opc = X86::TOP4BSSDrrr;
+ break;
+ case X86::PTOP4BSUDV:
+ Opc = X86::TOP4BSUDrrr;
+ break;
+ default:
+ llvm_unreachable("Unexpected ACE opcode");
+ }
+ MI.setDesc(TII->get(Opc));
+ MI.tieOperands(0, 1);
+ return true;
+ }
+ // ACE TOP4MX internal pseudo instructions - have 4 dimension operands (row,
+ // col, k, imm)
+ case X86::PTOP4MXHF8PSV:
+ case X86::PTOP4MXBHF8PSV:
+ case X86::PTOP4MXHBF8PSV:
+ case X86::PTOP4MXBF8PSV:
+ case X86::PTOP4MXBSSPSV: {
+ MI.untieRegOperand(5);
+ // Remove dimension operands 1-3, keep imm at position 4
+ for (unsigned i = 3; i > 0; --i)
+ MI.removeOperand(i);
+ // Now operand layout is: dst, imm, src1(acc), src2, src3
+ // Real instruction expects: dst, src1(acc), src2, src3, imm
+ // Need to move imm from position 1 to end
+ unsigned ImmVal = MI.getOperand(1).getImm();
+ MI.removeOperand(1);
+ unsigned Opc = 0;
+ switch (Opcode) {
+ case X86::PTOP4MXHF8PSV:
+ Opc = X86::TOP4MXHF8PSrrri;
+ break;
+ case X86::PTOP4MXBHF8PSV:
+ Opc = X86::TOP4MXBHF8PSrrri;
+ break;
+ case X86::PTOP4MXHBF8PSV:
+ Opc = X86::TOP4MXHBF8PSrrri;
+ break;
+ case X86::PTOP4MXBF8PSV:
+ Opc = X86::TOP4MXBF8PSrrri;
+ break;
+ case X86::PTOP4MXBSSPSV:
+ Opc = X86::TOP4MXBSSPSrrri;
+ break;
+ default:
+ llvm_unreachable("Unexpected ACE TOP4MX opcode");
+ }
+ MI.setDesc(TII->get(Opc));
+ MI.addOperand(MachineOperand::CreateImm(ImmVal));
+ MI.tieOperands(0, 1);
+ return true;
+ }
+ // ACE TILEMOV internal pseudo instructions - output TILE, remove dimension
+ // operands
+ case X86::PTILEMOVCOLV:
+ case X86::PTILEMOVROWV: {
+ // Operand layout: dst, row, col, src(VR512), idx(GR32)
+ // Remove row, col (operands 1, 2)
+ for (unsigned i = 2; i > 0; --i)
+ MI.removeOperand(i);
+ // Now layout: dst, src, idx - matches real instruction
+ unsigned Opc =
+ (Opcode == X86::PTILEMOVCOLV) ? X86::TILEMOVCOLrr : X86::TILEMOVROWrr;
+ MI.setDesc(TII->get(Opc));
+ return true;
+ }
case X86::PTILESTOREDV: {
for (int i = 1; i >= 0; --i)
MI.removeOperand(i);
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index b18c16f757a40..72865e71689be 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -210,8 +210,10 @@ void X86FastPreTileConfigImpl::InitializeTileConfigStackSpace() {
.addReg(Xmm);
}
// Fill in the palette first.
+ // Use Palette 2 for ACE v1, Palette 1 for AMX.
+ int PaletteId = ST->hasACEV1() ? 2 : 1;
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), CfgSS)
- .addImm(1);
+ .addImm(PaletteId);
}
/// Insert spill instruction for \p AssignedReg before \p Before.
@@ -223,9 +225,35 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');
const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
- // Don't need shape information for tile store, becasue it is adjacent to
- // the tile def instruction.
- TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, Register());
+
+ // ACE v1 doesn't have TILESTORED - use TILEMOVROW row-by-row
+ if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ const DebugLoc &DL = Before->getDebugLoc();
+ Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+
+ const unsigned NumRows = 16;
+ const unsigned RowSize = 64;
+
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // TILEMOVROW zmm, tmm, imm8 (read from tile)
+ // Only kill src on the last row read
+ bool KillSrcNow = (Row == NumRows - 1) && Kill;
+ BuildMI(*MBB, Before, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
+ .addReg(VirtReg, getKillRegState(KillSrcNow))
+ .addImm(Row);
+
+ // VMOVUPS [stack + row*64], zmm
+ MachineInstrBuilder MIB =
+ BuildMI(*MBB, Before, DL, TII->get(X86::VMOVUPSZmr));
+ addFrameReference(MIB, FI, Row * RowSize);
+ MIB.addReg(ScratchZMM, RegState::Kill);
+ }
+ } else {
+ // AMX: Use storeRegToStackSlot (TILESTORED)
+ // Don't need shape information for tile store, because it is adjacent to
+ // the tile def instruction.
+ TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, Register());
+ }
++NumStores;
// TODO: update DBG_VALUEs
@@ -238,6 +266,7 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
int FI = getStackSpaceFor(OrigReg);
const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg);
Register TileReg;
+
// Fold copy to tileload
// BB1:
// spill src to s
@@ -250,25 +279,50 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
TileReg = UseMI->getOperand(0).getReg();
else
TileReg = MRI->createVirtualRegister(&RC);
- // Can't use TII->loadRegFromStackSlot(), because we need the shape
- // information for reload.
- // tileloadd (%sp, %idx), %tmm
- unsigned Opc = X86::PTILELOADDV;
- Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
- // FIXME: MBB is not the parent of UseMI.
- MachineInstr *NewMI = BuildMI(*UseMI->getParent(), UseMI, DebugLoc(),
- TII->get(X86::MOV64ri), StrideReg)
- .addImm(64);
- NewMI = addFrameReference(
- BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg)
- .addReg(RowMO->getReg())
- .addReg(ColMO->getReg()),
- FI);
- MachineOperand &MO = NewMI->getOperand(5);
- MO.setReg(StrideReg);
- MO.setIsKill(true);
- RowMO->setIsKill(false);
- ColMO->setIsKill(false);
+
+ // ACE v1 doesn't have TILELOADD - use TILEMOVROW row-by-row
+ if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ const DebugLoc &DL = UseMI->getDebugLoc();
+ Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
+
+ const unsigned NumRows = 16;
+ const unsigned RowSize = 64;
+
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // VMOVUPS zmm, [stack + row*64]
+ MachineInstrBuilder MIB = BuildMI(*UseMI->getParent(), UseMI, DL,
+ TII->get(X86::VMOVUPSZrm), ScratchZMM);
+ addFrameReference(MIB, FI, Row * RowSize);
+
+ // TILEMOVROW tmm, zmm, imm8 (write to tile)
+ BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWri),
+ TileReg)
+ .addReg(ScratchZMM, RegState::Kill)
+ .addImm(Row);
+ }
+ } else {
+ // AMX: Use PTILELOADDV
+ // Can't use TII->loadRegFromStackSlot(), because we need the shape
+ // information for reload.
+ // tileloadd (%sp, %idx), %tmm
+ unsigned Opc = X86::PTILELOADDV;
+ Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
+ // FIXME: MBB is not the parent of UseMI.
+ MachineInstr *NewMI = BuildMI(*UseMI->getParent(), UseMI, DebugLoc(),
+ TII->get(X86::MOV64ri), StrideReg)
+ .addImm(64);
+ NewMI = addFrameReference(
+ BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg)
+ .addReg(RowMO->getReg())
+ .addReg(ColMO->getReg()),
+ FI);
+ MachineOperand &MO = NewMI->getOperand(5);
+ MO.setReg(StrideReg);
+ MO.setIsKill(true);
+ RowMO->setIsKill(false);
+ ColMO->setIsKill(false);
+ }
+
// Erase copy instruction after it is folded.
if (UseMI->isCopy()) {
UseMI->eraseFromParent();
@@ -684,8 +738,9 @@ bool X86FastPreTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
bool X86FastPreTileConfigImpl::runOnMachineFunction(MachineFunction &MFunc) {
X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
MF = &MFunc;
diff --git a/llvm/lib/Target/X86/X86FastTileConfig.cpp b/llvm/lib/Target/X86/X86FastTileConfig.cpp
index f9bf5575dd44a..f6ee4313174bb 100644
--- a/llvm/lib/Target/X86/X86FastTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastTileConfig.cpp
@@ -42,6 +42,7 @@ class X86FastTileConfigImpl {
private:
// context
MachineFunction *MF = nullptr;
+ const X86Subtarget *ST = nullptr;
const TargetInstrInfo *TII = nullptr;
MachineRegisterInfo *MRI = nullptr;
const TargetRegisterInfo *TRI = nullptr;
@@ -131,6 +132,13 @@ bool X86FastTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
} else { // PLDTILECFGV
// Rewrite the shape information to memory. Stack slot should have
// been initialized to zero in pre config.
+ // ACE Palette 2 uses fixed tile dimensions (16x64), so skip writing
+ // shapes - bytes 1-63 must remain zero for ACE.
+ if (ST->hasACEV1()) {
+ ShapeInfos.clear();
+ Change = true;
+ continue;
+ }
int SS = MI.getOperand(0).getIndex(); // tile config stack slot.
for (auto &ShapeInfo : ShapeInfos) {
DebugLoc DL;
@@ -176,15 +184,16 @@ bool X86FastTileConfigImpl::configBasicBlock(MachineBasicBlock &MBB) {
bool X86FastTileConfigImpl::runOnMachineFunction(MachineFunction &MFunc) {
X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
MF = &MFunc;
MRI = &MFunc.getRegInfo();
- const TargetSubtargetInfo *ST = &MFunc.getSubtarget<X86Subtarget>();
+ ST = &MFunc.getSubtarget<X86Subtarget>();
TRI = ST->getRegisterInfo();
- TII = MFunc.getSubtarget().getInstrInfo();
+ TII = ST->getInstrInfo();
bool Change = false;
// Loop over all of the basic blocks, eliminating virtual register references
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 2d02ac618b275..78c7257dee456 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2861,8 +2861,9 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
}
}
- // Emit tilerelease for AMX kernel.
- if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA)
+ // Emit tilerelease for AMX/ACE kernel.
+ if (X86FI->getAMXProgModel() == AMXProgModelEnum::ManagedRA ||
+ X86FI->getACEProgModel() == ACEProgModelEnum::ACE_ManagedRA)
BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
if (NeedsWin64CFI && MF.hasWinCFI())
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 779e4dfce513a..bf252d934fbdc 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -5683,6 +5683,25 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
ReplaceNode(Node, CNode);
return;
}
+ // AMX-AVX512 internal intrinsics - set program model for struct-based API
+ case Intrinsic::x86_tcvtrowd2ps_internal:
+ case Intrinsic::x86_tcvtrowps2bf16h_internal:
+ case Intrinsic::x86_tcvtrowps2bf16l_internal:
+ case Intrinsic::x86_tcvtrowps2phh_internal:
+ case Intrinsic::x86_tcvtrowps2phl_internal:
+ case Intrinsic::x86_tilemovrow_internal: {
+ if (!Subtarget->hasAMXAVX512())
+ break;
+ auto *MFI =
+ CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
+ // Set program model for struct-based API (ManagedRA)
+ if (Subtarget->hasACEV1())
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
+ else
+ MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ // Let pattern matching handle the rest
+ break;
+ }
}
break;
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a8a28ff94db45..5b90a771a6569 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2701,7 +2701,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
}
}
- if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
+ if (!Subtarget.useSoftFloat() &&
+ (Subtarget.hasAMXTILE() || Subtarget.hasACEV1())) {
addRegisterClass(MVT::x86amx, &X86::TILERegClass);
}
@@ -28656,6 +28657,42 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
Chain = LockArith.getValue(1);
return DAG.getMergeValues({getSETCC(CC, LockArith, DL, DAG), Chain}, DL);
}
+ // ACE Tile Movement Intrinsics - handle both immediate and register index
+ // forms using a single intrinsic. Check if index is constant to select
+ // the appropriate instruction form.
+ case Intrinsic::x86_tilesetrow:
+ case Intrinsic::x86_tilesetcol: {
+ SDLoc DL(Op);
+ SDValue Chain = Op.getOperand(0);
+ unsigned TileID = Op.getConstantOperandVal(2);
+ SDValue Src = Op.getOperand(3);
+ SDValue Idx = Op.getOperand(4);
+
+ unsigned PseudoImm, PseudoReg;
+ if (IntNo == Intrinsic::x86_tilesetrow) {
+ PseudoImm = X86::PTILEMOVROW;
+ PseudoReg = X86::PTILEMOVROW_REG;
+ } else {
+ PseudoImm = X86::PTILEMOVCOL;
+ PseudoReg = X86::PTILEMOVCOL_REG;
+ }
+
+ MachineSDNode *Node;
+ // Check if index is a compile-time constant
+ if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
+ // Use immediate form
+ Node = DAG.getMachineNode(
+ PseudoImm, DL, MVT::Other,
+ {DAG.getTargetConstant(TileID, DL, MVT::i8), Src,
+ DAG.getTargetConstant(CIdx->getZExtValue(), DL, MVT::i8), Chain});
+ } else {
+ // Use register form
+ Node = DAG.getMachineNode(
+ PseudoReg, DL, MVT::Other,
+ {DAG.getTargetConstant(TileID, DL, MVT::i8), Src, Idx, Chain});
+ }
+ return SDValue(Node, 0);
+ }
}
return SDValue();
}
@@ -38742,6 +38779,106 @@ X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
return BB;
}
+// Helper enum and table for centralized tile program model handling
+namespace {
+enum class TileProgModelType {
+ None, // Not a tile instruction
+ ACE_DirectReg, // ACE-only, always ACE_DirectReg
+ ACE_ManagedRA, // ACE-only, always ACE_ManagedRA
+ AMX_DirectReg, // AMX/ACE, DirectReg (checks subtarget)
+ AMX_ManagedRA, // AMX/ACE, ManagedRA (checks subtarget)
+};
+
+struct TileOpcodeModelEntry {
+ unsigned Opcode;
+ TileProgModelType ModelType;
+};
+
+// Centralized table mapping tile opcodes to their program model type
+static const TileOpcodeModelEntry TileOpcodeModels[] = {
+ // AMX/ACE DirectReg (macro API) - checks subtarget for ACE vs AMX
+ {X86::PTILEZERO, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Hrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Lrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHHrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHLrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWD2PSrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTILEMOVROWrti, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Hrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2BF16Lrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHHrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWPS2PHLrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTCVTROWD2PSrte, TileProgModelType::AMX_DirectReg},
+ {X86::PTILEMOVROWrte, TileProgModelType::AMX_DirectReg},
+
+ // AMX/ACE ManagedRA (struct API) - checks subtarget for ACE vs AMX
+ {X86::PTILEZEROV, TileProgModelType::AMX_ManagedRA},
+
+ // ACE-only DirectReg (macro API)
+ {X86::PTILEMOVCOL, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVCOL_REG, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVROW, TileProgModelType::ACE_DirectReg},
+ {X86::PTILEMOVROW_REG, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP2BF16PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BUUD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BUSD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BSSD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4BSUD, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXHF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBHF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXHBF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBF8PS, TileProgModelType::ACE_DirectReg},
+ {X86::PTOP4MXBSSPS, TileProgModelType::ACE_DirectReg},
+
+ // ACE-only ManagedRA (struct API)
+ {X86::PTILEMOVCOLV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTILEMOVROWV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP2BF16PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BUUDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BUSDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BSSDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4BSUDV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXHF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBHF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXHBF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBF8PSV, TileProgModelType::ACE_ManagedRA},
+ {X86::PTOP4MXBSSPSV, TileProgModelType::ACE_ManagedRA},
+};
+
+// Set tile program model based on opcode. Returns true if model was set.
+static bool setTileProgModelForOpcode(unsigned Opcode,
+ X86MachineFunctionInfo *MFI,
+ const X86Subtarget &Subtarget) {
+ for (const auto &Entry : TileOpcodeModels) {
+ if (Entry.Opcode == Opcode) {
+ switch (Entry.ModelType) {
+ case TileProgModelType::None:
+ return false;
+ case TileProgModelType::ACE_DirectReg:
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_DirectReg);
+ return true;
+ case TileProgModelType::ACE_ManagedRA:
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
+ return true;
+ case TileProgModelType::AMX_DirectReg:
+ if (Subtarget.hasACEV1())
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_DirectReg);
+ else
+ MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
+ return true;
+ case TileProgModelType::AMX_ManagedRA:
+ if (Subtarget.hasACEV1())
+ MFI->setACEProgModel(ACEProgModelEnum::ACE_ManagedRA);
+ else
+ MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ return true;
+ }
+ }
+ }
+ return false;
+}
+} // anonymous namespace
+
MachineBasicBlock *
X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *BB) const {
@@ -38749,6 +38886,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
const TargetInstrInfo *TII = Subtarget.getInstrInfo();
const MIMetadata MIMD(MI);
+ // Centralized tile program model handling
+ auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
+ setTileProgModelForOpcode(MI.getOpcode(), MFI, Subtarget);
+
auto TMMImmToTMMReg = [](unsigned Imm) {
assert (Imm < 8 && "Illegal tmm index");
return X86::TMM0 + Imm;
@@ -39164,16 +39305,23 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::PTILEZERO: {
unsigned Imm = MI.getOperand(0).getImm();
BuildMI(*BB, MI, MIMD, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm));
- MI.eraseFromParent(); // The pseudo is gone now.
- auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
- MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
+ MI.eraseFromParent();
return BB;
}
- case X86::PTILEZEROV: {
- auto *MFI = MF->getInfo<X86MachineFunctionInfo>();
- MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
+ case X86::PTILEZEROV:
+ case X86::PTILEMOVCOLV:
+ case X86::PTILEMOVROWV:
+ case X86::PTOP2BF16PSV:
+ case X86::PTOP4BUUDV:
+ case X86::PTOP4BUSDV:
+ case X86::PTOP4BSSDV:
+ case X86::PTOP4BSUDV:
+ case X86::PTOP4MXHF8PSV:
+ case X86::PTOP4MXBHF8PSV:
+ case X86::PTOP4MXHBF8PSV:
+ case X86::PTOP4MXBF8PSV:
+ case X86::PTOP4MXBSSPSV:
return BB;
- }
case X86::PTILELOADDRS:
case X86::PTILELOADDRST1:
case X86::PTILELOADD:
@@ -39296,6 +39444,132 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MI.eraseFromParent(); // The pseudo is gone now.
return BB;
}
+ // ACE pseudo instruction expansion
+ case X86::PBSRINIT: {
+ BuildMI(*BB, MI, MIMD, TII->get(X86::BSRINIT));
+ MI.eraseFromParent();
+ return BB;
+ }
+ // Note: BSRMOVF, BSRMOVH, BSRMOVL are handled directly by patterns on real
+ // instructions
+ case X86::PTILEMOVCOL: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ unsigned Idx = MI.getOperand(2).getImm();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLri), DstReg)
+ .addReg(SrcReg)
+ .addImm(Idx);
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTILEMOVCOL_REG: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVCOLrr), DstReg)
+ .addReg(SrcReg)
+ .addReg(MI.getOperand(2).getReg());
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTILEMOVROW: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ unsigned Idx = MI.getOperand(2).getImm();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWri), DstReg)
+ .addReg(SrcReg)
+ .addImm(Idx);
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTILEMOVROW_REG: {
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ Register SrcReg = MI.getOperand(1).getReg();
+ BuildMI(*BB, MI, MIMD, TII->get(X86::TILEMOVROWrr), DstReg)
+ .addReg(SrcReg)
+ .addReg(MI.getOperand(2).getReg());
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTOP2BF16PS:
+ case X86::PTOP4BUUD:
+ case X86::PTOP4BUSD:
+ case X86::PTOP4BSSD:
+ case X86::PTOP4BSUD: {
+ unsigned Opc;
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case X86::PTOP2BF16PS:
+ Opc = X86::TOP2BF16PSrrr;
+ break;
+ case X86::PTOP4BUUD:
+ Opc = X86::TOP4BUUDrrr;
+ break;
+ case X86::PTOP4BUSD:
+ Opc = X86::TOP4BUSDrrr;
+ break;
+ case X86::PTOP4BSSD:
+ Opc = X86::TOP4BSSDrrr;
+ break;
+ case X86::PTOP4BSUD:
+ Opc = X86::TOP4BSUDrrr;
+ break;
+ }
+ // These instructions: TILE dst (rw), ZMM src1, ZMM src2
+ // The pseudo has: u8imm dst, u8imm src1, u8imm src2
+ // src1 and src2 are ZMM register indices (0-31)
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ unsigned Src1Imm = MI.getOperand(1).getImm();
+ unsigned Src2Imm = MI.getOperand(2).getImm();
+ assert(Src1Imm < 32 && "Illegal ZMM register index");
+ assert(Src2Imm < 32 && "Illegal ZMM register index");
+ BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
+ .addReg(DstReg, RegState::Undef)
+ .addReg(X86::ZMM0 + Src1Imm)
+ .addReg(X86::ZMM0 + Src2Imm);
+ MI.eraseFromParent();
+ return BB;
+ }
+ case X86::PTOP4MXHF8PS:
+ case X86::PTOP4MXBHF8PS:
+ case X86::PTOP4MXHBF8PS:
+ case X86::PTOP4MXBF8PS:
+ case X86::PTOP4MXBSSPS: {
+ unsigned Opc;
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case X86::PTOP4MXHF8PS:
+ Opc = X86::TOP4MXHF8PSrrri;
+ break;
+ case X86::PTOP4MXBHF8PS:
+ Opc = X86::TOP4MXBHF8PSrrri;
+ break;
+ case X86::PTOP4MXHBF8PS:
+ Opc = X86::TOP4MXHBF8PSrrri;
+ break;
+ case X86::PTOP4MXBF8PS:
+ Opc = X86::TOP4MXBF8PSrrri;
+ break;
+ case X86::PTOP4MXBSSPS:
+ Opc = X86::TOP4MXBSSPSrrri;
+ break;
+ }
+ // These instructions: TILE dst (rw), ZMM src1, ZMM src2, imm8
+ unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
+ unsigned Src1Imm = MI.getOperand(1).getImm();
+ unsigned Src2Imm = MI.getOperand(2).getImm();
+ unsigned Imm = MI.getOperand(3).getImm();
+ assert(Src1Imm < 32 && "Illegal ZMM register index");
+ assert(Src2Imm < 32 && "Illegal ZMM register index");
+ BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
+ .addReg(DstReg, RegState::Undef)
+ .addReg(X86::ZMM0 + Src1Imm)
+ .addReg(X86::ZMM0 + Src2Imm)
+ .addImm(Imm);
+ MI.eraseFromParent();
+ return BB;
+ }
}
}
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
new file mode 100644
index 0000000000000..f786f8ca34583
--- /dev/null
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -0,0 +1,417 @@
+//===---- X86InstrACE.td - ACE Instruction Set Extension --*- tablegen -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the instructions that make up the ACE (AI Compute
+// Extensions) instruction set.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// ACE instructions
+
+let Predicates = [HasACEV1, In64BitMode] in {
+
+// BSRINIT - Initialize Block Scale Register
+// VEX.128.F2.0F38.W1 49 11:000:000 - BSRINIT bsr0
+// BSR is implicit destination, ModRM = 11:000:000 (mod=11, reg=000, r/m=000)
+// Note: BSR0 is implicit (via Defs), not in operand list. Asm syntax has no explicit operand.
+let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
+ def BSRINIT : I<0x49, MRM_C0, (outs), (ins),
+ "bsrinit", []>,
+ VEX, XD, T8, REX_W;
+}
+
+// BSRMOVF - Move Full to BSR
+// EVEX.512.NP.MAP6.W1 95 11:000:bbb - BSRMOVF bsr0, zmm1, zmm2/m512
+// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1, ModRM.r/m encodes zmm2
+// BSR is implicit destination (via Defs)
+let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
+ def BSRMOVFrr : I<0x95, MRM0r, (outs),
+ (ins VR512:$src1, VR512:$src2),
+ "bsrmovf\t{$src2, $src1|$src1, $src2}",
+ [(int_x86_bsrmovf (v16i32 VR512:$src1), (v16i32 VR512:$src2))]>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, VVVV;
+
+ let mayLoad = 1 in
+ def BSRMOVFrm : I<0x95, MRM0m, (outs),
+ (ins VR512:$src1, f512mem:$src2),
+ "bsrmovf\t{$src2, $src1|$src1, $src2}", []>,
+ EVEX, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>, VVVV;
+}
+
+// BSRMOVH - Move Half to/from BSR
+// EVEX.512.F2.MAP6.W1 95 /r - BSRMOVH bsr0, zmm1/m512, bsr0 (load, W1)
+// EVEX.512.F2.MAP6.W0 95 /r - BSRMOVH zmm1/m512, bsr0 (store, W0)
+// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
+// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
+// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+let SchedRW = [WriteVecLogic] in {
+ // Set BSR high half (W1): BSR is both input and output (read-modify-write)
+ let Defs = [BSR0], Uses = [BSR0] in {
+ def BSRMOVHrr_set : I<0x95, MRM0r, (outs),
+ (ins VR512:$src),
+ "bsrmovh_set\t$src",
+ [(int_x86_bsrmovh_set (v16i32 VR512:$src))]>,
+ EVEX, XD, T_MAP6, REX_W, EVEX_V512;
+
+ let mayLoad = 1 in
+ def BSRMOVHrm_set : I<0x95, MRM0m, (outs),
+ (ins f512mem:$src),
+ "bsrmovh_set\t$src", []>,
+ EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+
+ // Get BSR high half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ let Uses = [BSR0] in {
+ def BSRMOVHrr_get : I<0x95, MRM0r, (outs VR512:$dst),
+ (ins),
+ "bsrmovh_get\t$dst",
+ [(set (v16i32 VR512:$dst), (int_x86_bsrmovh_get))]>,
+ EVEX, XD, T_MAP6, EVEX_V512;
+
+ let mayStore = 1 in
+ def BSRMOVHmr_get : I<0x95, MRM0m, (outs),
+ (ins f512mem:$dst),
+ "bsrmovh_get\t$dst", []>,
+ EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+}
+
+// BSRMOVL - Move Low to/from BSR
+// EVEX.512.F3.MAP6.W1 95 /r - BSRMOVL bsr0, zmm1/m512 (load, W1)
+// EVEX.512.F3.MAP6.W0 95 /r - BSRMOVL zmm1/m512, bsr0 (store, W0)
+// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
+// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
+// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+let SchedRW = [WriteVecLogic] in {
+ // Set BSR low half (W1): BSR is both input and output (read-modify-write)
+ let Defs = [BSR0], Uses = [BSR0] in {
+ def BSRMOVLrr_set : I<0x95, MRM0r, (outs),
+ (ins VR512:$src),
+ "bsrmovl_set\t$src",
+ [(int_x86_bsrmovl_set (v16i32 VR512:$src))]>,
+ EVEX, XS, T_MAP6, REX_W, EVEX_V512;
+
+ let mayLoad = 1 in
+ def BSRMOVLrm_set : I<0x95, MRM0m, (outs),
+ (ins f512mem:$src),
+ "bsrmovl_set\t$src", []>,
+ EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+
+ // Get BSR low half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ let Uses = [BSR0] in {
+ def BSRMOVLrr_get : I<0x95, MRM0r, (outs VR512:$dst),
+ (ins),
+ "bsrmovl_get\t$dst",
+ [(set (v16i32 VR512:$dst), (int_x86_bsrmovl_get))]>,
+ EVEX, XS, T_MAP6, EVEX_V512;
+
+ let mayStore = 1 in
+ def BSRMOVLmr_get : I<0x95, MRM0m, (outs),
+ (ins f512mem:$dst),
+ "bsrmovl_get\t$dst", []>,
+ EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
+ }
+}
+
+// TILEMOVCOL - Move Column from Vector to Tile
+// EVEX.512.66.0F3A.W1 2F /r ib - TILEMOVCOL tmm1, zmm2, imm8
+// EVEX.512.66.0F38.W1 4B /r - TILEMOVCOL tmm1, zmm2, r32
+// For r32 variant: GR32 is encoded in EVEX.vvvv (use MRMSrcReg4VOp3 + VVVV)
+let SchedRW = [WriteVecLogic] in {
+ def TILEMOVCOLri : Ii8<0x2F, MRMSrcReg, (outs TILE:$dst),
+ (ins VR512:$src, u8imm:$idx),
+ "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def TILEMOVCOLrr : I<0x4B, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins VR512:$src, GR32:$idx),
+ "tilemovcol\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+}
+
+// TILEMOVROW - Move Row from Vector to Tile
+// EVEX.512.66.0F3A.W1 07 /r ib - TILEMOVROW tmm1, zmm2, imm8
+// EVEX.512.66.0F38.W1 4A /r - TILEMOVROW tmm1, zmm2, r32
+// For r32 variant: GR32 is encoded in EVEX.vvvv (use MRMSrcReg4VOp3 + VVVV)
+let SchedRW = [WriteVecLogic] in {
+ def TILEMOVROWri : Ii8<0x07, MRMSrcReg, (outs TILE:$dst),
+ (ins VR512:$src, u8imm:$idx),
+ "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, TA, REX_W, EVEX_V512;
+
+ def TILEMOVROWrr : I<0x4A, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins VR512:$src, GR32:$idx),
+ "tilemovrow\t{$idx, $src, $dst|$dst, $src, $idx}", []>,
+ EVEX, PD, T8, REX_W, VVVV, EVEX_V512;
+}
+
+// Outer Product Instructions
+// TOP2BF16PS - Outer Product BF16 to Single Precision
+// EVEX.512.F3.0F38.W0 5C /r - TOP2BF16PS tmm1, zmm2, zmm3
+// Operand encoding from ACE spec page 52:
+// Operand 1: ModRM:reg (rw) = tmm1
+// Operand 2: ModRM:r/m (r) = zmm2
+// Operand 3: EVEX.vvvv (r) = zmm3
+// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+ def TOP2BF16PSrrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top2bf16ps\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, XS, T8, EVEX_V512;
+}
+
+// TOP4 Instructions - All use opcode 0x5E, differentiated by prefix
+// EVEX.512.NP.0F38.W0 5E /r - TOP4BUUD tmm1, zmm2, zmm3 (unsigned/unsigned)
+// EVEX.512.66.0F38.W0 5E /r - TOP4BUSD tmm1, zmm2, zmm3 (unsigned/signed)
+// EVEX.512.F2.0F38.W0 5E /r - TOP4BSSD tmm1, zmm2, zmm3 (signed/signed)
+// EVEX.512.F3.0F38.W0 5E /r - TOP4BSUD tmm1, zmm2, zmm3 (signed/unsigned)
+// Operand encoding from ACE spec page 54:
+// Operand 1: ModRM:reg (rw) = tmm1
+// Operand 2: ModRM:r/m (r) = zmm2
+// Operand 3: EVEX.vvvv (r) = zmm3
+// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+ def TOP4BUUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4buud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, T8, EVEX_V512;
+
+ def TOP4BUSDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4busd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, PD, T8, EVEX_V512;
+
+ def TOP4BSSDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4bssd\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, XD, T8, EVEX_V512;
+
+ def TOP4BSUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src2, VR512:$src3),
+ "top4bsud\t{$src3, $src2, $dst|$dst, $src2, $src3}", []>,
+ EVEX, VVVV, XS, T8, EVEX_V512;
+}
+
+// TOP4MX Instructions - Mixed Precision Outer Products with Immediate
+// All use opcode 0x8D with different prefixes, or 0x8F
+// EVEX.512.NP.0F3A.W0 8D /r ib - TOP4MXBF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.66.0F3A.W0 8D /r ib - TOP4MXHF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.F2.0F3A.W0 8D /r ib - TOP4MXBHF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.F3.0F3A.W0 8D /r ib - TOP4MXHBF8PS tmm1, zmm2, zmm3, imm8
+// EVEX.512.F2.0F3A.W0 8F /r ib - TOP4MXBSSPS tmm1, zmm2, zmm3, imm8
+// Operand encoding from ACE spec page 47:
+// Operand 1: ModRM:reg (rw) = tmm1
+// Operand 2: ModRM:r/m (r) = zmm2
+// Operand 3: EVEX.vvvv (r) = zmm3
+// Operand 4: imm8
+// MRMSrcReg encodes: src1→reg, src2→r/m, src3→vvvv (so swap to get spec order)
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+ def TOP4MXBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, TA, EVEX_V512;
+
+ def TOP4MXHF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, PD, TA, EVEX_V512;
+
+ def TOP4MXBHF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxbhf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, XD, TA, EVEX_V512;
+
+ def TOP4MXHBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxhbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, XS, TA, EVEX_V512;
+
+ def TOP4MXBSSPSrrri : Ii8<0x8F, MRMSrcReg, (outs TILE:$dst),
+ (ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
+ "top4mxbssps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
+ EVEX, VVVV, XD, TA, EVEX_V512;
+}
+
+//===----------------------------------------------------------------------===//
+// ACE Pseudo Instructions
+// These are used by intrinsics and expanded via custom inserter
+//===----------------------------------------------------------------------===//
+
+// Pseudo instructions for user-facing intrinsics (immediate tile IDs)
+// These use usesCustomInserter to expand to real instructions in X86ISelLowering
+let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
+ // BSRINIT pseudo - no operands, just triggers BSR initialization
+ def PBSRINIT : PseudoI<(outs), (ins),
+ [(int_x86_bsrinit)]>;
+
+ // Note: BSRMOVF, BSRMOVH, BSRMOVL intrinsics are handled directly
+ // by patterns on the real instructions (BSRMOVFrr, BSRMOVHrr_set, etc.)
+
+ // TILEMOVCOL pseudo - immediate index variant
+ // dst: tile ID (imm), src: ZMM value (VR512), idx: column index (imm)
+ // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ def PTILEMOVCOL : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+
+ // TILEMOVCOL pseudo - register index variant
+ def PTILEMOVCOL_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+
+ // TILEMOVROW pseudo - immediate index variant (ACE: write to tile)
+ // dst: tile ID (imm), src: ZMM value (VR512), idx: row index (imm)
+ // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ def PTILEMOVROW : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
+
+ // TILEMOVROW pseudo - register index variant (ACE: write to tile)
+ def PTILEMOVROW_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
+
+ // TOP2BF16PS pseudo
+ def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top2bf16ps timm:$dst, timm:$src1, timm:$src2)]>;
+
+ // TOP4 pseudos - three tile operands (dst is also accumulator)
+ def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4buud timm:$dst, timm:$src1, timm:$src2)]>;
+
+ def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4busd timm:$dst, timm:$src1, timm:$src2)]>;
+
+ def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4bssd timm:$dst, timm:$src1, timm:$src2)]>;
+
+ def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
+ [(int_x86_top4bsud timm:$dst, timm:$src1, timm:$src2)]>;
+
+ // TOP4MX pseudos - four operands (dst, src1, src2, imm)
+ def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxbhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxhbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+
+ def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
+ [(int_x86_top4mxbssps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+}
+
+// Pseudo instructions for internal intrinsics (register allocation)
+// These are used for IR-level optimization and register allocation
+// usesCustomInserter is needed to set ACEProgModel in EmitInstrWithCustomInserter
+let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
+ // TILEMOVCOL internal - output is TILE, input is ZMM + index
+ def PTILEMOVCOLV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (int_x86_tilesetcol_internal GR16:$row, GR16:$col,
+ VR512:$src, GR32:$idx))]>;
+
+ // TILEMOVROW internal - output is TILE, input is ZMM + index
+ // ACE TILEMOVROW internal - write vector to tile row
+ def PTILEMOVROWV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
+ [(set TILE:$dst,
+ (int_x86_tilesetrow_internal GR16:$row, GR16:$col,
+ VR512:$src, GR32:$idx))]>;
+
+ // TOP2BF16PS internal - output TILE, inputs: dimensions, TILE acc, two ZMMs
+ let Constraints = "$src1 = $dst" in
+ def PTOP2BF16PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top2bf16ps_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BUUD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BUUDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4buud_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BUSD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BUSDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4busd_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BSSD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BSSDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4bssd_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4BSUD internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4BSUDV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4bsud_internal GR16:$row, GR16:$col, GR16:$k,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXHFBPS internal - imm is BSR index (u8imm matches timm in pattern)
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXHF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXBHFBPS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXBHF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxbhf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXHBFBPS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXHBF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxhbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXBF8PS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXBF8PSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxbf8ps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+
+ // TOP4MXBSSPS internal
+ let Constraints = "$src1 = $dst" in
+ def PTOP4MXBSSPSV : PseudoI<(outs TILE:$dst),
+ (ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3),
+ [(set TILE:$dst,
+ (int_x86_top4mxbssps_internal GR16:$row, GR16:$col, GR16:$k, timm:$imm,
+ TILE:$src1, VR512:$src2, VR512:$src3))]>;
+}
+
+} // end HasACEV1, In64BitMode
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
index 418046479a8a0..e0aba511d6fb1 100644
--- a/llvm/lib/Target/X86/X86InstrAMX.td
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -14,8 +14,9 @@
//===----------------------------------------------------------------------===//
// AMX instructions
-multiclass AMX_TILE_COMMON<string Suffix, Predicate HasEGPR> {
-let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
+// Tile configuration instructions - shared between AMX-TILE and ACE v1
+multiclass AMX_TILE_CONFIG<string Suffix, Predicate HasEGPR> {
+let Predicates = [HasAMXTILE_Or_ACEV1, HasEGPR, In64BitMode] in {
let hasSideEffects = 1,
Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
def LDTILECFG#Suffix : I<0x49, MRM0m, (outs), (ins opaquemem:$src),
@@ -27,6 +28,12 @@ let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
"sttilecfg\t$src",
[(int_x86_sttilecfg addr:$src)]>,
T8, PD;
+}
+}
+
+// Tile load/store instructions - AMX-TILE only (not available in ACE v1)
+multiclass AMX_TILE_LOADSTORE<string Suffix, Predicate HasEGPR> {
+let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
let mayLoad = 1 in
def TILELOADD#Suffix : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst),
(ins sibmem:$src),
@@ -46,10 +53,16 @@ let Predicates = [HasAMXTILE, HasEGPR, In64BitMode] in {
}
let SchedRW = [WriteSystem] in {
- defm "" : AMX_TILE_COMMON<"", NoEGPR>, VEX;
- defm "" : AMX_TILE_COMMON<"_EVEX", HasEGPR>, EVEX, NoCD8;
+ // Tile configuration instructions - shared between AMX-TILE and ACE v1
+ defm "" : AMX_TILE_CONFIG<"", NoEGPR>, VEX;
+ defm "" : AMX_TILE_CONFIG<"_EVEX", HasEGPR>, EVEX, NoCD8;
- let Predicates = [HasAMXTILE, In64BitMode] in {
+ // Tile load/store instructions - AMX-TILE only
+ defm "" : AMX_TILE_LOADSTORE<"", NoEGPR>, VEX;
+ defm "" : AMX_TILE_LOADSTORE<"_EVEX", HasEGPR>, EVEX, NoCD8;
+
+ // TILERELEASE and TILEZERO - shared between AMX-TILE and ACE v1
+ let Predicates = [HasAMXTILE_Or_ACEV1, In64BitMode] in {
let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
def TILERELEASE : I<0x49, MRM_C0, (outs), (ins),
"tilerelease", [(int_x86_tilerelease)]>, VEX, T8, PS;
@@ -57,10 +70,26 @@ let SchedRW = [WriteSystem] in {
"tilezero\t$dst", []>,
VEX, T8, XD;
- // Pseduo instruction for RA.
+ // Pseudo instructions for RA - shared between AMX-TILE and ACE v1
let isPseudo = true, mayLoad = 1, hasSideEffects = 1,
Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
def PLDTILECFGV : PseudoI<(outs), (ins opaquemem:$src), []>;
+
+ let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
+ canFoldAsLoad = 1, usesCustomInserter = 1 in
+ def PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
+ [(set TILE:$dst, (int_x86_tilezero_internal
+ GR16:$src1, GR16:$src2))]>;
+
+ let usesCustomInserter = 1 in {
+ // Pseudo instruction for TILEZERO with immediate
+ def PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
+ [(int_x86_tilezero timm:$src)]>;
+ }
+ } // Predicates - shared
+
+ // Tile load/store pseudo instructions - AMX-TILE only
+ let Predicates = [HasAMXTILE, In64BitMode] in {
let isPseudo = true, mayLoad = 1 in
def PTILELOADDV : PseudoI<(outs TILE:$dst), (ins GR16:$src1,
GR16:$src2,
@@ -73,15 +102,9 @@ let SchedRW = [WriteSystem] in {
def PTILESTOREDV : PseudoI<(outs), (ins GR16:$src1,
GR16:$src2, opaquemem:$src3,
TILE:$src4), []>;
- let isPseudo = true, isReMaterializable = 1, isAsCheapAsAMove = 1,
- canFoldAsLoad = 1, usesCustomInserter = 1 in
- def PTILEZEROV : PseudoI<(outs TILE:$dst), (ins GR16:$src1, GR16:$src2),
- [(set TILE:$dst, (int_x86_tilezero_internal
- GR16:$src1, GR16:$src2))]>;
let usesCustomInserter = 1 in {
- // Pseudo instructions, using immediates instead of tile registers.
- // To be translated to the actual instructions in X86ISelLowering.cpp
+ // Pseudo instructions for tile load/store with immediates
let mayLoad = 1 in
def PTILELOADD : PseudoI<(outs), (ins u8imm:$src1, sibmem:$src2), []>;
let mayLoad = 1 in
@@ -89,10 +112,8 @@ let SchedRW = [WriteSystem] in {
sibmem:$src2), []>;
let mayStore = 1 in
def PTILESTORED : PseudoI<(outs), (ins i8mem:$dst, u8imm:$src), []>;
- def PTILEZERO : PseudoI<(outs), (ins u8imm:$src),
- [(int_x86_tilezero timm:$src)]>;
}
- } // Predicates
+ } // Predicates - AMX-TILE only
} // SchedRW
let Predicates = [HasAMXINT8, In64BitMode] in {
@@ -368,7 +389,7 @@ let Predicates = [HasAMXMOVRS, In64BitMode], SchedRW = [WriteSystem] in {
} // HasAMXMOVRS, In64BitMode
multiclass m_tcvtrowd2ps {
- let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+ let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
def rti : Ii8<0x7, MRMSrcReg, (outs VR512:$dst),
(ins TILE:$src1, i32u8imm:$src2),
@@ -379,12 +400,12 @@ multiclass m_tcvtrowd2ps {
"tcvtrowd2ps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
[]>, T8,XS, EVEX, VVVV, EVEX_V512;
}
- } // HasAMXAVX512, HasAVX10_2, In64BitMode
+ } // HasAMXAVX512_Or_ACEV1, In64BitMode
}
defm TCVTROWD2PS : m_tcvtrowd2ps;
-let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
let usesCustomInserter = 1 in {
def PTCVTROWD2PSrti : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
@@ -448,7 +469,7 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
multiclass AMXAVX512_BASE<bits<8> Opcode1, bits<8> Opcode2, string Opstr,
Prefix P1, Prefix P2> {
- let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode], SchedRW = [WriteSystem] in {
+ let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode], SchedRW = [WriteSystem] in {
let OpPrefix = P1 in
def rte : I<Opcode1, MRMSrcReg4VOp3, (outs VR512:$dst),
(ins TILE:$src1, GR32:$src2),
@@ -476,7 +497,7 @@ defm TCVTROWPS2BF16H : AMXAVX512_BASE<0x6d, 0x07, "tcvtrowps2bf16h", XD, XD>;
defm TCVTROWPS2BF16L : AMXAVX512_BASE<0x6d, 0x77, "tcvtrowps2bf16l", XS, XS>;
multiclass AMXAVX512_TILEMOVE<bits<8> Opcode1, bits<8> Opcode2, string Opstr> {
- let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+ let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
def rti : Ii8<Opcode1, MRMSrcReg, (outs VR512:$dst),
(ins TILE:$src1, u8imm:$src2),
@@ -487,12 +508,12 @@ multiclass AMXAVX512_TILEMOVE<bits<8> Opcode1, bits<8> Opcode2, string Opstr> {
!strconcat(Opstr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
[]>, T8, PD, EVEX, VVVV, EVEX_V512;
}
- } // HasAMXAVX512, HasAVX10_2, In64BitMode
+ } // HasAMXAVX512_Or_ACEV1, In64BitMode
}
defm TILEMOVROW : AMXAVX512_TILEMOVE<0x07, 0x4A, "tilemovrow">;
-let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
+let Predicates = [HasAMXAVX512_Or_ACEV1, In64BitMode] in {
let SchedRW = [WriteSystem] in {
let usesCustomInserter = 1 in {
def PTILEMOVROWrti : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 11a37efd28948..3b37ff8cbfd3f 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4563,7 +4563,13 @@ static unsigned getLoadStoreRegOpcode(Register Reg,
return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
case 1024:
assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
- assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
+ assert((STI.hasAMXTILE() || STI.hasACEV1()) &&
+ "Using 8*1024-bit register requires AMX-TILE or ACE");
+ // ACE v1 doesn't have TILELOADD/TILESTORED - tile spilling should be
+ // handled by X86LowerTileCopy using TILEMOVROW row-by-row.
+ // This path should only be hit for AMX targets.
+ assert(STI.hasAMXTILE() &&
+ "ACE tile spill should use TILEMOVROW, not TILELOADD/TILESTORED");
#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
: GET_EGPR_IF_ENABLED(X86::TILESTORED);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index 0c4abc2c400f6..0f2a54c1408d5 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -85,6 +85,9 @@ include "X86InstrKL.td"
// AMX instructions
include "X86InstrAMX.td"
+// ACE instructions
+include "X86InstrACE.td"
+
// RAO-INT instructions
include "X86InstrRAOINT.td"
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 1b177db55d276..c075f58913b44 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -190,6 +190,9 @@ def HasAMXCOMPLEX : Predicate<"Subtarget->hasAMXCOMPLEX()">;
def HasAMXFP8 : Predicate<"Subtarget->hasAMXFP8()">;
def HasAMXMOVRS : Predicate<"Subtarget->hasAMXMOVRS()">;
def HasAMXAVX512 : Predicate<"Subtarget->hasAMXAVX512()">;
+def HasACEV1 : Predicate<"Subtarget->hasACEV1()">;
+def HasAMXTILE_Or_ACEV1 : Predicate<"Subtarget->hasAMXTILE() || Subtarget->hasACEV1()">;
+def HasAMXAVX512_Or_ACEV1 : Predicate<"(Subtarget->hasAMXAVX512() && Subtarget->hasAVX10_2()) || Subtarget->hasACEV1()">;
def HasUINTR : Predicate<"Subtarget->hasUINTR()">;
def HasUSERMSR : Predicate<"Subtarget->hasUSERMSR()">;
def HasCRC32 : Predicate<"Subtarget->hasCRC32()">;
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index e1c829833e04a..9b97025d8532c 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -39,6 +39,7 @@
//===----------------------------------------------------------------------===//
//
#include "X86.h"
+#include "X86Subtarget.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
@@ -100,6 +101,47 @@ static bool containsAMXCode(Function &F) {
return false;
}
+// Check if the instruction is an ACE-only intrinsic.
+// ACE (Palette 2) doesn't have TILELOADD/TILESTORED instructions,
+// so volatile tile data transformation should skip ACE intrinsics.
+static bool isACEOnlyIntrinsic(Instruction *I) {
+ auto *II = dyn_cast<IntrinsicInst>(I);
+ if (!II)
+ return false;
+ switch (II->getIntrinsicID()) {
+ // ACE outer product intrinsics
+ case Intrinsic::x86_top2bf16ps_internal:
+ case Intrinsic::x86_top4buud_internal:
+ case Intrinsic::x86_top4busd_internal:
+ case Intrinsic::x86_top4bssd_internal:
+ case Intrinsic::x86_top4bsud_internal:
+ // ACE mixed precision intrinsics
+ case Intrinsic::x86_top4mxhf8ps_internal:
+ case Intrinsic::x86_top4mxbhf8ps_internal:
+ case Intrinsic::x86_top4mxhbf8ps_internal:
+ case Intrinsic::x86_top4mxbf8ps_internal:
+ case Intrinsic::x86_top4mxbssps_internal:
+ // ACE tile movement intrinsics
+ case Intrinsic::x86_tilesetcol_internal:
+ case Intrinsic::x86_tilesetrow_internal:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// Check if PHI node has any ACE-only intrinsic as incoming value.
+static bool hasACEIncomingValue(PHINode *PHI) {
+ for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) {
+ Value *Op = PHI->getIncomingValue(I);
+ if (auto *Inst = dyn_cast<Instruction>(Op)) {
+ if (isACEOnlyIntrinsic(Inst))
+ return true;
+ }
+ }
+ return false;
+}
+
static AllocaInst *createAllocaInstAtEntry(IRBuilder<> &Builder, BasicBlock *BB,
Type *Ty) {
Function &F = *BB->getParent();
@@ -210,6 +252,59 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
Col = II->getArgOperand(1);
break;
}
+ // ACE internal intrinsics - outer products with ZMM sources
+ // Pattern: (m, n, k, acc_tile, zmm1, zmm2) -> acc_tile += zmm1 * zmm2
+ case Intrinsic::x86_top2bf16ps_internal:
+ case Intrinsic::x86_top4buud_internal:
+ case Intrinsic::x86_top4busd_internal:
+ case Intrinsic::x86_top4bssd_internal:
+ case Intrinsic::x86_top4bsud_internal: {
+ switch (OpNo) {
+ case 3: // Accumulator tile
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(1);
+ break;
+ case 4: // ZMM source 1 - doesn't have tile shape, use full dimensions
+ LLVM_FALLTHROUGH;
+ case 5: // ZMM source 2
+ // ZMM sources don't need shape calculation for tile purposes
+ // but if needed: Row = m, Col = k for src1; Row = k/4, Col = n for src2
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(2);
+ break;
+ }
+ break;
+ }
+ // ACE TOP4MX intrinsics - mixed precision with BSR index
+ // Pattern: (m, n, k, bsr_idx, acc_tile, zmm1, zmm2)
+ case Intrinsic::x86_top4mxhf8ps_internal:
+ case Intrinsic::x86_top4mxbhf8ps_internal:
+ case Intrinsic::x86_top4mxhbf8ps_internal:
+ case Intrinsic::x86_top4mxbf8ps_internal:
+ case Intrinsic::x86_top4mxbssps_internal: {
+ switch (OpNo) {
+ case 4: // Accumulator tile
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(1);
+ break;
+ case 5: // ZMM source 1
+ LLVM_FALLTHROUGH;
+ case 6: // ZMM source 2
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(2);
+ break;
+ }
+ break;
+ }
+ // ACE TILEMOV intrinsics - move ZMM to tile row/column
+ // Pattern: (m, n, zmm_src, idx) -> tile
+ case Intrinsic::x86_tilesetcol_internal:
+ case Intrinsic::x86_tilesetrow_internal: {
+ // Output tile shape
+ Row = II->getArgOperand(0);
+ Col = II->getArgOperand(1);
+ break;
+ }
}
return std::make_pair(Row, Col);
@@ -715,11 +810,18 @@ bool X86VolatileTileData::volatileTileData() {
for (Instruction *I : AMXDefInsts) {
if (isIncomingOfPHI(I))
continue;
+ // Skip ACE-only intrinsics - ACE (Palette 2) doesn't have
+ // TILELOADD/TILESTORED instructions needed for volatile model.
+ if (isACEOnlyIntrinsic(I))
+ continue;
volatileTileNonPHI(I);
Changed = true;
}
for (Instruction *I : PHIInsts) {
+ // Skip PHI nodes with ACE intrinsic incoming values.
+ if (hasACEIncomingValue(dyn_cast<PHINode>(I)))
+ continue;
volatileTilePHI(dyn_cast<PHINode>(I));
Changed = true;
}
@@ -734,9 +836,18 @@ namespace {
class X86LowerAMXCast {
Function &Func;
std::unique_ptr<DominatorTree> DT;
+ bool IsACEOnly; // ACE v1 without AMX TILELOADD/TILESTORED
public:
- X86LowerAMXCast(Function &F) : Func(F), DT(nullptr) {}
+ X86LowerAMXCast(Function &F, const TargetMachine *TM = nullptr)
+ : Func(F), DT(nullptr), IsACEOnly(false) {
+ if (TM) {
+ // Check if we're ACE-only (no AMX TILELOADD/TILESTORED available)
+ const X86Subtarget *ST =
+ static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
+ IsACEOnly = ST && ST->hasACEV1() && !ST->hasAMXTILE();
+ }
+ }
bool combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
bool combineLoadCast(IntrinsicInst *Cast, LoadInst *LD);
bool combineTilezero(IntrinsicInst *Cast);
@@ -962,6 +1073,10 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
return false;
auto *II = cast<IntrinsicInst>(Tile);
+ // ACE v1 doesn't have TILESTORED instruction - skip combining for ACE targets
+ // or ACE intrinsics. ACE uses struct-based API with vector load/store.
+ if (IsACEOnly || isACEOnlyIntrinsic(II))
+ return false;
// Tile is output from AMX intrinsic. The first operand of the
// intrinsic is row, the second operand of the intrinsic is column.
Value *Row = II->getOperand(0);
@@ -992,6 +1107,10 @@ bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
// shape information through def-use chain.
if (!isAMXIntrinsic(II))
return false;
+ // ACE v1 doesn't have TILELOADD instruction - skip combining for ACE targets
+ // or ACE intrinsics. ACE uses struct-based API with vector load/store.
+ if (IsACEOnly || isACEOnlyIntrinsic(II))
+ return false;
std::tie(Row, Col) = getShape(II, OpNo);
IRBuilder<> Builder(LD);
Value *I8Ptr;
@@ -1225,6 +1344,18 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
auto *II = dyn_cast<IntrinsicInst>(U.getUser());
if (!II)
return false; // May be bitcast from x86amx to <256 x i32>.
+ // ACE v1 doesn't have TILELOADD - for ACE targets or ACE intrinsics,
+ // use tilezero since ACE tiles are typically zero-initialized in struct
+ // API.
+ if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ Value *Row = nullptr, *Col = nullptr;
+ std::tie(Row, Col) = getShape(II, OpNo);
+ Value *NewInst = Builder.CreateIntrinsic(Intrinsic::x86_tilezero_internal,
+ {}, {Row, Col});
+ AMXCast->replaceAllUsesWith(NewInst);
+ AMXCast->eraseFromParent();
+ return true;
+ }
Prepare(AMXCast->getOperand(0)->getType());
Builder.CreateStore(Src, AllocaAddr);
// TODO we can pick an constant operand for the shape.
@@ -1247,6 +1378,15 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
auto *II = dyn_cast<IntrinsicInst>(Src);
if (!II)
return false; // May be bitcast from <256 x i32> to x86amx.
+ // ACE v1 doesn't have TILESTORED - for ACE targets or ACE intrinsics,
+ // just create a poison value since the tile result is typically not read
+ // back.
+ if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ Value *Poison = PoisonValue::get(AMXCast->getType());
+ AMXCast->replaceAllUsesWith(Poison);
+ AMXCast->eraseFromParent();
+ return true;
+ }
Prepare(AMXCast->getType());
Value *Row = II->getOperand(0);
Value *Col = II->getOperand(1);
@@ -1290,7 +1430,7 @@ bool lowerAmxType(Function &F, const TargetMachine *TM,
return false;
bool C = false;
- X86LowerAMXCast LAC(F);
+ X86LowerAMXCast LAC(F, TM);
C |= LAC.combineAMXcast(TLI);
// There might be remaining AMXcast after combineAMXcast and they should be
// handled elegantly.
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 628f691279600..09082ba585572 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -36,6 +36,62 @@ using namespace llvm;
#define DEBUG_TYPE "x86-lower-tile-copy"
+/// Generate ACE-specific tile copy using TILEMOVROW row-by-row.
+/// ACE v1 doesn't have TILELOADD/TILESTORED, so we need to copy
+/// tiles through ZMM registers, one row at a time.
+///
+/// Spill sequence (tile -> stack):
+/// for row = 0 to 15:
+/// TILEMOVROW zmm_temp, tmm_src, row ; read row from tile to ZMM
+/// VMOVUPS [stack + row*64], zmm_temp ; store ZMM to stack
+///
+/// Reload sequence (stack -> tile):
+/// for row = 0 to 15:
+/// VMOVUPS zmm_temp, [stack + row*64] ; load ZMM from stack
+/// TILEMOVROW tmm_dst, zmm_temp, row ; write row from ZMM to tile
+///
+static void emitACETileCopy(MachineBasicBlock &MBB, MachineInstr &MI,
+ const X86Subtarget &ST, const X86InstrInfo *TII,
+ const TargetRegisterInfo *TRI, Register SrcReg,
+ Register DstReg, bool SrcKill, int TileSS,
+ Register ScratchZMM) {
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ // Each tile has 16 rows of 64 bytes each (1KB total)
+ const unsigned NumRows = 16;
+ const unsigned RowSize = 64;
+
+ // Spill: Read each row from source tile to ZMM, then store to stack
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // TILEMOVROW zmm, tmm, imm8 (read direction: tile -> ZMM)
+ // Uses TILEMOVROWrti instruction
+ // Only kill src on the last row read
+ bool KillSrcNow = (Row == NumRows - 1) && SrcKill;
+ BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
+ .addReg(SrcReg, getKillRegState(KillSrcNow))
+ .addImm(Row);
+
+ // VMOVUPS [stack + row*64], zmm
+ MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
+ addFrameReference(MIB, TileSS, Row * RowSize);
+ MIB.addReg(ScratchZMM, RegState::Kill);
+ }
+
+ // Reload: Load each row from stack to ZMM, then write to dest tile
+ for (unsigned Row = 0; Row < NumRows; ++Row) {
+ // VMOVUPS zmm, [stack + row*64]
+ MachineInstrBuilder MIB =
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), ScratchZMM);
+ addFrameReference(MIB, TileSS, Row * RowSize);
+
+ // TILEMOVROW tmm, zmm, imm8 (write direction: ZMM -> tile)
+ // Uses TILEMOVROWri instruction (ACE-specific)
+ BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWri), DstReg)
+ .addReg(ScratchZMM, RegState::Kill)
+ .addImm(Row);
+ }
+}
+
namespace {
class X86LowerTileCopyLegacy : public MachineFunctionPass {
@@ -71,11 +127,13 @@ FunctionPass *llvm::createX86LowerTileCopyLegacyPass() {
static bool lowerTileCopy(MachineFunction &MF) {
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
- if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ if (FuncInfo->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ FuncInfo->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
- assert(ST.hasAMXTILE() && "Only supported on AMXTILE targets");
+ assert((ST.hasAMXTILE() || ST.hasACEV1()) &&
+ "Only supported on AMX-TILE or ACE v1 targets");
const X86InstrInfo *TII = ST.getInstrInfo();
const TargetRegisterInfo *TRI = ST.getRegisterInfo();
@@ -99,63 +157,109 @@ static bool lowerTileCopy(MachineFunction &MF) {
if (!X86::TILERegClass.contains(DstReg, SrcReg))
continue;
- // Allocate stack slot for tile register
+ // Allocate stack slot for tile register (1KB for ACE, same for AMX)
unsigned Size = TRI->getSpillSize(X86::TILERegClass);
Align Alignment = TRI->getSpillAlign(X86::TILERegClass);
int TileSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
- int StrideSS = 0;
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ // Check if this is ACE-only (no AMX TILELOADD/TILESTORED available)
+ if (ST.hasACEV1() && !ST.hasAMXTILE()) {
+ // ACE v1: Use TILEMOVROW row-by-row copy
+ // Need a scratch ZMM register for the transfer
- // Pick a killed register to avoid a save/reload.
- Register GR64Cand = X86::NoRegister;
- for (auto RegT : GR64Regs.set_bits()) {
- if (UsedRegs.available(RegT)) {
- GR64Cand = RegT;
- break;
+ // Find an available ZMM register
+ BitVector VR512Regs =
+ TRI->getAllocatableSet(MF, TRI->getRegClass(X86::VR512RegClassID));
+ Register ScratchZMM = X86::NoRegister;
+ for (auto RegT : VR512Regs.set_bits()) {
+ if (UsedRegs.available(RegT)) {
+ ScratchZMM = RegT;
+ break;
+ }
}
- }
- const DebugLoc &DL = MI.getDebugLoc();
- if (GR64Cand) {
- // mov 64 %reg
- BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), GR64Cand).addImm(64);
+ if (!ScratchZMM) {
+ // If no ZMM available, use ZMM0 and save/restore it
+ ScratchZMM = X86::ZMM0;
+
+ // Allocate stack slot for scratch ZMM
+ int ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
+
+ // Save ZMM0
+ MachineInstrBuilder MIB =
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
+ addFrameReference(MIB, ZmmSS);
+ MIB.addReg(X86::ZMM0);
+
+ // Emit ACE tile copy
+ emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
+ TileSS, ScratchZMM);
+
+ // Restore ZMM0
+ MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0);
+ addFrameReference(MIB, ZmmSS);
+ } else {
+ // Use available scratch ZMM
+ emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
+ TileSS, ScratchZMM);
+ }
} else {
- // No available register? Save RAX and reload it after use.
-
- // Allocate stack slot for stride register
- Size = TRI->getSpillSize(X86::GR64RegClass);
- Alignment = TRI->getSpillAlign(X86::GR64RegClass);
- StrideSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
-
- // mov %reg (%sp)
- addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV64mr)),
- StrideSS)
- .addReg(X86::RAX);
- // mov 64 %reg
- BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), X86::RAX).addImm(64);
- }
- // tilestored %tmm, (%sp, %idx)
+ // AMX: Use TILESTORED/TILELOADD (original code)
+ int StrideSS = 0;
+
+ // Pick a killed register to avoid a save/reload.
+ Register GR64Cand = X86::NoRegister;
+ for (auto RegT : GR64Regs.set_bits()) {
+ if (UsedRegs.available(RegT)) {
+ GR64Cand = RegT;
+ break;
+ }
+ }
+
+ if (GR64Cand) {
+ // mov 64 %reg
+ BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), GR64Cand).addImm(64);
+ } else {
+ // No available register? Save RAX and reload it after use.
+
+ // Allocate stack slot for stride register
+ Size = TRI->getSpillSize(X86::GR64RegClass);
+ Alignment = TRI->getSpillAlign(X86::GR64RegClass);
+ StrideSS = MF.getFrameInfo().CreateSpillStackObject(Size, Alignment);
+
+ // mov %reg (%sp)
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV64mr)),
+ StrideSS)
+ .addReg(X86::RAX);
+ // mov 64 %reg
+ BuildMI(MBB, MI, DL, TII->get(X86::MOV64ri), X86::RAX).addImm(64);
+ }
+ // tilestored %tmm, (%sp, %idx)
#define GET_EGPR_IF_ENABLED(OPC) (ST.hasEGPR() ? OPC##_EVEX : OPC)
- unsigned Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
- MachineInstr *NewMI =
- addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS)
- .addReg(SrcReg, getKillRegState(SrcMO.isKill()));
- MachineOperand *MO = &NewMI->getOperand(X86::AddrIndexReg);
- MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
- // tileloadd (%sp, %idx), %tmm
- Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
+ unsigned Opc = GET_EGPR_IF_ENABLED(X86::TILESTORED);
+ MachineInstr *NewMI =
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc)), TileSS)
+ .addReg(SrcReg, getKillRegState(SrcMO.isKill()));
+ MachineOperand *MO = &NewMI->getOperand(X86::AddrIndexReg);
+ MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
+ // tileloadd (%sp, %idx), %tmm
+ Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD);
#undef GET_EGPR_IF_ENABLED
- NewMI = addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc), DstReg),
- TileSS);
- MO = &NewMI->getOperand(1 + X86::AddrIndexReg);
- MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
- MO->setIsKill(true);
- if (!GR64Cand) {
- // restore %rax
- // mov (%sp) %rax
- addFrameReference(
- BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), X86::RAX), StrideSS);
+ NewMI = addFrameReference(BuildMI(MBB, MI, DL, TII->get(Opc), DstReg),
+ TileSS);
+ MO = &NewMI->getOperand(1 + X86::AddrIndexReg);
+ MO->setReg(GR64Cand ? GR64Cand : X86::RAX);
+ MO->setIsKill(true);
+ if (!GR64Cand) {
+ // restore %rax
+ // mov (%sp) %rax
+ addFrameReference(
+ BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm), X86::RAX), StrideSS);
+ }
}
+
MI.eraseFromParent();
Changed = true;
}
diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp b/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp
index 7b57f7c23bf4d..3197219f9e982 100644
--- a/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp
+++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.cpp
@@ -15,7 +15,8 @@ using namespace llvm;
yaml::X86MachineFunctionInfo::X86MachineFunctionInfo(
const llvm::X86MachineFunctionInfo &MFI)
- : AMXProgModel(MFI.getAMXProgModel()) {}
+ : AMXProgModel(MFI.getAMXProgModel()), ACEProgModel(MFI.getACEProgModel()) {
+}
void yaml::X86MachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
MappingTraits<X86MachineFunctionInfo>::mapping(YamlIO, *this);
@@ -31,6 +32,7 @@ MachineFunctionInfo *X86MachineFunctionInfo::clone(
void X86MachineFunctionInfo::initializeBaseYamlFields(
const yaml::X86MachineFunctionInfo &YamlMFI) {
AMXProgModel = YamlMFI.AMXProgModel;
+ ACEProgModel = YamlMFI.ACEProgModel;
}
void X86MachineFunctionInfo::anchor() { }
diff --git a/llvm/lib/Target/X86/X86MachineFunctionInfo.h b/llvm/lib/Target/X86/X86MachineFunctionInfo.h
index 1cfe86d57071f..f5691aaf4df03 100644
--- a/llvm/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/llvm/lib/Target/X86/X86MachineFunctionInfo.h
@@ -24,6 +24,7 @@
namespace llvm {
enum AMXProgModelEnum { None = 0, DirectReg = 1, ManagedRA = 2 };
+enum ACEProgModelEnum { ACE_None = 0, ACE_DirectReg = 1, ACE_ManagedRA = 2 };
class X86MachineFunctionInfo;
@@ -36,8 +37,17 @@ template <> struct ScalarEnumerationTraits<AMXProgModelEnum> {
}
};
+template <> struct ScalarEnumerationTraits<ACEProgModelEnum> {
+ static void enumeration(IO &YamlIO, ACEProgModelEnum &Value) {
+ YamlIO.enumCase(Value, "None", ACEProgModelEnum::ACE_None);
+ YamlIO.enumCase(Value, "DirectReg", ACEProgModelEnum::ACE_DirectReg);
+ YamlIO.enumCase(Value, "ManagedRA", ACEProgModelEnum::ACE_ManagedRA);
+ }
+};
+
struct X86MachineFunctionInfo final : public yaml::MachineFunctionInfo {
AMXProgModelEnum AMXProgModel;
+ ACEProgModelEnum ACEProgModel;
X86MachineFunctionInfo() = default;
X86MachineFunctionInfo(const llvm::X86MachineFunctionInfo &MFI);
@@ -49,6 +59,7 @@ struct X86MachineFunctionInfo final : public yaml::MachineFunctionInfo {
template <> struct MappingTraits<X86MachineFunctionInfo> {
static void mapping(IO &YamlIO, X86MachineFunctionInfo &MFI) {
YamlIO.mapOptional("amxProgModel", MFI.AMXProgModel);
+ YamlIO.mapOptional("aceProgModel", MFI.ACEProgModel);
}
};
} // end namespace yaml
@@ -131,6 +142,9 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
/// The AMX programing model used in the function.
AMXProgModelEnum AMXProgModel = AMXProgModelEnum::None;
+ /// The ACE programming model used in the function.
+ ACEProgModelEnum ACEProgModel = ACEProgModelEnum::ACE_None;
+
/// True if this function has a subset of CSRs that is handled explicitly via
/// copies.
bool IsSplitCSR = false;
@@ -260,6 +274,14 @@ class X86MachineFunctionInfo : public MachineFunctionInfo {
AMXProgModel = Model;
}
+ ACEProgModelEnum getACEProgModel() const { return ACEProgModel; }
+ void setACEProgModel(ACEProgModelEnum Model) {
+ assert(
+ (ACEProgModel == ACEProgModelEnum::ACE_None || ACEProgModel == Model) &&
+ "mixed ACE model is not supported");
+ ACEProgModel = Model;
+ }
+
SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
return ForwardedMustTailRegParms;
}
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 103b977ed7c84..3bad529686943 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -270,8 +270,9 @@ bool X86PreTileConfigImpl::runOnMachineFunction(MachineFunction &MF) {
scope_exit ClearStateOnExit([this] { releaseMemory(); });
X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
@@ -450,7 +451,10 @@ bool X86PreTileConfigImpl::runOnMachineFunction(MachineFunction &MF) {
.addReg(Xmm);
}
// Fill in the palette first.
- addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), SS).addImm(1);
+ // Use Palette 2 for ACE v1, Palette 1 for AMX.
+ int PaletteId = ST.hasACEV1() ? 2 : 1;
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), SS)
+ .addImm(PaletteId);
return true;
}
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 940825aa9b77f..1ee8f5ccb6db5 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -716,7 +716,7 @@ unsigned X86RegisterInfo::getNumSupportedRegs(const MachineFunction &MF) const {
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
if (ST.hasEGPR())
return X86::NUM_TARGET_REGS;
- if (ST.hasAMXTILE())
+ if (ST.hasAMXTILE() || ST.hasACEV1())
return X86::TMM7 + 1;
if (ST.hasAVX512())
return X86::K6_K7 + 1;
@@ -1154,7 +1154,20 @@ static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
case X86::PTDPBF8PSV:
case X86::PTDPBHF8PSV:
case X86::PTDPHBF8PSV:
- case X86::PTDPHF8PSV: {
+ case X86::PTDPHF8PSV:
+ // ACE internal pseudos - same pattern: operands 1,2 are row,col
+ case X86::PTOP2BF16PSV:
+ case X86::PTOP4BUUDV:
+ case X86::PTOP4BUSDV:
+ case X86::PTOP4BSSDV:
+ case X86::PTOP4BSUDV:
+ case X86::PTOP4MXHF8PSV:
+ case X86::PTOP4MXBHF8PSV:
+ case X86::PTOP4MXHBF8PSV:
+ case X86::PTOP4MXBF8PSV:
+ case X86::PTOP4MXBSSPSV:
+ case X86::PTILEMOVCOLV:
+ case X86::PTILEMOVROWV: {
MachineOperand &MO1 = MI->getOperand(1);
MachineOperand &MO2 = MI->getOperand(2);
ShapeT Shape(&MO1, &MO2, MRI);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index 63ad4b2bae5ac..158fceb552eaf 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -421,6 +421,8 @@ def KPAIRS : RegisterTuples<[sub_mask_0, sub_mask_1],
let PositionOrder = 3 in {
// Tile config registers.
def TMMCFG: X86Reg<"tmmcfg", 0>;
+// Block Scale Register (ACE)
+def BSR0 : X86Reg<"bsr0", 0>;
// Tile "registers".
def TMM0: X86Reg<"tmm0", 0>;
def TMM1: X86Reg<"tmm1", 1>;
@@ -857,6 +859,13 @@ let CopyCost = -1 in // Don't allow copying of tile registers
def TILE : RegisterClass<"X86", [x86amx], 8192,
(sequence "TMM%u", 0, 7)> {let Size = 8192;}
+// Block Scale Registers (ACE)
+let CopyCost = -1 in // Don't allow copying of BSR register
+def BSR : RegisterClass<"X86", [untyped], 0, (add BSR0)> {
+ let Size = 1024;
+ let isAllocatable = 0;
+}
+
//===----------------------------------------------------------------------===//
// Register categories.
//
diff --git a/llvm/lib/Target/X86/X86TileConfig.cpp b/llvm/lib/Target/X86/X86TileConfig.cpp
index 660bd0a77ab79..6667c8d1a6687 100644
--- a/llvm/lib/Target/X86/X86TileConfig.cpp
+++ b/llvm/lib/Target/X86/X86TileConfig.cpp
@@ -78,8 +78,9 @@ static bool tileConfig(MachineFunction &MF,
llvm::function_ref<LiveIntervals *()> GetLIs,
llvm::function_ref<VirtRegMap *()> GetVRM) {
X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
- // Early exit in the common case of non-AMX code.
- if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
+ // Early exit in the common case of non-AMX/ACE code.
+ if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA &&
+ X86FI->getACEProgModel() != ACEProgModelEnum::ACE_ManagedRA)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
@@ -137,6 +138,11 @@ static bool tileConfig(MachineFunction &MF,
}
// Fill in the shape of each tile physical register.
+ // ACE Palette 2 uses fixed tile dimensions (16x64), so skip writing
+ // shapes - bytes 1-63 must remain zero for ACE.
+ if (ST.hasACEV1())
+ return true;
+
for (unsigned I = 0; I < AMXRegNum; ++I) {
if (!Phys2Virt[I])
continue;
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index a8909072b037c..8f25a02de09b4 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -639,6 +639,7 @@ constexpr FeatureBitset ImpliedFeaturesAMX_FP8 = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_MOVRS = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_AVX512 =
FeatureAMX_TILE | FeatureAVX10_2;
+constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAMX_AVX512;
constexpr FeatureBitset ImpliedFeaturesHRESET = {};
constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {};
diff --git a/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll b/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
new file mode 100644
index 0000000000000..9afca28b9c9ac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
@@ -0,0 +1,244 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f | FileCheck %s
+
+;
+; ACE Intrinsic Tests
+;
+
+; Test BSRINIT intrinsic
+define void @test_bsrinit() {
+; CHECK-LABEL: test_bsrinit:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrinit %bsr0
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrinit()
+ ret void
+}
+
+; Test BSRMOVF intrinsic
+define void @test_bsrmovf(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_bsrmovf:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrmovf(<16 x i32> %a, <16 x i32> %b)
+ ret void
+}
+
+; Test BSRMOVH set intrinsic
+define void @test_bsrmovh_set(<16 x i32> %a) {
+; CHECK-LABEL: test_bsrmovh_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %zmm0, %bsr0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrmovh.set(<16 x i32> %a)
+ ret void
+}
+
+; Test BSRMOVH get intrinsic
+define <16 x i32> @test_bsrmovh_get() {
+; CHECK-LABEL: test_bsrmovh_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, %zmm0
+; CHECK-NEXT: retq
+ %result = call <16 x i32> @llvm.x86.bsrmovh.get()
+ ret <16 x i32> %result
+}
+
+; Test BSRMOVL set intrinsic
+define void @test_bsrmovl_set(<16 x i32> %a) {
+; CHECK-LABEL: test_bsrmovl_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %zmm0, %bsr0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.bsrmovl.set(<16 x i32> %a)
+ ret void
+}
+
+; Test BSRMOVL get intrinsic
+define <16 x i32> @test_bsrmovl_get() {
+; CHECK-LABEL: test_bsrmovl_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %bsr0, %zmm0
+; CHECK-NEXT: retq
+ %result = call <16 x i32> @llvm.x86.bsrmovl.get()
+ ret <16 x i32> %result
+}
+
+; Test TOP2BF16PS intrinsic
+define void @test_top2bf16ps() {
+; CHECK-LABEL: test_top2bf16ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top2bf16ps %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BUUD intrinsic
+define void @test_top4buud() {
+; CHECK-LABEL: test_top4buud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4buud %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BUSD intrinsic
+define void @test_top4busd() {
+; CHECK-LABEL: test_top4busd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4busd %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BSSD intrinsic
+define void @test_top4bssd() {
+; CHECK-LABEL: test_top4bssd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bssd %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4BSUD intrinsic
+define void @test_top4bsud() {
+; CHECK-LABEL: test_top4bsud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bsud %zmm2, %zmm1, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
+ ret void
+}
+
+; Test TOP4MXHF8PS intrinsic
+define void @test_top4mxhf8ps() {
+; CHECK-LABEL: test_top4mxhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXBHF8PS intrinsic
+define void @test_top4mxbhf8ps() {
+; CHECK-LABEL: test_top4mxbhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbhf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXHBF8PS intrinsic
+define void @test_top4mxhbf8ps() {
+; CHECK-LABEL: test_top4mxhbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhbf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXBF8PS intrinsic
+define void @test_top4mxbf8ps() {
+; CHECK-LABEL: test_top4mxbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbf8ps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TOP4MXBSSPS intrinsic (MX INT8 S×S with BSR scaling)
+define void @test_top4mxbssps() {
+; CHECK-LABEL: test_top4mxbssps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbssps $5, %zmm1, %zmm2, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
+ ret void
+}
+
+; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
+define void @test_tilemovcol(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovcol:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %src, i32 3)
+ ret void
+}
+
+; Test TILEMOVROW intrinsic - write to tile (ACE extension)
+; Uses constant index - generates immediate form
+define void @test_tilemovrow_to_tile(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovrow_to_tile:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetrow(i8 1, <16 x i32> %src, i32 5)
+ ret void
+}
+
+; Test TILEMOVCOL with register index
+define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovcol_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetcol(i8 2, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Test TILEMOVROW with register index
+define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovrow_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ call void @llvm.x86.tilesetrow(i8 3, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Intrinsic declarations
+declare void @llvm.x86.bsrinit()
+declare void @llvm.x86.bsrmovf(<16 x i32>, <16 x i32>)
+declare void @llvm.x86.bsrmovh.set(<16 x i32>)
+declare <16 x i32> @llvm.x86.bsrmovh.get()
+declare void @llvm.x86.bsrmovl.set(<16 x i32>)
+declare <16 x i32> @llvm.x86.bsrmovl.get()
+declare void @llvm.x86.tilesetcol(i8, <16 x i32>, i32)
+declare void @llvm.x86.tilesetrow(i8, <16 x i32>, i32)
+declare void @llvm.x86.top2bf16ps(i8, i8, i8)
+declare void @llvm.x86.top4buud(i8, i8, i8)
+declare void @llvm.x86.top4busd(i8, i8, i8)
+declare void @llvm.x86.top4bssd(i8, i8, i8)
+declare void @llvm.x86.top4bsud(i8, i8, i8)
+declare void @llvm.x86.top4mxhf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxbhf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxhbf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxbf8ps(i8, i8, i8, i8)
+declare void @llvm.x86.top4mxbssps(i8, i8, i8, i8)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll b/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
new file mode 100644
index 0000000000000..4b114d5bc9c08
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
@@ -0,0 +1,99 @@
+; ACE v1 Greedy Register Allocation Test
+; Tests that tile registers are allocated correctly in a separate pass
+; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
+;
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+
+; Test basic tile register allocation for ACE
+define void @test_acev1_ra(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra
+ ; CHECK: PLDTILECFGV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTOP2BF16PSV
+ ; CHECK: PTILEMOVROWrtiV
+entry:
+ %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract using TILEMOVROW (ACE v1 pattern)
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test multiple tile operations requiring allocation
+define void @test_acev1_ra_multi(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra_multi
+ ; CHECK: PLDTILECFGV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTOP2BF16PSV
+entry:
+ %c1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %c2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c2, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test integer outer products register allocation
+define void @test_acev1_ra_int(ptr %out, <64 x i8> %a_i8) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra_int
+ ; CHECK: PLDTILECFGV
+ ; CHECK: PTILEZEROV
+ ; CHECK: PTOP4BSSDV
+ ; CHECK: PTOP4BUUDV
+entry:
+ %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d1 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d2 = tail call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test register allocation with control flow
+define void @test_acev1_ra_branch(ptr %out, i32 %cond, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
+ ; CHECK-LABEL: name: test_acev1_ra_branch
+ ; CHECK: PLDTILECFGV
+entry:
+ %c = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %icmp = icmp eq i32 %cond, 0
+ br i1 %icmp, label %then, label %else
+
+then:
+ %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ br label %merge
+
+else:
+ %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_b, <32 x bfloat> %zmm_a)
+ br label %merge
+
+merge:
+ %result = phi x86_amx [ %d1, %then ], [ %d2, %else ]
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %result, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll b/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
new file mode 100644
index 0000000000000..47228dfb8e614
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
@@ -0,0 +1,204 @@
+; ACE v1 Outer Product Operations Test
+; Tests all ACE outer product variants: BF16, INT8 (signed/unsigned combinations)
+; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
+;
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+
+; Test BF16 outer product (TOP2BF16PS)
+define void @test_top2bf16ps(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
+; CHECK-LABEL: test_top2bf16ps:
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: top2bf16ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract using TILEMOVROW (ACE v1 pattern)
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test unsigned x unsigned -> dword (TOP4BUUD)
+define void @test_top4buud(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4buud:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4buud
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test signed x signed -> dword (TOP4BSSD)
+define void @test_top4bssd(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4bssd:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4bssd
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test unsigned x signed -> dword (TOP4BUSD)
+define void @test_top4busd(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4busd:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4busd
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test signed x unsigned -> dword (TOP4BSUD)
+define void @test_top4bsud(ptr %out, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_top4bsud:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4bsud
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 HF8xHF8 -> FP32 with BSR scaling (TOP4MXHF8PS)
+define void @test_top4mxhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxhf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxhf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 BF8xHF8 -> FP32 with BSR scaling (TOP4MXBHF8PS)
+define void @test_top4mxbhf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxbhf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxbhf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 HF8xBF8 -> FP32 with BSR scaling (TOP4MXHBF8PS)
+define void @test_top4mxhbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxhbf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxhbf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX FP8 BF8xBF8 -> FP32 with BSR scaling (TOP4MXBF8PS)
+define void @test_top4mxbf8ps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxbf8ps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxbf8ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test MX INT8 SxS -> FP32 with BSR scaling (TOP4MXBSSPS)
+define void @test_top4mxbssps_internal(ptr %out, <16 x i32> %zmm_a, <16 x i32> %zmm_b) {
+; CHECK-LABEL: test_top4mxbssps_internal:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top4mxbssps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %d = call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %c, <16 x i32> %zmm_a, <16 x i32> %zmm_b)
+
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+; Test chained outer products (accumulation) - simplified to avoid spills
+define void @test_chained_outer_products(ptr %out, <32 x bfloat> %zmm_bf16, <64 x i8> %a_i8) {
+; CHECK-LABEL: test_chained_outer_products:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top2bf16ps
+; CHECK: top4bssd
+; CHECK: top4buud
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; BF16 outer product
+ %d1 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_bf16, <32 x bfloat> %zmm_bf16)
+
+ ; Integer outer products
+ %d2 = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d1, <64 x i8> %a_i8, <64 x i8> %a_i8)
+ %d3 = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %d2, <64 x i8> %a_i8, <64 x i8> %a_i8)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4busd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4bsud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4mxhf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxbhf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxhbf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxbf8ps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare x86_amx @llvm.x86.top4mxbssps.internal(i16, i16, i16, i8, x86_amx, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll b/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
new file mode 100644
index 0000000000000..2d94260701e40
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
@@ -0,0 +1,94 @@
+; ACE v1 Basic Tile Operations Test
+; Tests basic ACE tile operations: tilezero, outer products
+; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
+;
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+
+define void @test_acev1_basic(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
+; CHECK-LABEL: test_acev1_basic:
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: top2bf16ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; ACE BF16 outer product
+ %d = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract result using TILEMOVROW (ACE v1 pattern - no TILESTORED)
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test ACE integer outer products
+define void @test_acev1_int_outer_products(ptr %out, <64 x i8> %a_i8, <64 x i8> %b_i8) {
+; CHECK-LABEL: test_acev1_int_outer_products:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK-DAG: top4buud
+; CHECK-DAG: top4bssd
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; Unsigned x Unsigned -> DWORD
+ %d0 = call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %c, <64 x i8> %a_i8, <64 x i8> %b_i8)
+
+ ; Signed x Signed -> DWORD
+ %d1 = call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %d0, <64 x i8> %a_i8, <64 x i8> %b_i8)
+
+ ; Extract result using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test multiple outer product accumulations
+define void @test_acev1_accumulation(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
+; CHECK-LABEL: test_acev1_accumulation:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: top2bf16ps
+; CHECK: top2bf16ps
+; CHECK: top2bf16ps
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; Multiple accumulations
+ %d0 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %c, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d1 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d0, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+ %d2 = call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %d1, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b)
+
+ ; Extract result using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+; Test basic tilezero operation
+define void @test_acev1_tilezero(ptr %out) {
+; CHECK-LABEL: test_acev1_tilezero:
+; CHECK: ldtilecfg
+; CHECK: tilezero
+; CHECK: tilemovrow
+; CHECK: tilerelease
+ %c = call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ ; Extract using TILEMOVROW
+ %row0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %c, i32 0)
+ store volatile <16 x i32> %row0, ptr %out, align 64
+
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4buud.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ipra-reg-usage.ll b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
index c008d691ee841..9d84ffdd48163 100644
--- a/llvm/test/CodeGen/X86/ipra-reg-usage.ll
+++ b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
@@ -7,7 +7,7 @@
target triple = "x86_64-unknown-unknown"
declare void @bar1()
define preserve_allcc void @foo()#0 {
-; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hip $hsp $ip $mxcsr $rflags $rip $riz $rsp $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $r11b $r11bh $r11d $r11w $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
+; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hip $hsp $ip $mxcsr $rflags $rip $riz $rsp $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $r11b $r11bh $r11d $r11w $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $bsr0 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
call void @bar1()
call void @bar2()
ret void
@@ -15,7 +15,7 @@ define preserve_allcc void @foo()#0 {
declare void @bar2()
define preserve_nonecc void @foo2()#0 {
-; CHECK: foo2 Clobbered Registers: $ah $al $ax $ch $cl $cs $cx $df $dh $di $dih $dil $dl $ds $dx $eax $ecx $edi $edx $eflags $eip $eiz $es $esi $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hax $hcx $hdi $hdx $hip $hsi $hsp $ip $mxcsr $rax $rcx $rdi $rdx $rflags $rip $riz $rsi $rsp $si $sih $sil $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r8 $r9 $r10 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm0 $xmm1 $xmm2 $xmm3 $xmm4 $xmm5 $xmm6 $xmm7 $xmm8 $xmm9 $xmm10 $xmm11 $xmm12 $xmm13 $xmm14 $xmm15 $r8b $r9b $r10b $r11b $r8bh $r9bh $r10bh $r11bh $r8d $r9d $r10d $r11d $r8w $r9w $r10w $r11w $r8wh $r9wh $r10wh $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
+; CHECK: foo2 Clobbered Registers: $ah $al $ax $ch $cl $cs $cx $df $dh $di $dih $dil $dl $ds $dx $eax $ecx $edi $edx $eflags $eip $eiz $es $esi $esp $fpcw $fpsw $fs $fs_base $gs $gs_base $hax $hcx $hdi $hdx $hip $hsi $hsp $ip $mxcsr $rax $rcx $rdi $rdx $rflags $rip $riz $rsi $rsp $si $sih $sil $sp $sph $spl $ss $ssp $_eflags $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r8 $r9 $r10 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm0 $xmm1 $xmm2 $xmm3 $xmm4 $xmm5 $xmm6 $xmm7 $xmm8 $xmm9 $xmm10 $xmm11 $xmm12 $xmm13 $xmm14 $xmm15 $r8b $r9b $r10b $r11b $r8bh $r9bh $r10bh $r11bh $r8d $r9d $r10d $r11d $r8w $r9w $r10w $r11w $r8wh $r9wh $r10wh $r11wh $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $k0_k1 $k2_k3 $k4_k5 $k6_k7 $tmmcfg $bsr0 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $r16 $r17 $r18 $r19 $r20 $r21 $r22 $r23 $r24 $r25 $r26 $r27 $r28 $r29 $r30 $r31 $r16b $r17b $r18b $r19b $r20b $r21b $r22b $r23b $r24b $r25b $r26b $r27b $r28b $r29b $r30b $r31b $r16bh $r17bh $r18bh $r19bh $r20bh $r21bh $r22bh $r23bh $r24bh $r25bh $r26bh $r27bh $r28bh $r29bh $r30bh $r31bh $r16d $r17d $r18d $r19d $r20d $r21d $r22d $r23d $r24d $r25d $r26d $r27d $r28d $r29d $r30d $r31d $r16w $r17w $r18w $r19w $r20w $r21w $r22w $r23w $r24w $r25w $r26w $r27w $r28w $r29w $r30w $r31w $r16wh $r17wh $r18wh $r19wh $r20wh $r21wh $r22wh $r23wh $r24wh $r25wh $r26wh $r27wh $r28wh $r29wh $r30wh $r31wh
call void @bar1()
call void @bar2()
ret void
diff --git a/llvm/test/MC/X86/ACE/ace-att.s b/llvm/test/MC/X86/ACE/ace-att.s
new file mode 100644
index 0000000000000..cf43cb899bb2c
--- /dev/null
+++ b/llvm/test/MC/X86/ACE/ace-att.s
@@ -0,0 +1,109 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1 -show-encoding %s | FileCheck %s
+
+// CHECK: bsrinit %bsr0
+// CHECK: encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
+ bsrinit %bsr0
+
+// CHECK: bsrmovf %zmm2, %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc2]
+ bsrmovf %zmm2, %zmm1, %bsr0
+
+// CHECK: bsrmovf (%rax), %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
+ bsrmovf (%rax), %zmm1, %bsr0
+
+// CHECK: bsrmovh %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0xc1]
+ bsrmovh %zmm1, %bsr0
+
+// CHECK: bsrmovh (%rbx), %bsr0
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0x03]
+ bsrmovh (%rbx), %bsr0
+
+// CHECK: bsrmovh %bsr0, %zmm1
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc1]
+ bsrmovh %bsr0, %zmm1
+
+// CHECK: bsrmovh %bsr0, (%rcx)
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0x01]
+ bsrmovh %bsr0, (%rcx)
+
+// CHECK: bsrmovl %zmm2, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc2]
+ bsrmovl %zmm2, %bsr0
+
+// CHECK: bsrmovl (%rdx), %bsr0
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0x02]
+ bsrmovl (%rdx), %bsr0
+
+// CHECK: bsrmovl %bsr0, %zmm3
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc3]
+ bsrmovl %bsr0, %zmm3
+
+// CHECK: bsrmovl %bsr0, (%rsi)
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0x06]
+ bsrmovl %bsr0, (%rsi)
+
+// CHECK: tilemovcol $5, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xca,0x05]
+ tilemovcol $5, %zmm2, %tmm1
+
+// CHECK: tilemovcol %ecx, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x48,0x4b,0xca]
+ tilemovcol %ecx, %zmm2, %tmm1
+
+// CHECK: tilemovrow $3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x07,0xca,0x03]
+ tilemovrow $3, %zmm2, %tmm1
+
+// CHECK: tilemovrow %edx, %zmm3, %tmm2
+// CHECK: encoding: [0x62,0xf2,0xed,0x48,0x4a,0xd3]
+ tilemovrow %edx, %zmm3, %tmm2
+
+// CHECK: top2bf16ps %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5c,0xca]
+ top2bf16ps %zmm3, %zmm2, %tmm1
+
+// CHECK: top4buud %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x64,0x48,0x5e,0xca]
+ top4buud %zmm3, %zmm2, %tmm1
+
+// CHECK: top4busd %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x65,0x48,0x5e,0xca]
+ top4busd %zmm3, %zmm2, %tmm1
+
+// CHECK: top4bssd %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x5e,0xca]
+ top4bssd %zmm3, %zmm2, %tmm1
+
+// CHECK: top4bsud %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5e,0xca]
+ top4bsud %zmm3, %zmm2, %tmm1
+
+// CHECK: top4mxbf8ps $7, %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0x64,0x48,0x8d,0xca,0x07]
+ top4mxbf8ps $7, %zmm3, %zmm2, %tmm1
+
+// CHECK: top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
+// CHECK: encoding: [0x62,0xf3,0x65,0x48,0x8d,0xca,0x07]
+ top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
+
+// CHECK: top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
+// CHECK: encoding: [0x62,0xf3,0x5f,0x48,0x8d,0xd5,0x03]
+ top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
+
+// CHECK: top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
+// CHECK: encoding: [0x62,0xf3,0x4e,0x48,0x8d,0xdf,0x01]
+ top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
+
+// CHECK: top4mxbssps $15, %zmm8, %zmm9, %tmm4
+// CHECK: encoding: [0x62,0xd3,0x3f,0x48,0x8f,0xe1,0x0f]
+ top4mxbssps $15, %zmm8, %zmm9, %tmm4
+
+// CHECK: top4buud %zmm31, %zmm30, %tmm7
+// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+ top4buud %zmm31, %zmm30, %tmm7
+
+// CHECK: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
+ bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
diff --git a/llvm/test/MC/X86/ACE/ace-error.s b/llvm/test/MC/X86/ACE/ace-error.s
new file mode 100644
index 0000000000000..e939b84356403
--- /dev/null
+++ b/llvm/test/MC/X86/ACE/ace-error.s
@@ -0,0 +1,39 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1 %s -o /dev/null 2>&1 | FileCheck %s
+
+// Test missing explicit BSR operand
+// CHECK: error: BSR instruction requires explicit %bsr0 operand
+bsrmovf %zmm1, %zmm2
+
+// CHECK: error:
+tilemovcol %tmm1, %xmm2, $5
+
+// CHECK: error:
+top4busd %tmm1, %ymm2, %zmm3
+
+// CHECK: error:
+tilemovrow $256, %zmm2, %tmm1
+
+// Test feature gating - ACEV1 instructions require ACEV1 feature
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=-acev1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOACEV1
+
+// NOACEV1: error:
+bsrinit %bsr0
+
+// NOACEV1: error:
+top2bf16ps %zmm3, %zmm2, %tmm1
+
+// Test that ACEV1 requires AMX-TILE (dependency check)
+// RUN: not llvm-mc -triple x86_64-unknown-unknown -mattr=+acev1,-amx-tile %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NOAMX
+
+// NOAMX: error:
+tilemovcol $5, %zmm2, %tmm1
+
+// Test invalid BSR register (only bsr0 exists)
+// CHECK: error:
+bsrmovf %zmm1, %zmm2, %bsr1
+
+// Test 32-bit mode rejection (ACEV1 is 64-bit only)
+// RUN: not llvm-mc -triple i386-unknown-unknown -mattr=+acev1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=NO32BIT
+
+// NO32BIT: error:
+bsrinit %bsr0
diff --git a/llvm/test/MC/X86/ACE/ace-intel.s b/llvm/test/MC/X86/ACE/ace-intel.s
new file mode 100644
index 0000000000000..81b5d8e0f7e24
--- /dev/null
+++ b/llvm/test/MC/X86/ACE/ace-intel.s
@@ -0,0 +1,109 @@
+// RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 -mattr=+acev1 --show-encoding %s | FileCheck %s
+
+// CHECK: bsrinit bsr0
+// CHECK: encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
+ bsrinit bsr0
+
+// CHECK: bsrmovf bsr0, zmm1, zmm2
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc2]
+ bsrmovf bsr0, zmm1, zmm2
+
+// CHECK: bsrmovf bsr0, zmm1, zmmword ptr [rax]
+// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
+ bsrmovf bsr0, zmm1, zmmword ptr [rax]
+
+// CHECK: bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
+// CHECK: encoding: [0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10]
+ bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
+
+// CHECK: bsrmovh bsr0, zmm1
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0xc1]
+ bsrmovh bsr0, zmm1
+
+// CHECK: bsrmovh bsr0, zmmword ptr [rbx]
+// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0x03]
+ bsrmovh bsr0, zmmword ptr [rbx]
+
+// CHECK: bsrmovh zmm1, bsr0
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc1]
+ bsrmovh zmm1, bsr0
+
+// CHECK: bsrmovh zmmword ptr [rcx], bsr0
+// CHECK: encoding: [0x62,0xf6,0x7f,0x48,0x95,0x01]
+ bsrmovh zmmword ptr [rcx], bsr0
+
+// CHECK: bsrmovl bsr0, zmm2
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc2]
+ bsrmovl bsr0, zmm2
+
+// CHECK: bsrmovl bsr0, zmmword ptr [rdx]
+// CHECK: encoding: [0x62,0xf6,0xfe,0x48,0x95,0x02]
+ bsrmovl bsr0, zmmword ptr [rdx]
+
+// CHECK: bsrmovl zmm3, bsr0
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc3]
+ bsrmovl zmm3, bsr0
+
+// CHECK: bsrmovl zmmword ptr [rsi], bsr0
+// CHECK: encoding: [0x62,0xf6,0x7e,0x48,0x95,0x06]
+ bsrmovl zmmword ptr [rsi], bsr0
+
+// CHECK: tilemovcol tmm1, zmm2, 5
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xca,0x05]
+ tilemovcol tmm1, zmm2, 5
+
+// CHECK: tilemovcol tmm1, zmm2, ecx
+// CHECK: encoding: [0x62,0xf2,0xf5,0x48,0x4b,0xca]
+ tilemovcol tmm1, zmm2, ecx
+
+// CHECK: tilemovrow tmm1, zmm2, 3
+// CHECK: encoding: [0x62,0xf3,0xfd,0x48,0x07,0xca,0x03]
+ tilemovrow tmm1, zmm2, 3
+
+// CHECK: tilemovrow tmm2, zmm3, edx
+// CHECK: encoding: [0x62,0xf2,0xed,0x48,0x4a,0xd3]
+ tilemovrow tmm2, zmm3, edx
+
+// CHECK: top2bf16ps tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5c,0xca]
+ top2bf16ps tmm1, zmm2, zmm3
+
+// CHECK: top4buud tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x64,0x48,0x5e,0xca]
+ top4buud tmm1, zmm2, zmm3
+
+// CHECK: top4busd tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x65,0x48,0x5e,0xca]
+ top4busd tmm1, zmm2, zmm3
+
+// CHECK: top4bssd tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x5e,0xca]
+ top4bssd tmm1, zmm2, zmm3
+
+// CHECK: top4bsud tmm1, zmm2, zmm3
+// CHECK: encoding: [0x62,0xf2,0x66,0x48,0x5e,0xca]
+ top4bsud tmm1, zmm2, zmm3
+
+// CHECK: top4mxbf8ps tmm1, zmm2, zmm3, 7
+// CHECK: encoding: [0x62,0xf3,0x64,0x48,0x8d,0xca,0x07]
+ top4mxbf8ps tmm1, zmm2, zmm3, 7
+
+// CHECK: top4mxhf8ps tmm1, zmm2, zmm3, 7
+// CHECK: encoding: [0x62,0xf3,0x65,0x48,0x8d,0xca,0x07]
+ top4mxhf8ps tmm1, zmm2, zmm3, 7
+
+// CHECK: top4mxbhf8ps tmm2, zmm4, zmm5, 3
+// CHECK: encoding: [0x62,0xf3,0x57,0x48,0x8d,0xd4,0x03]
+ top4mxbhf8ps tmm2, zmm4, zmm5, 3
+
+// CHECK: top4mxhbf8ps tmm3, zmm6, zmm7, 1
+// CHECK: encoding: [0x62,0xf3,0x46,0x48,0x8d,0xde,0x01]
+ top4mxhbf8ps tmm3, zmm6, zmm7, 1
+
+// CHECK: top4mxbssps tmm4, zmm8, zmm9, 15
+// CHECK: encoding: [0x62,0xd3,0x37,0x48,0x8f,0xe0,0x0f]
+ top4mxbssps tmm4, zmm8, zmm9, 15
+
+// CHECK: top4buud tmm7, zmm30, zmm31
+// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+ top4buud tmm7, zmm30, zmm31
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index 07466b31e7bb7..2d2e02f1b9968 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -354,6 +354,10 @@ static const X86FoldTableEntry Table2Addr[] = {
};
static const X86FoldTableEntry Table0[] = {
+ {X86::BSRMOVHrr_get, X86::BSRMOVHmr_get, TB_FOLDED_STORE},
+ {X86::BSRMOVHrr_set, X86::BSRMOVHrm_set, TB_FOLDED_LOAD},
+ {X86::BSRMOVLrr_get, X86::BSRMOVLmr_get, TB_FOLDED_STORE},
+ {X86::BSRMOVLrr_set, X86::BSRMOVLrm_set, TB_FOLDED_LOAD},
{X86::BT16ri8, X86::BT16mi8, TB_FOLDED_LOAD},
{X86::BT32ri8, X86::BT32mi8, TB_FOLDED_LOAD},
{X86::BT64ri8, X86::BT64mi8, TB_FOLDED_LOAD},
@@ -684,6 +688,7 @@ static const X86FoldTableEntry Table1[] = {
{X86::BLSR64rr, X86::BLSR64rm, 0},
{X86::BLSR64rr_EVEX, X86::BLSR64rm_EVEX, 0},
{X86::BLSR64rr_NF, X86::BLSR64rm_NF, 0},
+ {X86::BSRMOVFrr, X86::BSRMOVFrm, 0},
{X86::BZHI32rr, X86::BZHI32rm, 0},
{X86::BZHI32rr_EVEX, X86::BZHI32rm_EVEX, 0},
{X86::BZHI32rr_NF, X86::BZHI32rm_NF, 0},
>From 56eec16de8c5ea98573a05f4aff7ac7fd785b350 Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Tue, 1 Sep 2026 18:07:40 +0530
Subject: [PATCH 2/2] Address review comments
- Rename the ACE tests and directories to acev1 for consistency
- Build acev1 on top of avx10.1 rather than avx10.2
- Fix test encodings and the BSR0 implicit defs
- Make the scale register type consistent with the spec
- Add memory operand tests for bsrmov
- Confirm the ACEV1 ISA implication chain
- Take the outer product ZMM sources by value instead of by register ID,
so they are allocated conventionally, and mark the 512-bit ACE builtins
with RequiredVectorWidth<512>
- Use pseudos for SPILL and RELOAD. Spills use scratch zmm. Don't use
non-ace ISA.
- Fix missing vector r/m in MRMSrcReg4VOp3.
---
clang/include/clang/Basic/BuiltinsX86_64.td | 80 ++--
clang/lib/Headers/acev1intrin.h | 148 ++++----
clang/lib/Sema/SemaX86.cpp | 17 +-
clang/test/CodeGen/X86/ace-api.c | 145 -------
clang/test/CodeGen/X86/ace-builtins.c | 119 ------
clang/test/CodeGen/X86/ace-inline-asm.c | 85 -----
clang/test/CodeGen/X86/acev1.c | 37 +-
clang/test/CodeGen/X86/acev1_api.c | 55 ++-
clang/test/CodeGen/X86/acev1_bsr.c | 18 +-
clang/test/CodeGen/X86/acev1_errors.c | 10 +-
clang/test/CodeGen/X86/acev1_inline_asm.c | 67 +++-
clang/test/CodeGen/X86/acev1_mxfp8.c | 44 +--
clang/test/CodeGen/X86/acev1_tile_movement.c | 31 +-
clang/test/CodeGen/attr-target-x86.c | 2 +-
clang/test/Preprocessor/x86_target_features.c | 24 +-
llvm/include/llvm/IR/IntrinsicsX86.td | 99 +++--
.../lib/Target/X86/AsmParser/X86AsmParser.cpp | 2 +-
.../X86/MCTargetDesc/X86MCCodeEmitter.cpp | 4 +-
llvm/lib/Target/X86/X86.td | 13 +-
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 24 +-
llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 2 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 25 +-
llvm/lib/Target/X86/X86InstrACE.td | 133 ++++---
llvm/lib/Target/X86/X86InstrInfo.cpp | 11 +-
llvm/lib/Target/X86/X86LowerAMXType.cpp | 32 +-
llvm/lib/Target/X86/X86LowerTileCopy.cpp | 176 +++++----
llvm/lib/Target/X86/X86RegisterInfo.cpp | 3 +
llvm/lib/TargetParser/X86TargetParser.cpp | 7 +-
llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll | 244 ------------
.../X86/{ACE => ACEV1}/acev1-greedy-ra.ll | 13 +-
.../CodeGen/X86/ACEV1/acev1-intrinsics.ll | 357 ++++++++++++++++++
.../X86/{ACE => ACEV1}/acev1-outer-product.ll | 12 +-
llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll | 67 ++++
.../X86/{ACE => ACEV1}/acev1-tile-basic.ll | 12 +-
.../CodeGen/X86/ACEV1/acev1-tile-spill.ll | 124 ++++++
llvm/test/MC/Disassembler/X86/acev1.txt | 262 +++++++++++++
.../X86/{ACE/ace-att.s => ACEV1/acev1-att.s} | 62 ++-
.../{ACE/ace-error.s => ACEV1/acev1-error.s} | 0
.../{ACE/ace-intel.s => ACEV1/acev1-intel.s} | 38 +-
39 files changed, 1549 insertions(+), 1055 deletions(-)
delete mode 100644 clang/test/CodeGen/X86/ace-api.c
delete mode 100644 clang/test/CodeGen/X86/ace-builtins.c
delete mode 100644 clang/test/CodeGen/X86/ace-inline-asm.c
delete mode 100644 llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
rename llvm/test/CodeGen/X86/{ACE => ACEV1}/acev1-greedy-ra.ll (87%)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
rename llvm/test/CodeGen/X86/{ACE => ACEV1}/acev1-outer-product.ll (94%)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
rename llvm/test/CodeGen/X86/{ACE => ACEV1}/acev1-tile-basic.ll (88%)
create mode 100644 llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
create mode 100644 llvm/test/MC/Disassembler/X86/acev1.txt
rename llvm/test/MC/X86/{ACE/ace-att.s => ACEV1/acev1-att.s} (63%)
rename llvm/test/MC/X86/{ACE/ace-error.s => ACEV1/acev1-error.s} (100%)
rename llvm/test/MC/X86/{ACE/ace-intel.s => ACEV1/acev1-intel.s} (74%)
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index f2f52f029455f..20389049c42b2 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -225,7 +225,7 @@ let Features = "amx-tile", Attributes = [NoThrow] in {
def tilestored64_internal : X86Builtin<"void(unsigned short, unsigned short, void *, size_t, _Vector<256, int>)">;
}
-let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
+let Features = "amx-tile", Attributes = [NoThrow] in {
def tilezero_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short)">;
}
@@ -258,7 +258,7 @@ let Features = "amx-fp8", Attributes = [NoThrow] in {
def tdphf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<256, int>, _Vector<256, int>)">;
}
-let Features = "amx-tile|acev1", Attributes = [NoThrow] in {
+let Features = "amx-tile", Attributes = [NoThrow] in {
def tile_loadconfig : X86Builtin<"void(void const *)">;
def tile_storeconfig : X86Builtin<"void(void const *)">;
def tilerelease : X86Builtin<"void()">;
@@ -393,52 +393,56 @@ let Features = "movrs,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<512>]
def vmovrsw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short const *>)">;
}
-// ACE (AI Compute Extensions) Builtins
-// Tile movement - single builtin handles both immediate and register index forms
-// The lowering code checks if index is constant and selects appropriate instruction
-let Features = "acev1", Attributes = [NoThrow] in {
- def tilesetcol : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
- // ACE tilemovrow writes TO tile (AMX tilemovrow reads FROM tile)
- def tilesetrow : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+// ACE (AI Compute Extensions) builtins.
+// Tile movement - a single builtin handles both the immediate and register index
+// forms; the lowering code checks whether the index is constant and selects the
+// appropriate instruction.
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def tilemovcol_set : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
+ def tilemovrow_set : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, unsigned int)">;
}
-// Tile movement internal (for IR-level operations)
-let Features = "acev1", Attributes = [NoThrow] in {
- def tilesetcol_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
- // ACE tilemovrow_to_tile writes TO tile (AMX tilemovrow reads FROM tile)
- def tilesetrow_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+// Tile movement internal (for IR-level operations).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def tilemovcol_set_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
+ // ACE tilemovrow writes to the tile, whereas AMX tilemovrow reads from it.
+ def tilemovrow_set_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, _Vector<16, int>, unsigned int)">;
}
-// BSR management
+// BSR management.
let Features = "acev1", Attributes = [NoThrow] in {
def bsrinit : X86Builtin<"void()">;
- def bsrmovf : X86Builtin<"void(_Vector<16, int>, _Vector<16, int>)">;
- def bsrmovh_set : X86Builtin<"void(_Vector<16, int>)">;
- def bsrmovh_get : X86Builtin<"_Vector<16, int>()">;
- def bsrmovl_set : X86Builtin<"void(_Vector<16, int>)">;
- def bsrmovl_get : X86Builtin<"_Vector<16, int>()">;
}
-// Outer products - user-facing (immediate tile IDs)
-let Features = "acev1", Attributes = [NoThrow] in {
- def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4buud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4busd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4bssd : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4bsud : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def bsrmovf : X86Builtin<"void(_Vector<64, char>, _Vector<64, char>)">;
+ def bsrmovh_set : X86Builtin<"void(_Vector<64, char>)">;
+ def bsrmovh_get : X86Builtin<"_Vector<64, char>()">;
+ def bsrmovl_set : X86Builtin<"void(_Vector<64, char>)">;
+ def bsrmovl_get : X86Builtin<"_Vector<64, char>()">;
}
-// Mixed precision outer products - user-facing
-let Features = "acev1", Attributes = [NoThrow] in {
- def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
- def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Constant unsigned char, _Constant unsigned char, _Constant unsigned char)">;
+// Outer products - user-facing (immediate tile ID, ZMM source values).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def top2bf16ps : X86Builtin<"void(_Constant unsigned char, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
+ def top4buud : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4busd : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bssd : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
+ def top4bsud : X86Builtin<"void(_Constant unsigned char, _Vector<64, signed char>, _Vector<64, signed char>)">;
}
-// Outer products - internal (for IR-level operations with tile values)
-let Features = "acev1", Attributes = [NoThrow] in {
+// Mixed precision outer products - user-facing.
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
+ def top4mxhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxbhf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxhbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxbf8ps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+ def top4mxbssps : X86Builtin<"void(_Constant unsigned char, _Vector<16, int>, _Vector<16, int>, _Constant unsigned char)">;
+}
+
+// Outer products - internal (for IR-level operations with tile values).
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def top2bf16ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
def top4buud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
def top4busd_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
@@ -446,8 +450,8 @@ let Features = "acev1", Attributes = [NoThrow] in {
def top4bsud_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, _Vector<256, int>, _Vector<64, signed char>, _Vector<64, signed char>)">;
}
-// Mixed precision internal
-let Features = "acev1", Attributes = [NoThrow] in {
+// Mixed precision internal.
+let Features = "acev1", Attributes = [NoThrow, RequiredVectorWidth<512>] in {
def top4mxhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
def top4mxbhf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
def top4mxhbf8ps_internal : X86Builtin<"_Vector<256, int>(unsigned short, unsigned short, unsigned short, unsigned char, _Vector<256, int>, _Vector<16, int>, _Vector<16, int>)">;
diff --git a/clang/lib/Headers/acev1intrin.h b/clang/lib/Headers/acev1intrin.h
index 0491c7f0b383d..3cbab91759d2f 100644
--- a/clang/lib/Headers/acev1intrin.h
+++ b/clang/lib/Headers/acev1intrin.h
@@ -21,7 +21,7 @@
/// Load tile configuration from a 64-byte memory location. For ACE
/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1),
-/// ACE tiles have fixed dimensions of 16 rows × 64 bytes, so per-tile
+/// ACE tiles have fixed dimensions of 16 rows x 64 bytes, so per-tile
/// row/column configuration bytes are ignored. If palette_id is zero,
/// tiles return to init state and are zeroed. Invalid configurations
/// result in #GP fault.
@@ -39,7 +39,7 @@ _tile_ace_loadconfig(const void *__config) {
/// Store the current tile configuration to a 64-byte memory location.
/// For ACE (Palette 2), the stored palette_id will be 2 and per-tile
-/// configuration bytes reflect the fixed 16×64 dimensions. If tiles
+/// configuration bytes reflect the fixed 16x64 dimensions. If tiles
/// are not configured, all zeroes are stored.
///
/// \headerfile <immintrin.h>
@@ -65,7 +65,7 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
}
/// Zero the ACE tile specified by "tile". Sets all 1024 bytes
-/// (16 rows × 64 bytes) of the tile register to zero.
+/// (16 rows x 64 bytes) of the tile register to zero.
///
/// \headerfile <immintrin.h>
///
@@ -86,11 +86,11 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// \param dst
/// Destination tile register ID (0-7).
/// \param src
-/// Source ZMM register ID (0-31).
+/// Source ZMM vector (__m512i) containing 16 doublewords.
/// \param idx
/// Column index (0-15). Immediate or register form selected automatically.
#define _tile_setcol(dst, src, idx) \
- __builtin_ia32_tilesetcol((dst), (src), (idx))
+ __builtin_ia32_tilemovcol_set((dst), (src), (idx))
/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written
/// as a horizontal row in the tile at the specified index.
@@ -102,15 +102,16 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) {
/// \param dst
/// Destination tile register ID (0-7).
/// \param src
-/// Source ZMM register ID (0-31).
+/// Source ZMM vector (__m512i) containing 16 doublewords.
/// \param idx
/// Row index (0-15). Immediate or register form selected automatically.
#define _tile_setrow(dst, src, idx) \
- __builtin_ia32_tilesetrow((dst), (src), (idx))
+ __builtin_ia32_tilemovrow_set((dst), (src), (idx))
-/// Initialize the Block Scale Register (BSR) to zero. The BSR holds
-/// 32 scaling factors used by mixed-precision outer product instructions
-/// (TOP4MX* variants). Must be called before using BSR-scaled operations.
+/// Initialize the Block Scale Register (BSR), setting all 128 scale bytes to
+/// 0x7F (the E8M0 encoding of 1.0), the same state LDTILECFG leaves it in.
+/// The BSR holds the scale factors used by the mixed-precision outer product
+/// instructions (TOP4MX* variants).
///
/// \headerfile <immintrin.h>
///
@@ -119,67 +120,66 @@ static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) {
__builtin_ia32_bsrinit();
}
-/// Load the full BSR (32 scale factors) from two ZMM registers. The low
-/// half (16 factors) comes from __src1, high half from __src2. Each
-/// doubleword contains one scale factor.
+/// Load the full BSR (128 scale bytes) from two ZMM registers. The high half
+/// (A scales) comes from __src1 and the low half (B scales) from __src2.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVF </c> instruction.
///
/// \param __src1
-/// ZMM with scale factors for BSR low half (factors 0-15).
+/// ZMM with the 64 A-operand scale bytes for the BSR high half.
/// \param __src2
-/// ZMM with scale factors for BSR high half (factors 16-31).
+/// ZMM with the 64 B-operand scale bytes for the BSR low half.
static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1,
__m512i __src2) {
- __builtin_ia32_bsrmovf((__v16si)__src1, (__v16si)__src2);
+ __builtin_ia32_bsrmovf((__v64qi)__src1, (__v64qi)__src2);
}
-/// Load the high half of BSR (scale factors 16-31) from a ZMM register.
+/// Load the high half of BSR (64 A-operand scale bytes) from a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
///
/// \param __src
-/// ZMM with 16 scale factors to write to BSR high half.
+/// ZMM with 64 scale bytes to write to the BSR high half.
static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) {
- __builtin_ia32_bsrmovh_set((__v16si)__src);
+ __builtin_ia32_bsrmovh_set((__v64qi)__src);
}
-/// Read the high half of BSR (scale factors 16-31) to a ZMM register.
+/// Read the high half of BSR (64 A-operand scale bytes) to a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction.
///
/// \returns
-/// ZMM containing 16 scale factors from BSR high half.
+/// ZMM containing the 64 scale bytes from the BSR high half.
static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) {
return (__m512i)__builtin_ia32_bsrmovh_get();
}
-/// Load the low half of BSR (scale factors 0-15) from a ZMM register.
+/// Load the low half of BSR (64 B-operand scale bytes) from a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
///
/// \param __src
-/// ZMM with 16 scale factors to write to BSR low half.
+/// ZMM with 64 scale bytes to write to the BSR low half.
static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) {
- __builtin_ia32_bsrmovl_set((__v16si)__src);
+ __builtin_ia32_bsrmovl_set((__v64qi)__src);
}
-/// Read the low half of BSR (scale factors 0-15) to a ZMM register.
+/// Read the low half of BSR (64 B-operand scale bytes) to a ZMM register.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction.
///
/// \returns
-/// ZMM containing 16 scale factors from BSR low half.
+/// ZMM containing the 64 scale bytes from the BSR low half.
static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
return (__m512i)__builtin_ia32_bsrmovl_get();
}
@@ -195,13 +195,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 32 BF16 values.
+/// First source ZMM vector (__m512bh) containing 32 BF16 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 32 BF16 values.
+/// Second source ZMM vector (__m512bh) containing 32 BF16 values.
#define _tile_top2bf16ps(dst, src1, src2) \
- __builtin_ia32_top2bf16ps((dst), (src1), (src2))
+ __builtin_ia32_top2bf16ps((dst), (__v32bf)(src1), (__v32bf)(src2))
-/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Compute 4-way outer product of unsigned x unsigned bytes to INT32.
/// Each group of 4 unsigned byte pairs from src1 and src2 produces
/// 4 products accumulated into the destination tile as 32-bit integers.
///
@@ -212,13 +212,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// First source ZMM vector (__m512i) containing 64 unsigned bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// Second source ZMM vector (__m512i) containing 64 unsigned bytes.
#define _tile_top4buud(dst, src1, src2) \
- __builtin_ia32_top4buud((dst), (src1), (src2))
+ __builtin_ia32_top4buud((dst), (__v64qs)(src1), (__v64qs)(src2))
-/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Compute 4-way outer product of unsigned x signed bytes to INT32.
/// Each group of 4 byte pairs (unsigned from src1, signed from src2)
/// produces 4 products accumulated into the destination tile.
///
@@ -229,13 +229,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// First source ZMM vector (__m512i) containing 64 unsigned bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+/// Second source ZMM vector (__m512i) containing 64 signed bytes.
#define _tile_top4busd(dst, src1, src2) \
- __builtin_ia32_top4busd((dst), (src1), (src2))
+ __builtin_ia32_top4busd((dst), (__v64qs)(src1), (__v64qs)(src2))
-/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Compute 4-way outer product of signed x signed bytes to INT32.
/// Each group of 4 signed byte pairs produces 4 products accumulated
/// into the destination tile as 32-bit integers.
///
@@ -246,13 +246,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// First source ZMM vector (__m512i) containing 64 signed bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 signed bytes.
+/// Second source ZMM vector (__m512i) containing 64 signed bytes.
#define _tile_top4bssd(dst, src1, src2) \
- __builtin_ia32_top4bssd((dst), (src1), (src2))
+ __builtin_ia32_top4bssd((dst), (__v64qs)(src1), (__v64qs)(src2))
-/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Compute 4-way outer product of signed x unsigned bytes to INT32.
/// Each group of 4 byte pairs (signed from src1, unsigned from src2)
/// produces 4 products accumulated into the destination tile.
///
@@ -263,15 +263,15 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing 64 signed bytes.
+/// First source ZMM vector (__m512i) containing 64 signed bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing 64 unsigned bytes.
+/// Second source ZMM vector (__m512i) containing 64 unsigned bytes.
#define _tile_top4bsud(dst, src1, src2) \
- __builtin_ia32_top4bsud((dst), (src1), (src2))
+ __builtin_ia32_top4bsud((dst), (__v64qs)(src1), (__v64qs)(src2))
/// Compute 4-way mixed precision outer product with HF8 (E4M3) format.
-/// Multiplies HF8 values from src1 with BF8 values from src2, converting
-/// to FP32 and accumulating into the destination tile.
+/// Multiplies HF8 values from both sources, converting to FP32 and
+/// accumulating into the destination tile.
///
/// \headerfile <immintrin.h>
///
@@ -280,13 +280,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing HF8 values.
+/// First source ZMM vector (__m512i) containing HF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing BF8 values.
+/// Second source ZMM vector (__m512i) containing HF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxhf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxhf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxhf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product with BF8/HF8 format.
/// Multiplies BF8 values from src1 with HF8 (E4M3) values from src2,
@@ -299,13 +299,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing BF8 values.
+/// First source ZMM vector (__m512i) containing BF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing HF8 values.
+/// Second source ZMM vector (__m512i) containing HF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxbhf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbhf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxbhf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product with HF8/BF8 format.
/// Multiplies HF8 (E4M3) values from src1 with BF8 values from src2,
@@ -318,13 +318,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing HF8 values.
+/// First source ZMM vector (__m512i) containing HF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing BF8 values.
+/// Second source ZMM vector (__m512i) containing BF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxhbf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxhbf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxhbf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product with BF8 (E5M2) format.
/// Multiplies BF8 values from both sources, converting to FP32 and
@@ -337,13 +337,13 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing BF8 values.
+/// First source ZMM vector (__m512i) containing BF8 values.
/// \param src2
-/// Second source ZMM register ID (0-31) containing BF8 values.
+/// Second source ZMM vector (__m512i) containing BF8 values.
/// \param imm
/// 8-bit control immediate for scaling/rounding options.
#define _tile_top4mxbf8ps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbf8ps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxbf8ps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
/// Compute 4-way mixed precision outer product of signed INT8 with BSR
/// scaling. Multiplies signed bytes, applies scale factors from the BSR,
@@ -356,15 +356,15 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) {
/// \param dst
/// Destination/accumulator tile register ID (0-7).
/// \param src1
-/// First source ZMM register ID (0-31) containing signed bytes.
+/// First source ZMM vector (__m512i) containing signed bytes.
/// \param src2
-/// Second source ZMM register ID (0-31) containing signed bytes.
+/// Second source ZMM vector (__m512i) containing signed bytes.
/// \param imm
/// 8-bit immediate selecting BSR scale factors to apply.
#define _tile_top4mxbssps(dst, src1, src2, imm) \
- __builtin_ia32_top4mxbssps((dst), (src1), (src2), (imm))
+ __builtin_ia32_top4mxbssps((dst), (__v16si)(src1), (__v16si)(src2), (imm))
-/// ACE tile type with fixed dimensions (16 rows × 64 bytes = 1024 bytes).
+/// ACE tile type with fixed dimensions (16 rows x 64 bytes = 1024 bytes).
/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1.
typedef int __acetile __attribute__((__vector_size__(1024), __aligned__(64)));
@@ -399,9 +399,9 @@ static __inline__ void __tile_ace_zero(__acetile *dst) {
*dst = __builtin_ia32_tilezero_internal(16, 64);
}
-/// Compute 4-way outer product of unsigned×unsigned bytes to INT32.
+/// Compute 4-way outer product of unsigned x unsigned bytes to INT32.
/// Multiplies 64 unsigned bytes from each ZMM source, producing a
-/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+/// 16x16 grid of 32-bit sums accumulated into the ACE tile.
///
/// \headerfile <immintrin.h>
///
@@ -420,9 +420,9 @@ static __inline__ void __tile_ace_top4buud(__acetile *dst, __m512i src1,
(__v64qs)src2);
}
-/// Compute 4-way outer product of unsigned×signed bytes to INT32.
+/// Compute 4-way outer product of unsigned x signed bytes to INT32.
/// Multiplies 64 bytes from each ZMM source (unsigned from src1,
-/// signed from src2), producing a 16×16 grid of 32-bit sums.
+/// signed from src2), producing a 16x16 grid of 32-bit sums.
///
/// \headerfile <immintrin.h>
///
@@ -441,9 +441,9 @@ static __inline__ void __tile_ace_top4busd(__acetile *dst, __m512i src1,
(__v64qs)src2);
}
-/// Compute 4-way outer product of signed×signed bytes to INT32.
+/// Compute 4-way outer product of signed x signed bytes to INT32.
/// Multiplies 64 signed bytes from each ZMM source, producing a
-/// 16×16 grid of 32-bit sums accumulated into the ACE tile.
+/// 16x16 grid of 32-bit sums accumulated into the ACE tile.
///
/// \headerfile <immintrin.h>
///
@@ -462,9 +462,9 @@ static __inline__ void __tile_ace_top4bssd(__acetile *dst, __m512i src1,
(__v64qs)src2);
}
-/// Compute 4-way outer product of signed×unsigned bytes to INT32.
+/// Compute 4-way outer product of signed x unsigned bytes to INT32.
/// Multiplies 64 bytes from each ZMM source (signed from src1,
-/// unsigned from src2), producing a 16×16 grid of 32-bit sums.
+/// unsigned from src2), producing a 16x16 grid of 32-bit sums.
///
/// \headerfile <immintrin.h>
///
@@ -484,7 +484,7 @@ static __inline__ void __tile_ace_top4bsud(__acetile *dst, __m512i src1,
}
/// Compute 2-way outer product of BF16 to FP32.
-/// Multiplies 32 BF16 values from each source, producing a 16×16 grid
+/// Multiplies 32 BF16 values from each source, producing a 16x16 grid
/// of FP32 sums accumulated into the ACE tile.
///
/// \headerfile <immintrin.h>
@@ -620,7 +620,7 @@ static __inline__ void __tile_ace_top2bf16ps(__acetile *dst, __m512bh src1,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
unsigned int idx) {
- *dst = __builtin_ia32_tilesetcol_internal(16, 64, (__v16si)src, idx);
+ *dst = __builtin_ia32_tilemovcol_set_internal(16, 64, (__v16si)src, idx);
}
/// Write a ZMM vector as a row in an ACE tile.
@@ -639,7 +639,7 @@ static __inline__ void __tile_ace_setcol(__acetile *dst, __m512i src,
__DEFAULT_FN_ATTRS_ACE
static __inline__ void __tile_ace_setrow(__acetile *dst, __m512i src,
unsigned int idx) {
- *dst = __builtin_ia32_tilesetrow_internal(16, 64, (__v16si)src, idx);
+ *dst = __builtin_ia32_tilemovrow_set_internal(16, 64, (__v16si)src, idx);
}
/// Read a row from an ACE tile to a ZMM vector.
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index 94c2257b970aa..38135d2981cfd 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -495,6 +495,18 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tcvtrowps2phl:
case X86::BI__builtin_ia32_tcvtrowd2ps:
case X86::BI__builtin_ia32_tilemovrow:
+ // The ACE outer products name only the accumulator tile; their two sources
+ // are ZMM values, so neither the range nor the duplicate rule applies there.
+ case X86::BI__builtin_ia32_top2bf16ps:
+ case X86::BI__builtin_ia32_top4buud:
+ case X86::BI__builtin_ia32_top4busd:
+ case X86::BI__builtin_ia32_top4bssd:
+ case X86::BI__builtin_ia32_top4bsud:
+ case X86::BI__builtin_ia32_top4mxhf8ps:
+ case X86::BI__builtin_ia32_top4mxbhf8ps:
+ case X86::BI__builtin_ia32_top4mxhbf8ps:
+ case X86::BI__builtin_ia32_top4mxbf8ps:
+ case X86::BI__builtin_ia32_top4mxbssps:
return CheckBuiltinTileArgumentsRange(TheCall, 0);
case X86::BI__builtin_ia32_tdpbssd:
case X86::BI__builtin_ia32_tdpbsud:
@@ -508,11 +520,6 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tdpbhf8ps:
case X86::BI__builtin_ia32_tdphbf8ps:
case X86::BI__builtin_ia32_tdphf8ps:
- case X86::BI__builtin_ia32_top2bf16ps:
- case X86::BI__builtin_ia32_top4buud:
- case X86::BI__builtin_ia32_top4busd:
- case X86::BI__builtin_ia32_top4bssd:
- case X86::BI__builtin_ia32_top4bsud:
return CheckBuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
case X86::BI__builtin_ia32_tcvtrowps2bf16hi:
case X86::BI__builtin_ia32_tcvtrowps2bf16li:
diff --git a/clang/test/CodeGen/X86/ace-api.c b/clang/test/CodeGen/X86/ace-api.c
deleted file mode 100644
index 8f82fbf66e39a..0000000000000
--- a/clang/test/CodeGen/X86/ace-api.c
+++ /dev/null
@@ -1,145 +0,0 @@
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
-// RUN: -emit-llvm -o - -Werror | FileCheck %s
-
-// Test ACE struct-based API functions using __acetile type
-// ACE v1 uses fixed 16x64 tile dimensions (Palette 2)
-
-#include <immintrin.h>
-
-// CHECK-LABEL: @test_ace_top4buud
-// CHECK: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> %{{.*}})
-// CHECK: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-// CHECK: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx %{{.*}})
-void test_ace_top4buud(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4buud(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4busd
-// CHECK: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-void test_ace_top4busd(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4busd(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4bssd
-// CHECK: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-void test_ace_top4bssd(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4bssd(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4bsud
-// CHECK: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
-void test_ace_top4bsud(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4bsud(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top2bf16ps
-// CHECK: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
-void test_ace_top2bf16ps(__acetile *dst, __m512bh a, __m512bh b) {
- __tile_ace_top2bf16ps(dst, a, b);
-}
-
-// CHECK-LABEL: @test_ace_top4mxhf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxhf8ps.internal(i16 16, i16 64, i16 64, i8 5, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxhf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxhf8ps(dst, a, b, 5);
-}
-
-// CHECK-LABEL: @test_ace_top4mxbhf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxbhf8ps.internal(i16 16, i16 64, i16 64, i8 3, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxbhf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxbhf8ps(dst, a, b, 3);
-}
-
-// CHECK-LABEL: @test_ace_top4mxhbf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxhbf8ps.internal(i16 16, i16 64, i16 64, i8 7, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxhbf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxhbf8ps(dst, a, b, 7);
-}
-
-// CHECK-LABEL: @test_ace_top4mxbf8ps
-// CHECK: call x86_amx @llvm.x86.top4mxbf8ps.internal(i16 16, i16 64, i16 64, i8 1, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxbf8ps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxbf8ps(dst, a, b, 1);
-}
-
-// CHECK-LABEL: @test_ace_top4mxbssps
-// CHECK: call x86_amx @llvm.x86.top4mxbssps.internal(i16 16, i16 64, i16 64, i8 9, x86_amx %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_ace_top4mxbssps(__acetile *dst, __m512i a, __m512i b) {
- __tile_ace_top4mxbssps(dst, a, b, 9);
-}
-
-// CHECK-LABEL: @test_ace_setcol
-// CHECK: call x86_amx @llvm.x86.tilesetcol.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
-void test_ace_setcol(__acetile *dst, __m512i src) {
- __tile_ace_setcol(dst, src, 3);
-}
-
-// CHECK-LABEL: @test_ace_setrow
-// CHECK: call x86_amx @llvm.x86.tilesetrow.internal(i16 16, i16 64, <16 x i32> %{{.*}}, i32 %{{.*}})
-void test_ace_setrow(__acetile *dst, __m512i src) {
- __tile_ace_setrow(dst, src, 5);
-}
-
-// CHECK-LABEL: @test_ace_getrow
-// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512i test_ace_getrow(__acetile *src) {
- return __tile_ace_getrow(src, 7);
-}
-
-// CHECK-LABEL: @test_ace_zero
-// CHECK: call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
-void test_ace_zero(__acetile *dst) {
- __tile_ace_zero(dst);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowd2ps
-// CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512 test_ace_cvtrowd2ps(__acetile *src) {
- return __tile_ace_cvtrowd2ps(src, 2);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2bf16h
-// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512bh test_ace_cvtrowps2bf16h(__acetile *src) {
- return __tile_ace_cvtrowps2bf16h(src, 4);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2bf16l
-// CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512bh test_ace_cvtrowps2bf16l(__acetile *src) {
- return __tile_ace_cvtrowps2bf16l(src, 6);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2phh
-// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512h test_ace_cvtrowps2phh(__acetile *src) {
- return __tile_ace_cvtrowps2phh(src, 8);
-}
-
-// CHECK-LABEL: @test_ace_cvtrowps2phl
-// CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx %{{.*}}, i32 %{{.*}})
-__m512h test_ace_cvtrowps2phl(__acetile *src) {
- return __tile_ace_cvtrowps2phl(src, 10);
-}
-
-// CHECK-LABEL: @test_ace_workflow
-// CHECK: call x86_amx @llvm.x86.tilezero.internal
-// CHECK: call x86_amx @llvm.x86.tilesetrow.internal
-// CHECK: call x86_amx @llvm.x86.top4buud.internal
-// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
-void test_ace_workflow(__m512i *input, __m512i *output) {
- __acetile acc;
-
- // Zero the tile
- __tile_ace_zero(&acc);
-
- // Initialize with data via setrow
- __tile_ace_setrow(&acc, input[0], 0);
-
- // Perform outer product computation
- __tile_ace_top4buud(&acc, input[1], input[2]);
-
- // Extract result via getrow
- output[0] = __tile_ace_getrow(&acc, 0);
-}
diff --git a/clang/test/CodeGen/X86/ace-builtins.c b/clang/test/CodeGen/X86/ace-builtins.c
deleted file mode 100644
index edaa80c78486f..0000000000000
--- a/clang/test/CodeGen/X86/ace-builtins.c
+++ /dev/null
@@ -1,119 +0,0 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +acev1 -target-feature +avx512f -emit-llvm -o - | FileCheck %s
-
-#include <immintrin.h>
-
-// BSR Management Tests
-
-// CHECK-LABEL: @test_bsrinit
-// CHECK: call void @llvm.x86.bsrinit()
-void test_bsrinit(void) {
- _bsr0_init();
-}
-
-// CHECK-LABEL: @test_bsrmovf
-// CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
-void test_bsrmovf(__m512i a, __m512i b) {
- _bsr0_movf(a, b);
-}
-
-// CHECK-LABEL: @test_bsrmovh_set
-// CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
-void test_bsrmovh_set(__m512i a) {
- _bsr0_movh_set(a);
-}
-
-// CHECK-LABEL: @test_bsrmovh_get
-// CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
-__m512i test_bsrmovh_get(void) {
- return _bsr0_movh_get();
-}
-
-// CHECK-LABEL: @test_bsrmovl_set
-// CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
-void test_bsrmovl_set(__m512i a) {
- _bsr0_movl_set(a);
-}
-
-// CHECK-LABEL: @test_bsrmovl_get
-// CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
-__m512i test_bsrmovl_get(void) {
- return _bsr0_movl_get();
-}
-
-// Tile Movement Tests
-
-// CHECK-LABEL: @test_tilemovcol
-// CHECK: call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %{{.*}}, i32 5)
-void test_tilemovcol(__m512i src) {
- _tile_setcol(0, src, 5);
-}
-
-// CHECK-LABEL: @test_tilemovrow_to_tile
-// CHECK: call void @llvm.x86.tilesetrow(i8 0, <16 x i32> %{{.*}}, i32 5)
-void test_tilemovrow_to_tile(__m512i src) {
- _tile_setrow(0, src, 5);
-}
-
-// Outer Product Tests
-
-// CHECK-LABEL: @test_top2bf16ps
-// CHECK: call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
-void test_top2bf16ps(void) {
- _tile_top2bf16ps(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4buud
-// CHECK: call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
-void test_top4buud(void) {
- _tile_top4buud(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4busd
-// CHECK: call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
-void test_top4busd(void) {
- _tile_top4busd(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4bssd
-// CHECK: call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
-void test_top4bssd(void) {
- _tile_top4bssd(0, 1, 2);
-}
-
-// CHECK-LABEL: @test_top4bsud
-// CHECK: call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
-void test_top4bsud(void) {
- _tile_top4bsud(0, 1, 2);
-}
-
-// Mixed Precision Outer Product Tests
-
-// CHECK-LABEL: @test_top4mxhf8ps
-// CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxhf8ps(void) {
- _tile_top4mxhf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxbhf8ps
-// CHECK: call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxbhf8ps(void) {
- _tile_top4mxbhf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxhbf8ps
-// CHECK: call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxhbf8ps(void) {
- _tile_top4mxhbf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxbf8ps
-// CHECK: call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxbf8ps(void) {
- _tile_top4mxbf8ps(0, 1, 2, 5);
-}
-
-// CHECK-LABEL: @test_top4mxbssps
-// CHECK: call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
-void test_top4mxbssps(void) {
- _tile_top4mxbssps(0, 1, 2, 5);
-}
diff --git a/clang/test/CodeGen/X86/ace-inline-asm.c b/clang/test/CodeGen/X86/ace-inline-asm.c
deleted file mode 100644
index 27e36d84149fc..0000000000000
--- a/clang/test/CodeGen/X86/ace-inline-asm.c
+++ /dev/null
@@ -1,85 +0,0 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f \
-// RUN: -emit-llvm -o - -Wall -Werror | FileCheck %s
-
-// Test ACE instructions in inline assembly
-
-// BSR Management Instructions
-
-// CHECK-LABEL: @test_bsrinit_asm
-// CHECK: call void asm sideeffect "bsrinit", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrinit_asm(void) {
- __asm__ volatile("bsrinit");
-}
-
-// CHECK-LABEL: @test_bsrmovf_asm
-// CHECK: call void asm sideeffect "bsrmovf %zmm1, %zmm0", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrmovf_asm(void) {
- __asm__ volatile("bsrmovf %%zmm1, %%zmm0" :::);
-}
-
-// CHECK-LABEL: @test_bsrmovh_asm
-// CHECK: call void asm sideeffect "bsrmovh %zmm2", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrmovh_asm(void) {
- __asm__ volatile("bsrmovh %%zmm2" :::);
-}
-
-// CHECK-LABEL: @test_bsrmovl_asm
-// CHECK: call void asm sideeffect "bsrmovl %zmm3", "~{dirflag},~{fpsr},~{flags}"()
-void test_bsrmovl_asm(void) {
- __asm__ volatile("bsrmovl %%zmm3" :::);
-}
-
-// Tile Movement Instructions
-
-// CHECK-LABEL: @test_tilemovcol_asm
-// CHECK: call void asm sideeffect "tilemovcol $$$$5, %zmm0, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
-void test_tilemovcol_asm(void) {
- __asm__ volatile("tilemovcol $$5, %%zmm0, %%tmm1" ::: "tmm1");
-}
-
-// CHECK-LABEL: @test_tilemovrow_asm
-// CHECK: call void asm sideeffect "tilemovrow $$$$3, %zmm0, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
-void test_tilemovrow_asm(void) {
- __asm__ volatile("tilemovrow $$3, %%zmm0, %%tmm2" ::: "tmm2");
-}
-
-// Outer Product Instructions
-
-// CHECK-LABEL: @test_top4buud_asm
-// CHECK: call void asm sideeffect "top4buud %zmm1, %zmm0, %tmm0", "~{tmm0},~{dirflag},~{fpsr},~{flags}"()
-void test_top4buud_asm(void) {
- __asm__ volatile("top4buud %%zmm1, %%zmm0, %%tmm0" ::: "tmm0");
-}
-
-// CHECK-LABEL: @test_top4bssd_asm
-// CHECK: call void asm sideeffect "top4bssd %zmm2, %zmm1, %tmm1", "~{tmm1},~{dirflag},~{fpsr},~{flags}"()
-void test_top4bssd_asm(void) {
- __asm__ volatile("top4bssd %%zmm2, %%zmm1, %%tmm1" ::: "tmm1");
-}
-
-// CHECK-LABEL: @test_top2bf16ps_asm
-// CHECK: call void asm sideeffect "top2bf16ps %zmm3, %zmm2, %tmm2", "~{tmm2},~{dirflag},~{fpsr},~{flags}"()
-void test_top2bf16ps_asm(void) {
- __asm__ volatile("top2bf16ps %%zmm3, %%zmm2, %%tmm2" ::: "tmm2");
-}
-
-// Mixed Precision Outer Product Instructions
-
-// CHECK-LABEL: @test_top4mxhfbps_asm
-// CHECK: call void asm sideeffect "top4mxhfbps $$$$5, %zmm1, %zmm0, %tmm3", "~{tmm3},~{dirflag},~{fpsr},~{flags}"()
-void test_top4mxhfbps_asm(void) {
- __asm__ volatile("top4mxhfbps $$5, %%zmm1, %%zmm0, %%tmm3" ::: "tmm3");
-}
-
-// Full ACE operation sequence in inline assembly
-// CHECK-LABEL: @test_ace_sequence_asm
-// CHECK: call void asm sideeffect
-void test_ace_sequence_asm(void) {
- __asm__ volatile(
- "bsrinit \n\t"
- "top4buud %%zmm1, %%zmm0, %%tmm0 \n\t"
- "top4bssd %%zmm3, %%zmm2, %%tmm1 \n\t"
- ::: "memory", "tmm0", "tmm1"
- );
-}
diff --git a/clang/test/CodeGen/X86/acev1.c b/clang/test/CodeGen/X86/acev1.c
index 8f487b8ea2f51..50efae140ab56 100644
--- a/clang/test/CodeGen/X86/acev1.c
+++ b/clang/test/CodeGen/X86/acev1.c
@@ -1,22 +1,25 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
// Tests macro-based ACE v1 intrinsics emit correct LLVM IR
#include <immintrin.h>
-void test_acev1_outer_products(void) {
+// The outer products take the accumulator tile by ID, but their two sources are
+// ordinary ZMM values left to the register allocator.
+void test_acev1_outer_products(__m512bh bf1, __m512bh bf2, __m512i i1,
+ __m512i i2) {
// CHECK-LABEL: @test_acev1_outer_products
- // CHECK: call void @llvm.x86.top2bf16ps(i8 0,
- // CHECK: call void @llvm.x86.top4buud(i8 1,
- // CHECK: call void @llvm.x86.top4busd(i8 2,
- // CHECK: call void @llvm.x86.top4bssd(i8 3,
- // CHECK: call void @llvm.x86.top4bsud(i8 4,
- _tile_top2bf16ps(0, 1, 2);
- _tile_top4buud(1, 2, 3);
- _tile_top4busd(2, 3, 4);
- _tile_top4bssd(3, 4, 5);
- _tile_top4bsud(4, 5, 6);
+ // CHECK: call void @llvm.x86.top2bf16ps(i8 0, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
+ // CHECK: call void @llvm.x86.top4buud(i8 1, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.top4busd(i8 2, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.top4bssd(i8 3, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ // CHECK: call void @llvm.x86.top4bsud(i8 4, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
+ _tile_top2bf16ps(0, bf1, bf2);
+ _tile_top4buud(1, i1, i2);
+ _tile_top4busd(2, i1, i2);
+ _tile_top4bssd(3, i1, i2);
+ _tile_top4bsud(4, i1, i2);
}
void test_acev1_tile_config(void *data) {
@@ -31,6 +34,16 @@ void test_acev1_tile_config(void *data) {
_tile_ace_zero(0);
}
+// ACE v1 has no TILELOADD/TILESTORED, so the macro API moves data through
+// TILEMOVROW/TILEMOVCOL with an explicit tile register ID.
+void test_acev1_tile_movement(__m512i src) {
+ // CHECK-LABEL: @test_acev1_tile_movement
+ // CHECK: call void @llvm.x86.tilemovrow.set(i8 0, <16 x i32> %{{.*}}, i32 5)
+ // CHECK: call void @llvm.x86.tilemovcol.set(i8 1, <16 x i32> %{{.*}}, i32 3)
+ _tile_setrow(0, src, 5);
+ _tile_setcol(1, src, 3);
+}
+
void test_acev1_bsr(void) {
// CHECK-LABEL: @test_acev1_bsr
// CHECK: call void @llvm.x86.bsrinit()
diff --git a/clang/test/CodeGen/X86/acev1_api.c b/clang/test/CodeGen/X86/acev1_api.c
index acb12dae30cc4..55127fc23fa15 100644
--- a/clang/test/CodeGen/X86/acev1_api.c
+++ b/clang/test/CodeGen/X86/acev1_api.c
@@ -1,9 +1,9 @@
// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f -target-feature +avx512bf16 \
+// RUN: -target-feature +acev1 -target-feature +avx10.1 \
// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
-// Tests ACE v1 APIs using __acetile type (fixed 16x64 dimensions)
-// Note: ACE v1 does not have TILELOADD/TILESTORED - uses ACE-specific tile operations
+// Tests ACE v1 APIs using __acetile type (fixed 16x64 dimensions).
+// ACE v1 has no TILELOADD/TILESTORED, so it uses ACE-specific tile operations.
#include <immintrin.h>
@@ -20,7 +20,7 @@ void test_tile_ace_zero(void) {
void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4buud
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4buud.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4buud(&dst, src1, src2);
}
@@ -29,7 +29,7 @@ void test_tile_ace_top4buud(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4busd
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4busd.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4busd(&dst, src1, src2);
}
@@ -38,7 +38,7 @@ void test_tile_ace_top4busd(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4bssd
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4bssd(&dst, src1, src2);
}
@@ -47,7 +47,7 @@ void test_tile_ace_top4bssd(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
// CHECK-LABEL: @test_tile_ace_top4bsud
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top4bsud.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top4bsud(&dst, src1, src2);
}
@@ -56,7 +56,7 @@ void test_tile_ace_top4bsud(__acetile dst, __m512i src1, __m512i src2) {
void test_tile_ace_top2bf16ps(__acetile dst, __m512bh src1, __m512bh src2) {
// CHECK-LABEL: @test_tile_ace_top2bf16ps
// CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}})
- // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal
+ // CHECK-DAG: call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64,
// CHECK-DAG: call <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx {{%.*}})
__tile_ace_top2bf16ps(&dst, src1, src2);
}
@@ -109,14 +109,14 @@ void test_tile_ace_top4mxbssps(__acetile dst, __m512i src1, __m512i src2) {
// Test tile setrow (ACE-specific: ZMM to tile row)
void test_tile_ace_setrow(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__tile_ace_setrow(&dst, src, 0);
}
// Test tile setcol (ACE-specific: ZMM to tile column)
void test_tile_ace_setcol(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__tile_ace_setcol(&dst, src, 0);
}
@@ -127,6 +127,41 @@ __m512i test_tile_ace_getrow(__acetile src) {
return __tile_ace_getrow(&src, 0);
}
+// Test tile row read with INT32 to FP32 conversion (ACE-specific)
+__m512 test_tile_ace_cvtrowd2ps(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowd2ps
+ // CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowd2ps(&src, 2);
+}
+
+// Test tile row read with FP32 to BF16 conversion, high half of each dword
+__m512bh test_tile_ace_cvtrowps2bf16h(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2bf16h
+ // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2bf16h(&src, 4);
+}
+
+// Test tile row read with FP32 to BF16 conversion, low half of each dword
+__m512bh test_tile_ace_cvtrowps2bf16l(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2bf16l
+ // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2bf16l(&src, 6);
+}
+
+// Test tile row read with FP32 to FP16 conversion, high half of each dword
+__m512h test_tile_ace_cvtrowps2phh(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2phh
+ // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2phh(&src, 8);
+}
+
+// Test tile row read with FP32 to FP16 conversion, low half of each dword
+__m512h test_tile_ace_cvtrowps2phl(__acetile src) {
+ // CHECK-LABEL: @test_tile_ace_cvtrowps2phl
+ // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 16, i16 64, x86_amx {{%.*}}, i32 {{%.*}})
+ return __tile_ace_cvtrowps2phl(&src, 10);
+}
+
// Integration test: ACE computation workflow
void test_ace_workflow(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_ace_workflow
diff --git a/clang/test/CodeGen/X86/acev1_bsr.c b/clang/test/CodeGen/X86/acev1_bsr.c
index d68ecdcb59f7d..be7d1d8dfa416 100644
--- a/clang/test/CodeGen/X86/acev1_bsr.c
+++ b/clang/test/CodeGen/X86/acev1_bsr.c
@@ -1,7 +1,7 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
-// Tests ACE v1 BSR (Block Sparse Register) operations emit correct LLVM IR
+// Tests ACE v1 BSR (Block Scale Register) operations emit correct LLVM IR
#include <immintrin.h>
#include <stdint.h>
@@ -14,31 +14,31 @@ void test_bsr_init(void) {
void test_bsr_movf(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_bsr_movf
- // CHECK: call void @llvm.x86.bsrmovf(<16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+ // CHECK: call void @llvm.x86.bsrmovf(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
_bsr0_movf(src1, src2);
}
void test_bsr_movh_set(__m512i src) {
// CHECK-LABEL: @test_bsr_movh_set
- // CHECK: call void @llvm.x86.bsrmovh.set(<16 x i32> %{{.*}})
+ // CHECK: call void @llvm.x86.bsrmovh.set(<64 x i8> %{{.*}})
_bsr0_movh_set(src);
}
__m512i test_bsr_movh_get(void) {
// CHECK-LABEL: @test_bsr_movh_get
- // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovh.get()
return _bsr0_movh_get();
}
void test_bsr_movl_set(__m512i src) {
// CHECK-LABEL: @test_bsr_movl_set
- // CHECK: call void @llvm.x86.bsrmovl.set(<16 x i32> %{{.*}})
+ // CHECK: call void @llvm.x86.bsrmovl.set(<64 x i8> %{{.*}})
_bsr0_movl_set(src);
}
__m512i test_bsr_movl_get(void) {
// CHECK-LABEL: @test_bsr_movl_get
- // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovl.get()
return _bsr0_movl_get();
}
@@ -47,8 +47,8 @@ void test_bsr_sequence(void) {
// CHECK-LABEL: @test_bsr_sequence
// CHECK: call void @llvm.x86.bsrinit()
// CHECK: call void @llvm.x86.bsrmovf
- // CHECK: call <16 x i32> @llvm.x86.bsrmovh.get()
- // CHECK: call <16 x i32> @llvm.x86.bsrmovl.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovh.get()
+ // CHECK: call <64 x i8> @llvm.x86.bsrmovl.get()
_bsr0_init();
__m512i zmm1 = _mm512_set1_epi32(1);
diff --git a/clang/test/CodeGen/X86/acev1_errors.c b/clang/test/CodeGen/X86/acev1_errors.c
index c928882b62f4a..0776e3c026d3e 100644
--- a/clang/test/CodeGen/X86/acev1_errors.c
+++ b/clang/test/CodeGen/X86/acev1_errors.c
@@ -1,17 +1,17 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -fsyntax-only -verify
+// RUN: -target-feature +avx10.1 -fsyntax-only -verify
// Tests compile-time semantic errors for ACE v1 intrinsics
#include <immintrin.h>
-void test_tile_range_errors(void) {
+void test_tile_range_errors(__m512bh bf, __m512i i) {
// Tile index must be in range [0, 7]
_tile_ace_zero(8); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
_tile_ace_zero(16); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
// Outer product tile index errors
- _tile_top4buud(8, 0, 0); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
- _tile_top4bssd(16, 0, 0); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
- _tile_top2bf16ps(9, 0, 0); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
+ _tile_top4buud(8, i, i); // expected-error {{argument value 8 is outside the valid range [0, 7]}}
+ _tile_top4bssd(16, i, i); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ _tile_top2bf16ps(9, bf, bf); // expected-error {{argument value 9 is outside the valid range [0, 7]}}
}
diff --git a/clang/test/CodeGen/X86/acev1_inline_asm.c b/clang/test/CodeGen/X86/acev1_inline_asm.c
index 1dbf7d76a5355..57da62d269b75 100644
--- a/clang/test/CodeGen/X86/acev1_inline_asm.c
+++ b/clang/test/CodeGen/X86/acev1_inline_asm.c
@@ -1,20 +1,24 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s
// Tests inline assembly with ACE v1 tile registers
#include <immintrin.h>
+// TILESTORED is not available in palette 2, so the result leaves the tile one
+// row at a time through TILEMOVROW and is stored from the ZMM.
void test_acev1_inline_asm_outer_product(void) {
// CHECK-LABEL: @test_acev1_inline_asm_outer_product
// CHECK: call void asm sideeffect "tilezero %tmm0
// CHECK: top4buud %zmm0, %zmm1, %tmm0
- // CHECK: tilestored %tmm0
+ // CHECK: tilemovrow $$0, %tmm0, %zmm2
+ // CHECK: vmovups %zmm2, 0(%rdi)
__asm__ volatile (
- "tilezero %%tmm0 \n\t"
+ "tilezero %%tmm0 \n\t"
"top4buud %%zmm0, %%zmm1, %%tmm0 \n\t"
- "tilestored %%tmm0, 0(%%rdi) \n\t"
- ::: "memory", "tmm0", "zmm0", "zmm1"
+ "tilemovrow $0, %%tmm0, %%zmm2 \n\t"
+ "vmovups %%zmm2, 0(%%rdi) \n\t"
+ ::: "memory", "tmm0", "zmm0", "zmm1", "zmm2"
);
}
@@ -53,6 +57,59 @@ void test_acev1_inline_asm_tile_config(void *cfg) {
);
}
+// Tile movement is the only way in or out of an ACE tile, since ACE v1 has no
+// TILELOADD/TILESTORED. Immediate index form.
+void test_acev1_inline_asm_tile_movement(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_tile_movement
+ // CHECK: call void asm sideeffect "tilemovrow $$3, %zmm0, %tmm2
+ // CHECK: tilemovcol $$5, %zmm1, %tmm3
+ __asm__ volatile (
+ "tilemovrow $3, %%zmm0, %%tmm2 \n\t"
+ "tilemovcol $5, %%zmm1, %%tmm3 \n\t"
+ ::: "memory", "tmm2", "tmm3", "zmm0", "zmm1"
+ );
+}
+
+// Tile movement with the row/column index supplied in a GPR.
+void test_acev1_inline_asm_tile_movement_reg_index(int idx) {
+ // CHECK-LABEL: @test_acev1_inline_asm_tile_movement_reg_index
+ // CHECK: call void asm sideeffect "tilemovrow $0, %zmm0, %tmm0
+ // CHECK: tilemovcol $0, %zmm0, %tmm1
+ __asm__ volatile (
+ "tilemovrow %0, %%zmm0, %%tmm0 \n\t"
+ "tilemovcol %0, %%zmm0, %%tmm1 \n\t"
+ :: "r"(idx) : "memory", "tmm0", "tmm1", "zmm0"
+ );
+}
+
+// BSR high/low halves, both set (ZMM to BSR) and get (BSR to ZMM) directions.
+void test_acev1_inline_asm_bsr_movh_movl(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_bsr_movh_movl
+ // CHECK: call void asm sideeffect "bsrmovh %zmm1, %bsr0
+ // CHECK: bsrmovl %zmm2, %bsr0
+ // CHECK: bsrmovh %bsr0, %zmm3
+ // CHECK: bsrmovl %bsr0, %zmm4
+ __asm__ volatile (
+ "bsrmovh %%zmm1, %%bsr0 \n\t"
+ "bsrmovl %%zmm2, %%bsr0 \n\t"
+ "bsrmovh %%bsr0, %%zmm3 \n\t"
+ "bsrmovl %%bsr0, %%zmm4 \n\t"
+ ::: "memory", "zmm1", "zmm2", "zmm3", "zmm4"
+ );
+}
+
+// MX outer product, which takes the BSR scaling mode as an immediate.
+void test_acev1_inline_asm_mx_outer_product(void) {
+ // CHECK-LABEL: @test_acev1_inline_asm_mx_outer_product
+ // CHECK: call void asm sideeffect "tilezero %tmm1
+ // CHECK: top4mxhf8ps $$7, %zmm3, %zmm2, %tmm1
+ __asm__ volatile (
+ "tilezero %%tmm1 \n\t"
+ "top4mxhf8ps $7, %%zmm3, %%zmm2, %%tmm1 \n\t"
+ ::: "memory", "tmm1", "zmm2", "zmm3"
+ );
+}
+
void test_acev1_inline_asm_all_outer_products(void) {
// CHECK-LABEL: @test_acev1_inline_asm_all_outer_products
// CHECK: top4buud
diff --git a/clang/test/CodeGen/X86/acev1_mxfp8.c b/clang/test/CodeGen/X86/acev1_mxfp8.c
index e99ca9dda4067..dc0d04cdf76ab 100644
--- a/clang/test/CodeGen/X86/acev1_mxfp8.c
+++ b/clang/test/CodeGen/X86/acev1_mxfp8.c
@@ -1,51 +1,51 @@
// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +acev1 \
-// RUN: -target-feature +avx512f -emit-llvm -o - -Werror -pedantic | FileCheck %s
+// RUN: -target-feature +avx10.1 -emit-llvm -o - -Werror -pedantic | FileCheck %s
// Tests ACE v1 MX FP8 outer product macro-based intrinsics
#include <immintrin.h>
-void test_top4mxhf8ps(void) {
+void test_top4mxhf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxhf8ps
- // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 0)
- _tile_top4mxhf8ps(0, 1, 2, 0);
+ // CHECK: call void @llvm.x86.top4mxhf8ps(i8 0, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
+ _tile_top4mxhf8ps(0, src1, src2, 0);
}
-void test_top4mxbhf8ps(void) {
+void test_top4mxbhf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxbhf8ps
- // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, i8 2, i8 3, i8 1)
- _tile_top4mxbhf8ps(1, 2, 3, 1);
+ // CHECK: call void @llvm.x86.top4mxbhf8ps(i8 1, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 1)
+ _tile_top4mxbhf8ps(1, src1, src2, 1);
}
-void test_top4mxhbf8ps(void) {
+void test_top4mxhbf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxhbf8ps
- // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, i8 3, i8 4, i8 0)
- _tile_top4mxhbf8ps(2, 3, 4, 0);
+ // CHECK: call void @llvm.x86.top4mxhbf8ps(i8 2, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
+ _tile_top4mxhbf8ps(2, src1, src2, 0);
}
-void test_top4mxbf8ps(void) {
+void test_top4mxbf8ps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxbf8ps
- // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, i8 4, i8 5, i8 1)
- _tile_top4mxbf8ps(3, 4, 5, 1);
+ // CHECK: call void @llvm.x86.top4mxbf8ps(i8 3, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 1)
+ _tile_top4mxbf8ps(3, src1, src2, 1);
}
-void test_top4mxbssps(void) {
+void test_top4mxbssps(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_top4mxbssps
- // CHECK: call void @llvm.x86.top4mxbssps(i8 4, i8 5, i8 6, i8 0)
- _tile_top4mxbssps(4, 5, 6, 0);
+ // CHECK: call void @llvm.x86.top4mxbssps(i8 4, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, i8 0)
+ _tile_top4mxbssps(4, src1, src2, 0);
}
// Test all MX FP8 variants in sequence
-void test_all_mxfp8_variants(void) {
+void test_all_mxfp8_variants(__m512i src1, __m512i src2) {
// CHECK-LABEL: @test_all_mxfp8_variants
// CHECK: call void @llvm.x86.top4mxhf8ps
// CHECK: call void @llvm.x86.top4mxbhf8ps
// CHECK: call void @llvm.x86.top4mxhbf8ps
// CHECK: call void @llvm.x86.top4mxbf8ps
// CHECK: call void @llvm.x86.top4mxbssps
- _tile_top4mxhf8ps(0, 1, 2, 0);
- _tile_top4mxbhf8ps(3, 4, 5, 0);
- _tile_top4mxhbf8ps(6, 7, 0, 0);
- _tile_top4mxbf8ps(1, 2, 3, 1);
- _tile_top4mxbssps(4, 5, 6, 1);
+ _tile_top4mxhf8ps(0, src1, src2, 0);
+ _tile_top4mxbhf8ps(3, src1, src2, 0);
+ _tile_top4mxhbf8ps(6, src1, src2, 0);
+ _tile_top4mxbf8ps(1, src1, src2, 1);
+ _tile_top4mxbssps(4, src1, src2, 1);
}
diff --git a/clang/test/CodeGen/X86/acev1_tile_movement.c b/clang/test/CodeGen/X86/acev1_tile_movement.c
index f1383104f5db2..a2648b41b1d38 100644
--- a/clang/test/CodeGen/X86/acev1_tile_movement.c
+++ b/clang/test/CodeGen/X86/acev1_tile_movement.c
@@ -1,25 +1,26 @@
// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \
-// RUN: -target-feature +acev1 -target-feature +avx512f \
+// RUN: -target-feature +acev1 -target-feature +avx10.1 \
// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s
-// Tests ACE v1 tile movement operations (ZMM <-> Tile row/col)
-// Note: ACE v1 does not have TILELOADD/TILESTORED - uses TILEMOVROW/TILEMOVCOL for data movement
+// Tests ACE v1 tile movement operations (ZMM <-> Tile row/col).
+// ACE v1 has no TILELOADD/TILESTORED, so it moves data with
+// TILEMOVROW/TILEMOVCOL.
#include <immintrin.h>
// Test tile setrow - write ZMM to tile row (ACE-specific)
void test_tile_ace_setrow(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__tile_ace_setrow(&dst, src, 0);
}
// Test tile setrow with different row indices
void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setrow_indices
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__tile_ace_setrow(&dst, src, 0);
__tile_ace_setrow(&dst, src, 8);
__tile_ace_setrow(&dst, src, 15);
@@ -28,15 +29,15 @@ void test_tile_ace_setrow_indices(__acetile dst, __m512i src) {
// Test tile setcol - write ZMM to tile column (ACE-specific)
void test_tile_ace_setcol(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__tile_ace_setcol(&dst, src, 0);
}
// Test tile setcol with different column indices
void test_tile_ace_setcol_indices(__acetile dst, __m512i src) {
// CHECK-LABEL: @test_tile_ace_setcol_indices
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__tile_ace_setcol(&dst, src, 0);
__tile_ace_setcol(&dst, src, 8);
}
@@ -63,7 +64,7 @@ void test_tile_ace_getrow_indices(__acetile src, __m512i *out) {
void test_tile_init_via_setrow(void) {
// CHECK-LABEL: @test_tile_init_via_setrow
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
__acetile tile;
__tile_ace_zero(&tile);
@@ -75,9 +76,9 @@ void test_tile_init_via_setrow(void) {
void test_tile_movement_combined(void) {
// CHECK-LABEL: @test_tile_movement_combined
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
- // CHECK: call x86_amx @llvm.x86.tilesetcol.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovcol.set.internal
__acetile tile;
__tile_ace_zero(&tile);
@@ -96,7 +97,7 @@ void test_tile_movement_combined(void) {
void test_ace_data_workflow(__m512i *input, __m512i *output) {
// CHECK-LABEL: @test_ace_data_workflow
// CHECK: call x86_amx @llvm.x86.tilezero.internal
- // CHECK: call x86_amx @llvm.x86.tilesetrow.internal
+ // CHECK: call x86_amx @llvm.x86.tilemovrow.set.internal
// CHECK: call x86_amx @llvm.x86.top4buud.internal
// CHECK: call <16 x i32> @llvm.x86.tilemovrow.internal
__acetile tile;
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index fa917aaccbadd..6efad6ed2c843 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -501,7 +501,7 @@ __attribute__((target("fxsr")))
void f_fxsr(void) {}
// CHECK: [[f_general_regs_only]] = {{.*}}"target-cpu"="i686"
-// CHECK-SAME: "target-features"="+cmov,+cx8,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-mmx,-pclmul,-sha,-sha512,-sm3,-sm4,-sse,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-x87,-xop"
+// CHECK-SAME: "target-features"="+cmov,+cx8,-acev1,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-mmx,-pclmul,-sha,-sha512,-sm3,-sm4,-sse,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-x87,-xop"
__attribute__((target("general-regs-only")))
void f_general_regs_only(void) {}
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 3bc63f8031df6..52ec99b87516d 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -547,13 +547,33 @@
// NO-AMX-AVX512-NOT: #define __AMX_AVX512__ 1
+// ACE v1 requires AVX10.1 as its vector baseline and AMX-TILE for the tile
+// registers and tile management instructions, but nothing above either.
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 -x c \
-// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 %s
-// ACEV1: #define __ACEV1__ 1
+// RUN: -E -dM -o - %s | FileCheck -check-prefix=ACEV1 \
+// RUN: --implicit-check-not=__AVX10_2 --implicit-check-not=__AMX_AVX512__ %s
+// ACEV1-DAG: #define __ACEV1__ 1
+// ACEV1-DAG: #define __AVX10_1__ 1
+// ACEV1-DAG: #define __AVX512F__ 1
+// ACEV1-DAG: #define __AMX_TILE__ 1
// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mno-acev1 -x c \
// RUN: -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
// NO-ACEV1-NOT: #define __ACEV1__ 1
+// AVX10.2 is not required: disabling it keeps ACE v1 and its baseline.
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-avx10.2 -x c -E -dM -o - %s | FileCheck \
+// RUN: -check-prefix=ACEV1-BASELINE %s
+// ACEV1-BASELINE-DAG: #define __ACEV1__ 1
+// ACEV1-BASELINE-DAG: #define __AVX10_1__ 1
+// ACEV1-BASELINE-DAG: #define __AMX_TILE__ 1
+
+// Both baselines are necessary: dropping either drops ACE v1 with it.
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-avx10.1 -x c -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -macev1 \
+// RUN: -mno-amx-tile -x c -E -dM -o - %s | FileCheck -check-prefix=NO-ACEV1 %s
+
// RUN: %clang -target i386-unknown-unknown -march=atom -mavxvnni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVXVNNI %s
// AVXVNNI: #define __AVX2__ 1
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 1f3a8160e606e..48fd9c77b0b09 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5687,110 +5687,105 @@ let TargetPrefix = "x86" in {
Intrinsic<[], [], []>;
// BSRMOVF - Move Full to BSR
- // Takes two ZMM registers as input (doubleword integers), writes to BSR (implicit)
+ // Takes two ZMM registers of E8M0 scale bytes, writes to BSR (implicit)
def int_x86_bsrmovf : ClangBuiltin<"__builtin_ia32_bsrmovf">,
- Intrinsic<[], [llvm_v16i32_ty, llvm_v16i32_ty], []>;
+ Intrinsic<[], [llvm_v64i8_ty, llvm_v64i8_ty], []>;
// BSRMOVH - Move Half to/from BSR
- // Load variant: read from ZMM (doubleword integers), update BSR
+ // Load variant: read 64 scale bytes from ZMM, update BSR
def int_x86_bsrmovh_set : ClangBuiltin<"__builtin_ia32_bsrmovh_set">,
- Intrinsic<[], [llvm_v16i32_ty], []>;
- // Store variant: read from BSR, return ZMM (doubleword integers)
+ Intrinsic<[], [llvm_v64i8_ty], []>;
+ // Store variant: read from BSR, return 64 scale bytes in a ZMM
def int_x86_bsrmovh_get : ClangBuiltin<"__builtin_ia32_bsrmovh_get">,
- Intrinsic<[llvm_v16i32_ty], [], []>;
+ Intrinsic<[llvm_v64i8_ty], [], []>;
// BSRMOVL - Move Low to/from BSR
- // Load variant: read from ZMM (doubleword integers), write to BSR
+ // Load variant: read 64 scale bytes from ZMM, write to BSR
def int_x86_bsrmovl_set : ClangBuiltin<"__builtin_ia32_bsrmovl_set">,
- Intrinsic<[], [llvm_v16i32_ty], []>;
- // Store variant: read from BSR, return ZMM (doubleword integers)
+ Intrinsic<[], [llvm_v64i8_ty], []>;
+ // Store variant: read from BSR, return 64 scale bytes in a ZMM
def int_x86_bsrmovl_get : ClangBuiltin<"__builtin_ia32_bsrmovl_get">,
- Intrinsic<[llvm_v16i32_ty], [], []>;
+ Intrinsic<[llvm_v64i8_ty], [], []>;
// TILEMOVCOL - Move Column from Vector to Tile
// Single intrinsic handles both immediate and register index forms.
// Lowering checks if index is constant and selects appropriate instruction.
- def int_x86_tilesetcol : ClangBuiltin<"__builtin_ia32_tilesetcol">,
+ def int_x86_tilemovcol_set : ClangBuiltin<"__builtin_ia32_tilemovcol_set">,
Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
- // ACE TILEMOVROW - Move Row from Vector TO Tile (extends AMX TILEMOVROW)
- // Note: AMX has int_x86_tilemovrow which reads FROM tile to vector
- // ACE adds the reverse direction: writing TO tile from vector
+ // ACE TILEMOVROW - Move Row from Vector to Tile (extends AMX TILEMOVROW)
+ // AMX has int_x86_tilemovrow, which reads from the tile into a vector; ACE
+ // adds the reverse direction, writing to the tile from a vector.
// Single intrinsic handles both immediate and register index forms.
// Lowering checks if index is constant and selects appropriate instruction.
- def int_x86_tilesetrow : ClangBuiltin<"__builtin_ia32_tilesetrow">,
+ def int_x86_tilemovrow_set : ClangBuiltin<"__builtin_ia32_tilemovrow_set">,
Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
// TOP2BF16PS - Outer Product BF16 to Single Precision
def int_x86_top2bf16ps : ClangBuiltin<"__builtin_ia32_top2bf16ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v32bf16_ty, llvm_v32bf16_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4 - Tile Outer Product (4x4 block operations)
// TOP4BUUD - Unsigned/Unsigned
def int_x86_top4buud : ClangBuiltin<"__builtin_ia32_top4buud">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4BUSD - Unsigned/Signed
def int_x86_top4busd : ClangBuiltin<"__builtin_ia32_top4busd">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4BSSD - Signed/Signed
def int_x86_top4bssd : ClangBuiltin<"__builtin_ia32_top4bssd">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4BSUD - Signed/Unsigned
def int_x86_top4bsud : ClangBuiltin<"__builtin_ia32_top4bsud">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v64i8_ty, llvm_v64i8_ty],
+ [ImmArg<ArgIndex<0>>]>;
// TOP4MX - Mixed Precision Outer Products
- // TOP4MXHFBPS - FP16/BF16 to FP32
+ // TOP4MXHF8PS - MX FP8 HF8 (E4M3) to FP32
def int_x86_top4mxhf8ps : ClangBuiltin<"__builtin_ia32_top4mxhf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
- // TOP4MXBHFBPS - BF16/FP16 to FP32
+ // TOP4MXBHF8PS - MX FP8 BF8xHF8 to FP32
def int_x86_top4mxbhf8ps : ClangBuiltin<"__builtin_ia32_top4mxbhf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
- // TOP4MXHBFBPS - FP16/BF16/BF16 to FP32
+ // TOP4MXHBF8PS - MX FP8 HF8xBF8 to FP32
def int_x86_top4mxhbf8ps : ClangBuiltin<"__builtin_ia32_top4mxhbf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
// TOP4MXBF8PS - MX FP8 BF8xBF8 to FP32
def int_x86_top4mxbf8ps : ClangBuiltin<"__builtin_ia32_top4mxbf8ps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
// TOP4MXBSSPS - MX INT8 SxS to FP32 (BSR scaled)
def int_x86_top4mxbssps : ClangBuiltin<"__builtin_ia32_top4mxbssps">,
- Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty, llvm_i8_ty],
- [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>,
- ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>]>;
+ Intrinsic<[], [llvm_i8_ty, llvm_v16i32_ty, llvm_v16i32_ty,
+ llvm_i8_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
// ACE - internal intrinsics (for IR-level optimization)
// These take tile/ZMM values directly instead of immediate tile IDs
// TILEMOVCOL internal - takes dimensions and returns tile
- def int_x86_tilesetcol_internal :
- ClangBuiltin<"__builtin_ia32_tilesetcol_internal">,
+ def int_x86_tilemovcol_set_internal :
+ ClangBuiltin<"__builtin_ia32_tilemovcol_set_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
[]>;
@@ -5798,8 +5793,8 @@ let TargetPrefix = "x86" in {
// ACE TILEMOVROW internal - takes dimensions, vector, index and returns tile
// Note: AMX has int_x86_tilemovrow_internal which extracts from tile to vector
// ACE adds the reverse: write vector to tile row
- def int_x86_tilesetrow_internal :
- ClangBuiltin<"__builtin_ia32_tilesetrow_internal">,
+ def int_x86_tilemovrow_set_internal :
+ ClangBuiltin<"__builtin_ia32_tilemovrow_set_internal">,
Intrinsic<[llvm_x86amx_ty],
[llvm_i16_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_i32_ty],
[]>;
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index f8fbe99529494..819e7e9aa0af7 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -4013,7 +4013,7 @@ bool X86AsmParser::parseInstruction(ParseInstructionInfo &Info, StringRef Name,
// For BSRMOVH and BSRMOVL, detect set vs get by operand order
if (Name == "bsrmovh" || Name == "bsrmovl") {
- // Intel: BSR first = set (ZMM→BSR); ATT: non-BSR first = set
+ // Intel: BSR first = set (ZMM->BSR); ATT: non-BSR first = set
bool IsLoad = isParsingIntelSyntax() ? (FirstBsrPos < FirstNonBsrPos)
: (FirstNonBsrPos < FirstBsrPos);
StringRef NewMnemonic =
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index d4e24f52bb895..f69de37d85fa6 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -1231,7 +1231,9 @@ X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
// Instruction format for 4VOp3:
// src1(ModR/M), src2(ModR/M), src3(VEX_4V)
Prefix.setRR2(MI, CurOp++);
- Prefix.setBB2(MI, CurOp++);
+ Prefix.setBB2(MI, CurOp);
+ Prefix.setX(MI, CurOp, 4);
+ ++CurOp;
Prefix.set4VV2(MI, CurOp++);
break;
}
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 2b21219a29f02..dbcba08a8cd86 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -296,9 +296,6 @@ def FeatureAMXAVX512 : SubtargetFeature<"amx-avx512",
"HasAMXAVX512", "true",
"Support AMX-AVX512 instructions",
[FeatureAMXTILE]>;
-def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
- "Support ACEV1 (AI Compute Extensions v1) instructions",
- [FeatureAMXAVX512]>;
def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true",
"Support CMPCCXADD instructions">;
def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true",
@@ -363,6 +360,16 @@ def FeatureAVX10_2 : SubtargetFeature<"avx10.2", "HasAVX10_2", "true",
def FeatureAVX10_2_512 : SubtargetFeature<"avx10.2-512", "HasAVX10_2_512", "true",
"Support AVX10.2 instruction",
[FeatureAVX10_2]>;
+// Defined after AVX10.1 since ACE v1 builds on it as its vector baseline.
+// ACE v1 reuses the AMX tile architecture: per spec section 15.5.5 an
+// implementation that supports only ACE still reports AMX-TILE, which is what
+// enumerates the tile registers and LDTILECFG/STTILECFG/TILEZERO/TILERELEASE.
+// The AMX compute features and palette 1 are not required, so AMX-AVX512 is not
+// implied; TILEMOVROW and the TCVTROW* family are enumerated under
+// "AMX-AVX512 || ACE_VSN >= 1".
+def FeatureACEV1 : SubtargetFeature<"acev1", "HasACEV1", "true",
+ "Support ACEV1 (AI Compute Extensions v1) instructions",
+ [FeatureAMXTILE, FeatureAVX10_1]>;
def FeatureEGPR : SubtargetFeature<"egpr", "HasEGPR", "true",
"Support extended general purpose register">;
def FeaturePush2Pop2 : SubtargetFeature<"push2pop2", "HasPush2Pop2", "true",
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 72865e71689be..9cebef57ca151 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -226,8 +226,12 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
- // ACE v1 doesn't have TILESTORED - use TILEMOVROW row-by-row
- if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ // ACE configures palette 2, which has no TILESTORED - use TILEMOVROW
+ // row-by-row through a scratch ZMM.
+ // FIXME: This defines the tile and the scratch register once per row, which
+ // is not valid SSA. X86VolatileTileData also still emits the AMX load/store
+ // intrinsics at -O0, so the ACE -O0 path needs both fixed together.
+ if (ST->hasACEV1()) {
const DebugLoc &DL = Before->getDebugLoc();
Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
@@ -235,14 +239,14 @@ void X86FastPreTileConfigImpl::spill(MachineBasicBlock::iterator Before,
const unsigned RowSize = 64;
for (unsigned Row = 0; Row < NumRows; ++Row) {
- // TILEMOVROW zmm, tmm, imm8 (read from tile)
- // Only kill src on the last row read
+ // tilemovrow $row, %tmm, %zmm
+ // Only kill src on the last row read.
bool KillSrcNow = (Row == NumRows - 1) && Kill;
BuildMI(*MBB, Before, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
.addReg(VirtReg, getKillRegState(KillSrcNow))
.addImm(Row);
- // VMOVUPS [stack + row*64], zmm
+ // vmovups %zmm, row*64(%sp)
MachineInstrBuilder MIB =
BuildMI(*MBB, Before, DL, TII->get(X86::VMOVUPSZmr));
addFrameReference(MIB, FI, Row * RowSize);
@@ -280,8 +284,10 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
else
TileReg = MRI->createVirtualRegister(&RC);
- // ACE v1 doesn't have TILELOADD - use TILEMOVROW row-by-row
- if (ST->hasACEV1() && !ST->hasAMXTILE()) {
+ // ACE configures palette 2, which has no TILELOADD - use TILEMOVROW
+ // row-by-row through a scratch ZMM.
+ // FIXME: See the corresponding comment in spill().
+ if (ST->hasACEV1()) {
const DebugLoc &DL = UseMI->getDebugLoc();
Register ScratchZMM = MRI->createVirtualRegister(&X86::VR512RegClass);
@@ -289,12 +295,12 @@ void X86FastPreTileConfigImpl::reload(MachineBasicBlock::iterator UseMI,
const unsigned RowSize = 64;
for (unsigned Row = 0; Row < NumRows; ++Row) {
- // VMOVUPS zmm, [stack + row*64]
+ // vmovups row*64(%sp), %zmm
MachineInstrBuilder MIB = BuildMI(*UseMI->getParent(), UseMI, DL,
TII->get(X86::VMOVUPSZrm), ScratchZMM);
addFrameReference(MIB, FI, Row * RowSize);
- // TILEMOVROW tmm, zmm, imm8 (write to tile)
+ // tilemovrow $row, %zmm, %tmm
BuildMI(*UseMI->getParent(), UseMI, DL, TII->get(X86::TILEMOVROWri),
TileReg)
.addReg(ScratchZMM, RegState::Kill)
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index bf252d934fbdc..f94416a372c43 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -5690,7 +5690,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
case Intrinsic::x86_tcvtrowps2phh_internal:
case Intrinsic::x86_tcvtrowps2phl_internal:
case Intrinsic::x86_tilemovrow_internal: {
- if (!Subtarget->hasAMXAVX512())
+ if (!Subtarget->hasAMXAVX512() && !Subtarget->hasACEV1())
break;
auto *MFI =
CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5b90a771a6569..33afabe14af94 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28660,8 +28660,8 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
// ACE Tile Movement Intrinsics - handle both immediate and register index
// forms using a single intrinsic. Check if index is constant to select
// the appropriate instruction form.
- case Intrinsic::x86_tilesetrow:
- case Intrinsic::x86_tilesetcol: {
+ case Intrinsic::x86_tilemovrow_set:
+ case Intrinsic::x86_tilemovcol_set: {
SDLoc DL(Op);
SDValue Chain = Op.getOperand(0);
unsigned TileID = Op.getConstantOperandVal(2);
@@ -28669,7 +28669,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
SDValue Idx = Op.getOperand(4);
unsigned PseudoImm, PseudoReg;
- if (IntNo == Intrinsic::x86_tilesetrow) {
+ if (IntNo == Intrinsic::x86_tilemovrow_set) {
PseudoImm = X86::PTILEMOVROW;
PseudoReg = X86::PTILEMOVROW_REG;
} else {
@@ -39516,17 +39516,12 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
break;
}
// These instructions: TILE dst (rw), ZMM src1, ZMM src2
- // The pseudo has: u8imm dst, u8imm src1, u8imm src2
- // src1 and src2 are ZMM register indices (0-31)
+ // The pseudo has: u8imm dst, VR512 src1, VR512 src2
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
- unsigned Src1Imm = MI.getOperand(1).getImm();
- unsigned Src2Imm = MI.getOperand(2).getImm();
- assert(Src1Imm < 32 && "Illegal ZMM register index");
- assert(Src2Imm < 32 && "Illegal ZMM register index");
BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
.addReg(DstReg, RegState::Undef)
- .addReg(X86::ZMM0 + Src1Imm)
- .addReg(X86::ZMM0 + Src2Imm);
+ .addReg(MI.getOperand(1).getReg())
+ .addReg(MI.getOperand(2).getReg());
MI.eraseFromParent();
return BB;
}
@@ -39557,15 +39552,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
}
// These instructions: TILE dst (rw), ZMM src1, ZMM src2, imm8
unsigned DstReg = TMMImmToTMMReg(MI.getOperand(0).getImm());
- unsigned Src1Imm = MI.getOperand(1).getImm();
- unsigned Src2Imm = MI.getOperand(2).getImm();
unsigned Imm = MI.getOperand(3).getImm();
- assert(Src1Imm < 32 && "Illegal ZMM register index");
- assert(Src2Imm < 32 && "Illegal ZMM register index");
BuildMI(*BB, MI, MIMD, TII->get(Opc), DstReg)
.addReg(DstReg, RegState::Undef)
- .addReg(X86::ZMM0 + Src1Imm)
- .addReg(X86::ZMM0 + Src2Imm)
+ .addReg(MI.getOperand(1).getReg())
+ .addReg(MI.getOperand(2).getReg())
.addImm(Imm);
MI.eraseFromParent();
return BB;
diff --git a/llvm/lib/Target/X86/X86InstrACE.td b/llvm/lib/Target/X86/X86InstrACE.td
index f786f8ca34583..26f531a19ff0b 100644
--- a/llvm/lib/Target/X86/X86InstrACE.td
+++ b/llvm/lib/Target/X86/X86InstrACE.td
@@ -19,7 +19,8 @@ let Predicates = [HasACEV1, In64BitMode] in {
// BSRINIT - Initialize Block Scale Register
// VEX.128.F2.0F38.W1 49 11:000:000 - BSRINIT bsr0
// BSR is implicit destination, ModRM = 11:000:000 (mod=11, reg=000, r/m=000)
-// Note: BSR0 is implicit (via Defs), not in operand list. Asm syntax has no explicit operand.
+// BSR0 is implicit (via Defs) rather than an operand, so the asm syntax takes
+// no explicit operand.
let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
def BSRINIT : I<0x49, MRM_C0, (outs), (ins),
"bsrinit", []>,
@@ -28,13 +29,14 @@ let SchedRW = [WriteSystem], hasSideEffects = 1, Defs = [BSR0] in {
// BSRMOVF - Move Full to BSR
// EVEX.512.NP.MAP6.W1 95 11:000:bbb - BSRMOVF bsr0, zmm1, zmm2/m512
-// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1, ModRM.r/m encodes zmm2
+// ModRM.reg = 000 (opcode extension), EVEX.vvvv encodes zmm1 and ModRM.r/m
+// encodes zmm2
// BSR is implicit destination (via Defs)
let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
def BSRMOVFrr : I<0x95, MRM0r, (outs),
(ins VR512:$src1, VR512:$src2),
"bsrmovf\t{$src2, $src1|$src1, $src2}",
- [(int_x86_bsrmovf (v16i32 VR512:$src1), (v16i32 VR512:$src2))]>,
+ [(int_x86_bsrmovf (v64i8 VR512:$src1), (v64i8 VR512:$src2))]>,
EVEX, T_MAP6, REX_W, EVEX_V512, VVVV;
let mayLoad = 1 in
@@ -49,14 +51,15 @@ let SchedRW = [WriteVecLogic], Defs = [BSR0] in {
// EVEX.512.F2.MAP6.W0 95 /r - BSRMOVH zmm1/m512, bsr0 (store, W0)
// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
-// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+// AsmString uses a "_set"/"_get" suffix for asm matching; the printer emits the
+// real mnemonic.
let SchedRW = [WriteVecLogic] in {
// Set BSR high half (W1): BSR is both input and output (read-modify-write)
let Defs = [BSR0], Uses = [BSR0] in {
def BSRMOVHrr_set : I<0x95, MRM0r, (outs),
(ins VR512:$src),
"bsrmovh_set\t$src",
- [(int_x86_bsrmovh_set (v16i32 VR512:$src))]>,
+ [(int_x86_bsrmovh_set (v64i8 VR512:$src))]>,
EVEX, XD, T_MAP6, REX_W, EVEX_V512;
let mayLoad = 1 in
@@ -66,18 +69,20 @@ let SchedRW = [WriteVecLogic] in {
EVEX, XD, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
- // Get BSR high half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ // Get BSR high half (W0): BSR is the implicit source, ZMM/mem the explicit
+ // destination
let Uses = [BSR0] in {
def BSRMOVHrr_get : I<0x95, MRM0r, (outs VR512:$dst),
(ins),
"bsrmovh_get\t$dst",
- [(set (v16i32 VR512:$dst), (int_x86_bsrmovh_get))]>,
+ [(set (v64i8 VR512:$dst), (int_x86_bsrmovh_get))]>,
EVEX, XD, T_MAP6, EVEX_V512;
let mayStore = 1 in
def BSRMOVHmr_get : I<0x95, MRM0m, (outs),
(ins f512mem:$dst),
- "bsrmovh_get\t$dst", []>,
+ "bsrmovh_get\t$dst",
+ [(store (v64i8 (int_x86_bsrmovh_get)), addr:$dst)]>,
EVEX, XD, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
}
@@ -87,14 +92,15 @@ let SchedRW = [WriteVecLogic] in {
// EVEX.512.F3.MAP6.W0 95 /r - BSRMOVL zmm1/m512, bsr0 (store, W0)
// BSR is implicit operand (via Defs/Uses), ModRM.reg field = 0
// Use MRM0r/MRM0m: ModRM.reg = 0 (BSR implicit), ModRM.r/m = source/destination
-// AsmString uses "_set"/"_get" suffix for asm matching; printer uses custom print.
+// AsmString uses a "_set"/"_get" suffix for asm matching; the printer emits the
+// real mnemonic.
let SchedRW = [WriteVecLogic] in {
// Set BSR low half (W1): BSR is both input and output (read-modify-write)
let Defs = [BSR0], Uses = [BSR0] in {
def BSRMOVLrr_set : I<0x95, MRM0r, (outs),
(ins VR512:$src),
"bsrmovl_set\t$src",
- [(int_x86_bsrmovl_set (v16i32 VR512:$src))]>,
+ [(int_x86_bsrmovl_set (v64i8 VR512:$src))]>,
EVEX, XS, T_MAP6, REX_W, EVEX_V512;
let mayLoad = 1 in
@@ -104,18 +110,20 @@ let SchedRW = [WriteVecLogic] in {
EVEX, XS, T_MAP6, REX_W, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
- // Get BSR low half (W0): BSR is implicit source, ZMM/mem is explicit destination
+ // Get BSR low half (W0): BSR is the implicit source, ZMM/mem the explicit
+ // destination
let Uses = [BSR0] in {
def BSRMOVLrr_get : I<0x95, MRM0r, (outs VR512:$dst),
(ins),
"bsrmovl_get\t$dst",
- [(set (v16i32 VR512:$dst), (int_x86_bsrmovl_get))]>,
+ [(set (v64i8 VR512:$dst), (int_x86_bsrmovl_get))]>,
EVEX, XS, T_MAP6, EVEX_V512;
let mayStore = 1 in
def BSRMOVLmr_get : I<0x95, MRM0m, (outs),
(ins f512mem:$dst),
- "bsrmovl_get\t$dst", []>,
+ "bsrmovl_get\t$dst",
+ [(store (v64i8 (int_x86_bsrmovl_get)), addr:$dst)]>,
EVEX, XS, T_MAP6, EVEX_V512, EVEX_CD8<64, CD8VF>;
}
}
@@ -159,7 +167,8 @@ let SchedRW = [WriteVecLogic] in {
// Operand 1: ModRM:reg (rw) = tmm1
// Operand 2: ModRM:r/m (r) = zmm2
// Operand 3: EVEX.vvvv (r) = zmm3
-// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+// Use MRMSrcReg4VOp3: operand 1 goes to r/m and operand 2 to vvvv, so src2
+// and src3 are listed in the opposite order from the spec.
let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
def TOP2BF16PSrrr : I<0x5C, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
@@ -176,7 +185,8 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
// Operand 1: ModRM:reg (rw) = tmm1
// Operand 2: ModRM:r/m (r) = zmm2
// Operand 3: EVEX.vvvv (r) = zmm3
-// Use MRMSrcReg4VOp3: operand 1 → r/m, operand 2 → vvvv (so swap src2/src3 order)
+// Use MRMSrcReg4VOp3: operand 1 goes to r/m and operand 2 to vvvv, so src2
+// and src3 are listed in the opposite order from the spec.
let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
def TOP4BUUDrrr : I<0x5E, MRMSrcReg4VOp3, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src2, VR512:$src3),
@@ -211,8 +221,9 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
// Operand 2: ModRM:r/m (r) = zmm2
// Operand 3: EVEX.vvvv (r) = zmm3
// Operand 4: imm8
-// MRMSrcReg encodes: src1→reg, src2→r/m, src3→vvvv (so swap to get spec order)
-let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
+// MRMSrcReg encodes src1 in reg, src2 in r/m and src3 in vvvv, so the operands
+// are listed swapped to match the spec order.
+let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic], Uses = [BSR0] in {
def TOP4MXBF8PSrrri : Ii8<0x8D, MRMSrcReg, (outs TILE:$dst),
(ins TILE:$src1, VR512:$src3, VR512:$src2, u8imm:$imm),
"top4mxbf8ps\t{$imm, $src3, $src2, $dst|$dst, $src2, $src3, $imm}", []>,
@@ -244,8 +255,8 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteVecLogic] in {
// These are used by intrinsics and expanded via custom inserter
//===----------------------------------------------------------------------===//
-// Pseudo instructions for user-facing intrinsics (immediate tile IDs)
-// These use usesCustomInserter to expand to real instructions in X86ISelLowering
+// Pseudo instructions for user-facing intrinsics (immediate tile IDs).
+// usesCustomInserter expands these to real instructions in X86ISelLowering.
let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
// BSRINIT pseudo - no operands, just triggers BSR initialization
def PBSRINIT : PseudoI<(outs), (ins),
@@ -256,7 +267,7 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
// TILEMOVCOL pseudo - immediate index variant
// dst: tile ID (imm), src: ZMM value (VR512), idx: column index (imm)
- // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ // Custom lowering picks the imm or reg variant, so there is no pattern here.
def PTILEMOVCOL : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
// TILEMOVCOL pseudo - register index variant
@@ -264,63 +275,63 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1 in {
// TILEMOVROW pseudo - immediate index variant (ACE: write to tile)
// dst: tile ID (imm), src: ZMM value (VR512), idx: row index (imm)
- // Pattern matching removed - handled by custom lowering in X86ISelLowering.cpp
+ // Custom lowering picks the imm or reg variant, so there is no pattern here.
def PTILEMOVROW : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, u8imm:$idx), []>;
// TILEMOVROW pseudo - register index variant (ACE: write to tile)
def PTILEMOVROW_REG : PseudoI<(outs), (ins u8imm:$dst, VR512:$src, GR32:$idx), []>;
- // TOP2BF16PS pseudo
- def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top2bf16ps timm:$dst, timm:$src1, timm:$src2)]>;
+ // TOP2BF16PS pseudo - tile dst by ID, ZMM sources by value
+ def PTOP2BF16PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top2bf16ps timm:$dst, VR512:$src1, VR512:$src2)]>;
- // TOP4 pseudos - three tile operands (dst is also accumulator)
- def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4buud timm:$dst, timm:$src1, timm:$src2)]>;
+ // TOP4 pseudos - tile dst by ID (also the accumulator), ZMM sources by value
+ def PTOP4BUUD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4buud timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4busd timm:$dst, timm:$src1, timm:$src2)]>;
+ def PTOP4BUSD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4busd timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4bssd timm:$dst, timm:$src1, timm:$src2)]>;
+ def PTOP4BSSD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4bssd timm:$dst, VR512:$src1, VR512:$src2)]>;
- def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2),
- [(int_x86_top4bsud timm:$dst, timm:$src1, timm:$src2)]>;
+ def PTOP4BSUD : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2),
+ [(int_x86_top4bsud timm:$dst, VR512:$src1, VR512:$src2)]>;
- // TOP4MX pseudos - four operands (dst, src1, src2, imm)
- def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ // TOP4MX pseudos - tile dst by ID, ZMM sources by value, plus imm8
+ def PTOP4MXHF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxbhf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXBHF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxbhf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxhbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXHBF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxhbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxbf8ps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXBF8PS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxbf8ps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
- def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, u8imm:$src1, u8imm:$src2, u8imm:$imm),
- [(int_x86_top4mxbssps timm:$dst, timm:$src1, timm:$src2, timm:$imm)]>;
+ def PTOP4MXBSSPS : PseudoI<(outs), (ins u8imm:$dst, VR512:$src1, VR512:$src2, u8imm:$imm),
+ [(int_x86_top4mxbssps timm:$dst, VR512:$src1, VR512:$src2, timm:$imm)]>;
}
-// Pseudo instructions for internal intrinsics (register allocation)
-// These are used for IR-level optimization and register allocation
-// usesCustomInserter is needed to set ACEProgModel in EmitInstrWithCustomInserter
+// Pseudo instructions for internal intrinsics (register allocation).
+// These are used for IR-level optimization and register allocation.
+// usesCustomInserter is needed to set ACEProgModel in
+// EmitInstrWithCustomInserter.
let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
// TILEMOVCOL internal - output is TILE, input is ZMM + index
def PTILEMOVCOLV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
[(set TILE:$dst,
- (int_x86_tilesetcol_internal GR16:$row, GR16:$col,
+ (int_x86_tilemovcol_set_internal GR16:$row, GR16:$col,
VR512:$src, GR32:$idx))]>;
// TILEMOVROW internal - output is TILE, input is ZMM + index
- // ACE TILEMOVROW internal - write vector to tile row
def PTILEMOVROWV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, VR512:$src, GR32:$idx),
[(set TILE:$dst,
- (int_x86_tilesetrow_internal GR16:$row, GR16:$col,
+ (int_x86_tilemovrow_set_internal GR16:$row, GR16:$col,
VR512:$src, GR32:$idx))]>;
// TOP2BF16PS internal - output TILE, inputs: dimensions, TILE acc, two ZMMs
@@ -369,7 +380,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXHFBPS internal - imm is BSR index (u8imm matches timm in pattern)
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXHF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -378,7 +389,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXBHFBPS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXBHF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -387,7 +398,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXHBFBPS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXHBF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -396,7 +407,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXBF8PS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXBF8PSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -405,7 +416,7 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
// TOP4MXBSSPS internal
- let Constraints = "$src1 = $dst" in
+ let Constraints = "$src1 = $dst", Uses = [BSR0] in
def PTOP4MXBSSPSV : PseudoI<(outs TILE:$dst),
(ins GR16:$row, GR16:$col, GR16:$k, u8imm:$imm,
TILE:$src1, VR512:$src2, VR512:$src3),
@@ -414,4 +425,16 @@ let isPseudo = true, hasNoSchedulingInfo = 1, usesCustomInserter = 1 in {
TILE:$src1, VR512:$src2, VR512:$src3))]>;
}
+// Pseudo instructions for tile spill and reload (register allocation)
+// ACE has no TILELOADD/TILESTORED, so the register allocator emits these and
+// X86LowerTileCopy expands them into a row-by-row TILEMOVROW sequence.
+// Operand order matches storeRegToStackSlot/loadRegFromStackSlot.
+let isPseudo = true, hasNoSchedulingInfo = 1 in {
+ let mayStore = 1 in
+ def ACE_TILESPILL : PseudoI<(outs), (ins opaquemem:$dst, TILE:$src), []>;
+
+ let mayLoad = 1 in
+ def ACE_TILERELOAD : PseudoI<(outs TILE:$dst), (ins opaquemem:$src), []>;
+}
+
} // end HasACEV1, In64BitMode
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 3b37ff8cbfd3f..5d3af944465c7 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -4565,11 +4565,12 @@ static unsigned getLoadStoreRegOpcode(Register Reg,
assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
assert((STI.hasAMXTILE() || STI.hasACEV1()) &&
"Using 8*1024-bit register requires AMX-TILE or ACE");
- // ACE v1 doesn't have TILELOADD/TILESTORED - tile spilling should be
- // handled by X86LowerTileCopy using TILEMOVROW row-by-row.
- // This path should only be hit for AMX targets.
- assert(STI.hasAMXTILE() &&
- "ACE tile spill should use TILEMOVROW, not TILELOADD/TILESTORED");
+ // ACE configures palette 2, which has no TILELOADD/TILESTORED. Spill and
+ // reload through pseudos that X86LowerTileCopy expands into TILEMOVROW.
+ // This is the only place these pseudos are created, so it is what keeps
+ // the row-by-row sequences confined to ACE targets.
+ if (STI.hasACEV1())
+ return Load ? X86::ACE_TILERELOAD : X86::ACE_TILESPILL;
#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
: GET_EGPR_IF_ENABLED(X86::TILESTORED);
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index 9b97025d8532c..c6754b1e9022a 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -122,8 +122,8 @@ static bool isACEOnlyIntrinsic(Instruction *I) {
case Intrinsic::x86_top4mxbf8ps_internal:
case Intrinsic::x86_top4mxbssps_internal:
// ACE tile movement intrinsics
- case Intrinsic::x86_tilesetcol_internal:
- case Intrinsic::x86_tilesetrow_internal:
+ case Intrinsic::x86_tilemovcol_set_internal:
+ case Intrinsic::x86_tilemovrow_set_internal:
return true;
default:
return false;
@@ -298,8 +298,8 @@ std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) {
}
// ACE TILEMOV intrinsics - move ZMM to tile row/column
// Pattern: (m, n, zmm_src, idx) -> tile
- case Intrinsic::x86_tilesetcol_internal:
- case Intrinsic::x86_tilesetrow_internal: {
+ case Intrinsic::x86_tilemovcol_set_internal:
+ case Intrinsic::x86_tilemovrow_set_internal: {
// Output tile shape
Row = II->getArgOperand(0);
Col = II->getArgOperand(1);
@@ -836,16 +836,17 @@ namespace {
class X86LowerAMXCast {
Function &Func;
std::unique_ptr<DominatorTree> DT;
- bool IsACEOnly; // ACE v1 without AMX TILELOADD/TILESTORED
+ bool IsACE; // ACE configures palette 2, which has no TILELOADD/TILESTORED
public:
X86LowerAMXCast(Function &F, const TargetMachine *TM = nullptr)
- : Func(F), DT(nullptr), IsACEOnly(false) {
+ : Func(F), DT(nullptr), IsACE(false) {
if (TM) {
- // Check if we're ACE-only (no AMX TILELOADD/TILESTORED available)
+ // ACE implies AMX-TILE, but ACE code runs under palette 2 where
+ // TILELOADD/TILESTORED are unavailable, so key off ACE itself.
const X86Subtarget *ST =
static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
- IsACEOnly = ST && ST->hasACEV1() && !ST->hasAMXTILE();
+ IsACE = ST && ST->hasACEV1();
}
}
bool combineCastStore(IntrinsicInst *Cast, StoreInst *ST);
@@ -1075,7 +1076,7 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
auto *II = cast<IntrinsicInst>(Tile);
// ACE v1 doesn't have TILESTORED instruction - skip combining for ACE targets
// or ACE intrinsics. ACE uses struct-based API with vector load/store.
- if (IsACEOnly || isACEOnlyIntrinsic(II))
+ if (IsACE || isACEOnlyIntrinsic(II))
return false;
// Tile is output from AMX intrinsic. The first operand of the
// intrinsic is row, the second operand of the intrinsic is column.
@@ -1109,7 +1110,7 @@ bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
return false;
// ACE v1 doesn't have TILELOADD instruction - skip combining for ACE targets
// or ACE intrinsics. ACE uses struct-based API with vector load/store.
- if (IsACEOnly || isACEOnlyIntrinsic(II))
+ if (IsACE || isACEOnlyIntrinsic(II))
return false;
std::tie(Row, Col) = getShape(II, OpNo);
IRBuilder<> Builder(LD);
@@ -1347,7 +1348,7 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
// ACE v1 doesn't have TILELOADD - for ACE targets or ACE intrinsics,
// use tilezero since ACE tiles are typically zero-initialized in struct
// API.
- if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ if (IsACE || isACEOnlyIntrinsic(II)) {
Value *Row = nullptr, *Col = nullptr;
std::tie(Row, Col) = getShape(II, OpNo);
Value *NewInst = Builder.CreateIntrinsic(Intrinsic::x86_tilezero_internal,
@@ -1381,7 +1382,7 @@ bool X86LowerAMXCast::transformAMXCast(IntrinsicInst *AMXCast) {
// ACE v1 doesn't have TILESTORED - for ACE targets or ACE intrinsics,
// just create a poison value since the tile result is typically not read
// back.
- if (IsACEOnly || isACEOnlyIntrinsic(II)) {
+ if (IsACE || isACEOnlyIntrinsic(II)) {
Value *Poison = PoisonValue::get(AMXCast->getType());
AMXCast->replaceAllUsesWith(Poison);
AMXCast->eraseFromParent();
@@ -1447,7 +1448,12 @@ bool lowerAmxType(Function &F, const TargetMachine *TM,
// "Clang -O2 -S -emit-llvm t.c" + "llc t.ll") we should make
// sure the amx data is volatile, that is necessary for AMX fast
// register allocation.
- if (!F.hasFnAttribute(Attribute::OptimizeNone)) {
+ // This spills tiles with TILESTORED and reloads them with TILELOADD, which
+ // ACE cannot do under palette 2. ACE tile data is left non-volatile, so the
+ // tile registers stay live and X86FastPreTileConfig spills them instead.
+ const X86Subtarget *ST =
+ static_cast<const X86Subtarget *>(TM->getSubtargetImpl(F));
+ if (!F.hasFnAttribute(Attribute::OptimizeNone) && !(ST && ST->hasACEV1())) {
X86VolatileTileData VTD(F);
C = VTD.volatileTileData() || C;
}
diff --git a/llvm/lib/Target/X86/X86LowerTileCopy.cpp b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
index 09082ba585572..4b9bf9639c242 100644
--- a/llvm/lib/Target/X86/X86LowerTileCopy.cpp
+++ b/llvm/lib/Target/X86/X86LowerTileCopy.cpp
@@ -36,62 +36,63 @@ using namespace llvm;
#define DEBUG_TYPE "x86-lower-tile-copy"
-/// Generate ACE-specific tile copy using TILEMOVROW row-by-row.
-/// ACE v1 doesn't have TILELOADD/TILESTORED, so we need to copy
-/// tiles through ZMM registers, one row at a time.
-///
-/// Spill sequence (tile -> stack):
-/// for row = 0 to 15:
-/// TILEMOVROW zmm_temp, tmm_src, row ; read row from tile to ZMM
-/// VMOVUPS [stack + row*64], zmm_temp ; store ZMM to stack
-///
-/// Reload sequence (stack -> tile):
-/// for row = 0 to 15:
-/// VMOVUPS zmm_temp, [stack + row*64] ; load ZMM from stack
-/// TILEMOVROW tmm_dst, zmm_temp, row ; write row from ZMM to tile
-///
-static void emitACETileCopy(MachineBasicBlock &MBB, MachineInstr &MI,
- const X86Subtarget &ST, const X86InstrInfo *TII,
- const TargetRegisterInfo *TRI, Register SrcReg,
- Register DstReg, bool SrcKill, int TileSS,
- Register ScratchZMM) {
- const DebugLoc &DL = MI.getDebugLoc();
-
- // Each tile has 16 rows of 64 bytes each (1KB total)
- const unsigned NumRows = 16;
- const unsigned RowSize = 64;
-
- // Spill: Read each row from source tile to ZMM, then store to stack
- for (unsigned Row = 0; Row < NumRows; ++Row) {
- // TILEMOVROW zmm, tmm, imm8 (read direction: tile -> ZMM)
- // Uses TILEMOVROWrti instruction
- // Only kill src on the last row read
- bool KillSrcNow = (Row == NumRows - 1) && SrcKill;
+// ACE has no TILELOADD/TILESTORED, so a tile moves to and from memory one row
+// at a time through a scratch ZMM. Each tile is 16 rows of 64 bytes (1KB).
+static const unsigned ACENumTileRows = 16;
+static const unsigned ACETileRowSize = 64;
+
+/// Emit the ACE tile spill sequence, storing \p SrcReg to \p TileSS.
+static void emitACETileSpill(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ const X86InstrInfo *TII, Register SrcReg,
+ bool SrcKill, int TileSS, Register ScratchZMM) {
+ const DebugLoc &DL = MI->getDebugLoc();
+ for (unsigned Row = 0; Row < ACENumTileRows; ++Row) {
+ // tilemovrow $row, %tmm, %zmm
+ // Only kill src on the last row read.
+ bool KillSrcNow = (Row == ACENumTileRows - 1) && SrcKill;
BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWrti), ScratchZMM)
.addReg(SrcReg, getKillRegState(KillSrcNow))
.addImm(Row);
- // VMOVUPS [stack + row*64], zmm
+ // vmovups %zmm, row*64(%sp)
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
- addFrameReference(MIB, TileSS, Row * RowSize);
+ addFrameReference(MIB, TileSS, Row * ACETileRowSize);
MIB.addReg(ScratchZMM, RegState::Kill);
}
+}
- // Reload: Load each row from stack to ZMM, then write to dest tile
- for (unsigned Row = 0; Row < NumRows; ++Row) {
- // VMOVUPS zmm, [stack + row*64]
+/// Emit the ACE tile reload sequence, loading \p DstReg from \p TileSS.
+static void emitACETileReload(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MI,
+ const X86InstrInfo *TII, Register DstReg,
+ int TileSS, Register ScratchZMM) {
+ const DebugLoc &DL = MI->getDebugLoc();
+ for (unsigned Row = 0; Row < ACENumTileRows; ++Row) {
+ // vmovups row*64(%sp), %zmm
MachineInstrBuilder MIB =
BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), ScratchZMM);
- addFrameReference(MIB, TileSS, Row * RowSize);
+ addFrameReference(MIB, TileSS, Row * ACETileRowSize);
- // TILEMOVROW tmm, zmm, imm8 (write direction: ZMM -> tile)
- // Uses TILEMOVROWri instruction (ACE-specific)
+ // tilemovrow $row, %zmm, %tmm
BuildMI(MBB, MI, DL, TII->get(X86::TILEMOVROWri), DstReg)
.addReg(ScratchZMM, RegState::Kill)
.addImm(Row);
}
}
+/// Pick a ZMM that is dead here, or NoRegister if none is available.
+static Register findScratchZMM(MachineFunction &MF,
+ const TargetRegisterInfo *TRI,
+ const LiveRegUnits &UsedRegs) {
+ BitVector VR512Regs =
+ TRI->getAllocatableSet(MF, TRI->getRegClass(X86::VR512RegClassID));
+ for (auto RegT : VR512Regs.set_bits())
+ if (UsedRegs.available(RegT))
+ return RegT;
+ return X86::NoRegister;
+}
+
namespace {
class X86LowerTileCopyLegacy : public MachineFunctionPass {
@@ -148,6 +149,48 @@ static bool lowerTileCopy(MachineFunction &MF) {
if (MI.isDebugInstr())
continue;
UsedRegs.stepBackward(MI);
+
+ // Expand the tile spill/reload pseudos inserted by the register
+ // allocator. getLoadStoreRegOpcode is the only place these are created,
+ // and it only does so for ACE targets.
+ unsigned Opcode = MI.getOpcode();
+ if (Opcode == X86::ACE_TILESPILL || Opcode == X86::ACE_TILERELOAD) {
+ assert(ST.hasACEV1() && "ACE tile spill pseudo on non-ACE target");
+ bool IsSpill = Opcode == X86::ACE_TILESPILL;
+ // ACE_TILESPILL is (mem, tile), ACE_TILERELOAD is (tile, mem).
+ MachineOperand &TileMO =
+ MI.getOperand(IsSpill ? X86::AddrNumOperands : 0);
+ int TileSS =
+ MI.getOperand((IsSpill ? 0 : 1) + X86::AddrBaseReg).getIndex();
+ const DebugLoc &DL = MI.getDebugLoc();
+
+ Register ScratchZMM = findScratchZMM(MF, TRI, UsedRegs);
+ int ZmmSS = -1;
+ if (!ScratchZMM) {
+ // No available register? Save ZMM0 and reload it after use.
+ ScratchZMM = X86::ZMM0;
+ ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)),
+ ZmmSS)
+ .addReg(X86::ZMM0);
+ }
+
+ if (IsSpill)
+ emitACETileSpill(MBB, MI, TII, TileMO.getReg(), TileMO.isKill(),
+ TileSS, ScratchZMM);
+ else
+ emitACETileReload(MBB, MI, TII, TileMO.getReg(), TileSS, ScratchZMM);
+
+ if (ZmmSS != -1)
+ addFrameReference(
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0),
+ ZmmSS);
+
+ MI.eraseFromParent();
+ Changed = true;
+ continue;
+ }
+
if (!MI.isCopy())
continue;
MachineOperand &DstMO = MI.getOperand(0);
@@ -164,47 +207,28 @@ static bool lowerTileCopy(MachineFunction &MF) {
const DebugLoc &DL = MI.getDebugLoc();
- // Check if this is ACE-only (no AMX TILELOADD/TILESTORED available)
- if (ST.hasACEV1() && !ST.hasAMXTILE()) {
- // ACE v1: Use TILEMOVROW row-by-row copy
- // Need a scratch ZMM register for the transfer
-
- // Find an available ZMM register
- BitVector VR512Regs =
- TRI->getAllocatableSet(MF, TRI->getRegClass(X86::VR512RegClassID));
- Register ScratchZMM = X86::NoRegister;
- for (auto RegT : VR512Regs.set_bits()) {
- if (UsedRegs.available(RegT)) {
- ScratchZMM = RegT;
- break;
- }
- }
-
+ // ACE configures palette 2, which has no TILELOADD/TILESTORED, so copy
+ // the tile row-by-row through a scratch ZMM.
+ if (ST.hasACEV1()) {
+ Register ScratchZMM = findScratchZMM(MF, TRI, UsedRegs);
+ int ZmmSS = -1;
if (!ScratchZMM) {
- // If no ZMM available, use ZMM0 and save/restore it
+ // No available register? Save ZMM0 and reload it after use.
ScratchZMM = X86::ZMM0;
+ ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
+ addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)),
+ ZmmSS)
+ .addReg(X86::ZMM0);
+ }
- // Allocate stack slot for scratch ZMM
- int ZmmSS = MF.getFrameInfo().CreateSpillStackObject(64, Align(64));
-
- // Save ZMM0
- MachineInstrBuilder MIB =
- BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr));
- addFrameReference(MIB, ZmmSS);
- MIB.addReg(X86::ZMM0);
-
- // Emit ACE tile copy
- emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
- TileSS, ScratchZMM);
+ emitACETileSpill(MBB, MI, TII, SrcReg, SrcMO.isKill(), TileSS,
+ ScratchZMM);
+ emitACETileReload(MBB, MI, TII, DstReg, TileSS, ScratchZMM);
- // Restore ZMM0
- MIB = BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0);
- addFrameReference(MIB, ZmmSS);
- } else {
- // Use available scratch ZMM
- emitACETileCopy(MBB, MI, ST, TII, TRI, SrcReg, DstReg, SrcMO.isKill(),
- TileSS, ScratchZMM);
- }
+ if (ZmmSS != -1)
+ addFrameReference(
+ BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZrm), X86::ZMM0),
+ ZmmSS);
} else {
// AMX: Use TILESTORED/TILELOADD (original code)
int StrideSS = 0;
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 1ee8f5ccb6db5..d0c17308413a0 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -543,6 +543,9 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
// Set the SIMD floating point control register as reserved.
Reserved.set(X86::MXCSR);
+ // Set the ACE block scale register as reserved.
+ Reserved.set(X86::BSR0);
+
// Set the stack-pointer register and its aliases as reserved.
for (const MCPhysReg &SubReg : subregs_inclusive(X86::RSP))
Reserved.set(SubReg);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 8f25a02de09b4..2ca83c7dd484f 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -639,9 +639,14 @@ constexpr FeatureBitset ImpliedFeaturesAMX_FP8 = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_MOVRS = FeatureAMX_TILE;
constexpr FeatureBitset ImpliedFeaturesAMX_AVX512 =
FeatureAMX_TILE | FeatureAVX10_2;
-constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAMX_AVX512;
constexpr FeatureBitset ImpliedFeaturesHRESET = {};
+// ACE v1 Features. AVX10.1 is the vector baseline required by the ACE v1
+// feature detection algorithm. AMX-TILE provides the tile registers and tile
+// management instructions that ACE reuses; per spec section 15.5.5 an ACE-only
+// implementation still reports it.
+constexpr FeatureBitset ImpliedFeaturesACEV1 = FeatureAVX10_1 | FeatureAMX_TILE;
+
constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {};
constexpr FeatureBitset ImpliedFeaturesCMPCCXADD = {};
constexpr FeatureBitset ImpliedFeaturesRAOINT = {};
diff --git a/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll b/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
deleted file mode 100644
index 9afca28b9c9ac..0000000000000
--- a/llvm/test/CodeGen/X86/ACE/ace-intrinsics.ll
+++ /dev/null
@@ -1,244 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f | FileCheck %s
-
-;
-; ACE Intrinsic Tests
-;
-
-; Test BSRINIT intrinsic
-define void @test_bsrinit() {
-; CHECK-LABEL: test_bsrinit:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrinit %bsr0
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrinit()
- ret void
-}
-
-; Test BSRMOVF intrinsic
-define void @test_bsrmovf(<16 x i32> %a, <16 x i32> %b) {
-; CHECK-LABEL: test_bsrmovf:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrmovf(<16 x i32> %a, <16 x i32> %b)
- ret void
-}
-
-; Test BSRMOVH set intrinsic
-define void @test_bsrmovh_set(<16 x i32> %a) {
-; CHECK-LABEL: test_bsrmovh_set:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovh %zmm0, %bsr0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrmovh.set(<16 x i32> %a)
- ret void
-}
-
-; Test BSRMOVH get intrinsic
-define <16 x i32> @test_bsrmovh_get() {
-; CHECK-LABEL: test_bsrmovh_get:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovh %bsr0, %zmm0
-; CHECK-NEXT: retq
- %result = call <16 x i32> @llvm.x86.bsrmovh.get()
- ret <16 x i32> %result
-}
-
-; Test BSRMOVL set intrinsic
-define void @test_bsrmovl_set(<16 x i32> %a) {
-; CHECK-LABEL: test_bsrmovl_set:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovl %zmm0, %bsr0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.bsrmovl.set(<16 x i32> %a)
- ret void
-}
-
-; Test BSRMOVL get intrinsic
-define <16 x i32> @test_bsrmovl_get() {
-; CHECK-LABEL: test_bsrmovl_get:
-; CHECK: # %bb.0:
-; CHECK-NEXT: bsrmovl %bsr0, %zmm0
-; CHECK-NEXT: retq
- %result = call <16 x i32> @llvm.x86.bsrmovl.get()
- ret <16 x i32> %result
-}
-
-; Test TOP2BF16PS intrinsic
-define void @test_top2bf16ps() {
-; CHECK-LABEL: test_top2bf16ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top2bf16ps %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top2bf16ps(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BUUD intrinsic
-define void @test_top4buud() {
-; CHECK-LABEL: test_top4buud:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4buud %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4buud(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BUSD intrinsic
-define void @test_top4busd() {
-; CHECK-LABEL: test_top4busd:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4busd %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4busd(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BSSD intrinsic
-define void @test_top4bssd() {
-; CHECK-LABEL: test_top4bssd:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4bssd %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4bssd(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4BSUD intrinsic
-define void @test_top4bsud() {
-; CHECK-LABEL: test_top4bsud:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4bsud %zmm2, %zmm1, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4bsud(i8 0, i8 1, i8 2)
- ret void
-}
-
-; Test TOP4MXHF8PS intrinsic
-define void @test_top4mxhf8ps() {
-; CHECK-LABEL: test_top4mxhf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxhf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxhf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXBHF8PS intrinsic
-define void @test_top4mxbhf8ps() {
-; CHECK-LABEL: test_top4mxbhf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxbhf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxbhf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXHBF8PS intrinsic
-define void @test_top4mxhbf8ps() {
-; CHECK-LABEL: test_top4mxhbf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxhbf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxhbf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXBF8PS intrinsic
-define void @test_top4mxbf8ps() {
-; CHECK-LABEL: test_top4mxbf8ps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxbf8ps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxbf8ps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TOP4MXBSSPS intrinsic (MX INT8 S×S with BSR scaling)
-define void @test_top4mxbssps() {
-; CHECK-LABEL: test_top4mxbssps:
-; CHECK: # %bb.0:
-; CHECK-NEXT: top4mxbssps $5, %zmm1, %zmm2, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.top4mxbssps(i8 0, i8 1, i8 2, i8 5)
- ret void
-}
-
-; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
-define void @test_tilemovcol(<16 x i32> %src) {
-; CHECK-LABEL: test_tilemovcol:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetcol(i8 0, <16 x i32> %src, i32 3)
- ret void
-}
-
-; Test TILEMOVROW intrinsic - write to tile (ACE extension)
-; Uses constant index - generates immediate form
-define void @test_tilemovrow_to_tile(<16 x i32> %src) {
-; CHECK-LABEL: test_tilemovrow_to_tile:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetrow(i8 1, <16 x i32> %src, i32 5)
- ret void
-}
-
-; Test TILEMOVCOL with register index
-define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
-; CHECK-LABEL: test_tilemovcol_reg:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetcol(i8 2, <16 x i32> %src, i32 %idx)
- ret void
-}
-
-; Test TILEMOVROW with register index
-define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
-; CHECK-LABEL: test_tilemovrow_reg:
-; CHECK: # %bb.0:
-; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
- call void @llvm.x86.tilesetrow(i8 3, <16 x i32> %src, i32 %idx)
- ret void
-}
-
-; Intrinsic declarations
-declare void @llvm.x86.bsrinit()
-declare void @llvm.x86.bsrmovf(<16 x i32>, <16 x i32>)
-declare void @llvm.x86.bsrmovh.set(<16 x i32>)
-declare <16 x i32> @llvm.x86.bsrmovh.get()
-declare void @llvm.x86.bsrmovl.set(<16 x i32>)
-declare <16 x i32> @llvm.x86.bsrmovl.get()
-declare void @llvm.x86.tilesetcol(i8, <16 x i32>, i32)
-declare void @llvm.x86.tilesetrow(i8, <16 x i32>, i32)
-declare void @llvm.x86.top2bf16ps(i8, i8, i8)
-declare void @llvm.x86.top4buud(i8, i8, i8)
-declare void @llvm.x86.top4busd(i8, i8, i8)
-declare void @llvm.x86.top4bssd(i8, i8, i8)
-declare void @llvm.x86.top4bsud(i8, i8, i8)
-declare void @llvm.x86.top4mxhf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxbhf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxhbf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxbf8ps(i8, i8, i8, i8)
-declare void @llvm.x86.top4mxbssps(i8, i8, i8, i8)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
similarity index 87%
rename from llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
rename to llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
index 4b114d5bc9c08..ab72e042e0b99 100644
--- a/llvm/test/CodeGen/X86/ACE/acev1-greedy-ra.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-greedy-ra.ll
@@ -1,10 +1,11 @@
-; ACE v1 Greedy Register Allocation Test
-; Tests that tile registers are allocated correctly in a separate pass
-; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
-;
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs -stop-after x86-tile-config | FileCheck %s
+
+; Test the tile register is allocated in a separate pass. ACE v1 alone, and
+; composed with AMX-TILE, must both work. ACE v1 extracts tile data with
+; TILEMOVROW rather than TILESTORED.
-; Test basic tile register allocation for ACE
define void @test_acev1_ra(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) nounwind {
; CHECK-LABEL: name: test_acev1_ra
; CHECK: PLDTILECFGV
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
new file mode 100644
index 0000000000000..d7a913f8e2290
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-intrinsics.ll
@@ -0,0 +1,357 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,-avx10.2 -verify-machineinstrs --show-mc-encoding | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs --show-mc-encoding | FileCheck %s
+
+; Test the ACE v1 intrinsics. acev1 implies its AVX10.1 and AMX-TILE baselines,
+; so it is sufficient alone. Nothing here depends on AVX10.2, which is above the
+; baseline, and requesting AMX-TILE explicitly is redundant but must not change
+; anything.
+
+; Test BSRINIT intrinsic
+define void @test_bsrinit() {
+; CHECK-LABEL: test_bsrinit:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrinit %bsr0 # encoding: [0xc4,0xe2,0xfb,0x49,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrinit()
+ ret void
+}
+
+; Test BSRMOVF intrinsic
+define void @test_bsrmovf(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_bsrmovf:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf %zmm1, %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0xc1]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with memory operand
+define void @test_bsrmovf_mem(<64 x i8> %a, ptr %p) {
+; CHECK-LABEL: test_bsrmovf_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf (%rdi), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x07]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %b = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with memory operand (base+index*scale+disp)
+define void @test_bsrmovf_mem_amode(<64 x i8> %a, ptr %base, i64 %idx) {
+; CHECK-LABEL: test_bsrmovf_mem_amode:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf 64(%rdi,%rsi,4), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x44,0xb7,0x01]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %elt = getelementptr i32, ptr %base, i64 %idx
+ %addr = getelementptr i8, ptr %elt, i64 64
+ %b = load <64 x i8>, ptr %addr
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with unaligned memory operand
+define void @test_bsrmovf_mem_unaligned(<64 x i8> %a, ptr %p) {
+; CHECK-LABEL: test_bsrmovf_mem_unaligned:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovf (%rdi), %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfc,0x48,0x95,0x07]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %b = load <64 x i8>, ptr %p, align 1
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVF intrinsic with memory operand in first source (not foldable)
+define void @test_bsrmovf_mem_src1(ptr %p, <64 x i8> %b) {
+; CHECK-LABEL: test_bsrmovf_mem_src1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vmovaps (%rdi), %zmm1 # encoding: [0x62,0xf1,0x7c,0x48,0x28,0x0f]
+; CHECK-NEXT: bsrmovf %zmm0, %zmm1, %bsr0 # encoding: [0x62,0xf6,0xf4,0x48,0x95,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %a = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovf(<64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test BSRMOVH set intrinsic
+define void @test_bsrmovh_set(<64 x i8> %a) {
+; CHECK-LABEL: test_bsrmovh_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %zmm0, %bsr0 # encoding: [0x62,0xf6,0xff,0x48,0x95,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrmovh.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVH set intrinsic with memory operand
+define void @test_bsrmovh_set_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovh_set_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh (%rdi), %bsr0 # encoding: [0x62,0xf6,0xff,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %a = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovh.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVH get intrinsic
+define <64 x i8> @test_bsrmovh_get() {
+; CHECK-LABEL: test_bsrmovh_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, %zmm0 # encoding: [0x62,0xf6,0x7f,0x48,0x95,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ ret <64 x i8> %result
+}
+
+; Test BSRMOVH get intrinsic with memory operand
+define void @test_bsrmovh_get_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovh_get_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7f,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ store <64 x i8> %result, ptr %p
+ ret void
+}
+
+; Test BSRMOVH get intrinsic with unaligned memory operand
+define void @test_bsrmovh_get_mem_unaligned(ptr %p) {
+; CHECK-LABEL: test_bsrmovh_get_mem_unaligned:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovh %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7f,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovh.get()
+ store <64 x i8> %result, ptr %p, align 1
+ ret void
+}
+
+; Test BSRMOVL set intrinsic
+define void @test_bsrmovl_set(<64 x i8> %a) {
+; CHECK-LABEL: test_bsrmovl_set:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %zmm0, %bsr0 # encoding: [0x62,0xf6,0xfe,0x48,0x95,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.bsrmovl.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVL set intrinsic with memory operand
+define void @test_bsrmovl_set_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovl_set_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl (%rdi), %bsr0 # encoding: [0x62,0xf6,0xfe,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %a = load <64 x i8>, ptr %p
+ call void @llvm.x86.bsrmovl.set(<64 x i8> %a)
+ ret void
+}
+
+; Test BSRMOVL get intrinsic
+define <64 x i8> @test_bsrmovl_get() {
+; CHECK-LABEL: test_bsrmovl_get:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %bsr0, %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x95,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovl.get()
+ ret <64 x i8> %result
+}
+
+; Test BSRMOVL get intrinsic with memory operand
+define void @test_bsrmovl_get_mem(ptr %p) {
+; CHECK-LABEL: test_bsrmovl_get_mem:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bsrmovl %bsr0, (%rdi) # encoding: [0x62,0xf6,0x7e,0x48,0x95,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ %result = call <64 x i8> @llvm.x86.bsrmovl.get()
+ store <64 x i8> %result, ptr %p
+ ret void
+}
+
+; The outer products name only the accumulator tile by ID; both ZMM sources are
+; ordinary values assigned by the register allocator.
+
+; Test TOP2BF16PS intrinsic
+define void @test_top2bf16ps(<32 x bfloat> %a, <32 x bfloat> %b) {
+; CHECK-LABEL: test_top2bf16ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top2bf16ps %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x76,0x48,0x5c,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top2bf16ps(i8 0, <32 x bfloat> %a, <32 x bfloat> %b)
+ ret void
+}
+
+; Test TOP4BUUD intrinsic
+define void @test_top4buud(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4buud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4buud %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x74,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4buud(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4BUSD intrinsic
+define void @test_top4busd(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4busd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4busd %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x75,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4busd(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4BSSD intrinsic
+define void @test_top4bssd(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4bssd:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bssd %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x77,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4bssd(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4BSUD intrinsic
+define void @test_top4bsud(<64 x i8> %a, <64 x i8> %b) {
+; CHECK-LABEL: test_top4bsud:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4bsud %zmm1, %zmm0, %tmm0 # encoding: [0x62,0xf2,0x76,0x48,0x5e,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4bsud(i8 0, <64 x i8> %a, <64 x i8> %b)
+ ret void
+}
+
+; Test TOP4MXHF8PS intrinsic
+define void @test_top4mxhf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxhf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXBHF8PS intrinsic
+define void @test_top4mxbhf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxbhf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbhf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxbhf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXHBF8PS intrinsic
+define void @test_top4mxhbf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxhbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxhbf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxhbf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXBF8PS intrinsic
+define void @test_top4mxbf8ps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxbf8ps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbf8ps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7c,0x48,0x8d,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxbf8ps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TOP4MXBSSPS intrinsic (MX INT8 SxS with BSR scaling)
+define void @test_top4mxbssps(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: test_top4mxbssps:
+; CHECK: # %bb.0:
+; CHECK-NEXT: top4mxbssps $5, %zmm0, %zmm1, %tmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x8f,0xc1,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.top4mxbssps(i8 0, <16 x i32> %a, <16 x i32> %b, i8 5)
+ ret void
+}
+
+; Test TILEMOVCOL intrinsic (constant index - generates immediate form)
+define void @test_tilemovcol(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovcol:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol $3, %zmm0, %tmm0 # encoding: [0x62,0xf3,0xfd,0x48,0x2f,0xc0,0x03]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovcol.set(i8 0, <16 x i32> %src, i32 3)
+ ret void
+}
+
+; Test TILEMOVROW intrinsic - write to tile (ACE extension)
+; Uses constant index - generates immediate form
+define void @test_tilemovrow_to_tile(<16 x i32> %src) {
+; CHECK-LABEL: test_tilemovrow_to_tile:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow $5, %zmm0, %tmm1 # encoding: [0x62,0xf3,0xfd,0x48,0x07,0xc8,0x05]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovrow.set(i8 1, <16 x i32> %src, i32 5)
+ ret void
+}
+
+; Test TILEMOVCOL with register index
+define void @test_tilemovcol_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovcol_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovcol %edi, %zmm0, %tmm2 # encoding: [0x62,0xf2,0xc5,0x48,0x4b,0xd0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovcol.set(i8 2, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Test TILEMOVROW with register index
+define void @test_tilemovrow_reg(<16 x i32> %src, i32 %idx) {
+; CHECK-LABEL: test_tilemovrow_reg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: tilemovrow %edi, %zmm0, %tmm3 # encoding: [0x62,0xf2,0xc5,0x48,0x4a,0xd8]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+ call void @llvm.x86.tilemovrow.set(i8 3, <16 x i32> %src, i32 %idx)
+ ret void
+}
+
+; Intrinsic declarations
+declare void @llvm.x86.bsrinit()
+declare void @llvm.x86.bsrmovf(<64 x i8>, <64 x i8>)
+declare void @llvm.x86.bsrmovh.set(<64 x i8>)
+declare <64 x i8> @llvm.x86.bsrmovh.get()
+declare void @llvm.x86.bsrmovl.set(<64 x i8>)
+declare <64 x i8> @llvm.x86.bsrmovl.get()
+declare void @llvm.x86.tilemovcol.set(i8, <16 x i32>, i32)
+declare void @llvm.x86.tilemovrow.set(i8, <16 x i32>, i32)
+declare void @llvm.x86.top2bf16ps(i8, <32 x bfloat>, <32 x bfloat>)
+declare void @llvm.x86.top4buud(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4busd(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4bssd(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4bsud(i8, <64 x i8>, <64 x i8>)
+declare void @llvm.x86.top4mxhf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxbhf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxhbf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxbf8ps(i8, <16 x i32>, <16 x i32>, i8)
+declare void @llvm.x86.top4mxbssps(i8, <16 x i32>, <16 x i32>, i8)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
similarity index 94%
rename from llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
rename to llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
index 47228dfb8e614..7586363819a69 100644
--- a/llvm/test/CodeGen/X86/ACE/acev1-outer-product.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-outer-product.ll
@@ -1,8 +1,10 @@
-; ACE v1 Outer Product Operations Test
-; Tests all ACE outer product variants: BF16, INT8 (signed/unsigned combinations)
-; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
-;
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs | FileCheck %s
+
+; Test all ACE outer product variants: BF16 and INT8 signed/unsigned
+; combinations. ACE v1 alone, and composed with AMX-TILE, must both work. ACE v1
+; extracts tile data with TILEMOVROW rather than TILESTORED.
; Test BF16 outer product (TOP2BF16PS)
define void @test_top2bf16ps(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
new file mode 100644
index 0000000000000..a97adffe41db8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-O0.ll
@@ -0,0 +1,67 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -O0 \
+; RUN: -verify-machineinstrs | FileCheck %s \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 \
+; RUN: -verify-machineinstrs | FileCheck %s \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+
+; Test that tile data is not made volatile at -O0. X86VolatileTileData shortens
+; tile live ranges before fast register allocation by round-tripping every tile
+; through memory with TILESTORED/TILELOADD. ACE configures palette 2, where
+; those are unavailable, so the transform must be skipped and no tile load/store
+; may appear. -O2 is included as a control: the same IR must be clean at both
+; levels.
+;
+; Only one tile is live per function here: with the tile data left non-volatile,
+; anything that keeps a second tile live at -O0 reaches the row-by-row spill in
+; X86FastPreTileConfig, which does not yet produce valid SSA. Extend this test
+; once that is fixed.
+
+; A tile is produced and read back a row at a time, never stored as a tile.
+define void @extract_one_row(ptr %out) nounwind {
+; CHECK-LABEL: extract_one_row:
+; CHECK: movb $2, {{-?[0-9]+}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+entry:
+ %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %r = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x i32> %r, ptr %out, align 64
+ ret void
+}
+
+; Several rows of the same tile, so the tile stays live across extractions.
+define void @extract_several_rows(ptr %out) nounwind {
+; CHECK-LABEL: extract_several_rows:
+; CHECK: movb $2, {{-?[0-9]+}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; CHECK: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+entry:
+ %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x i32> %r0, ptr %out, align 64
+ %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %t, i32 7)
+ store volatile <16 x i32> %r7, ptr %out, align 64
+ ret void
+}
+
+; The converting row reads are gated the same way and must also stay clean.
+define void @extract_row_convert(ptr %out) nounwind {
+; CHECK-LABEL: extract_row_convert:
+; CHECK: movb $2, {{-?[0-9]+}}(%rsp)
+; CHECK: ldtilecfg
+; CHECK: tilezero %tmm{{[0-7]}}
+; CHECK: tcvtrowd2ps ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+entry:
+ %t = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %r = call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 16, i16 64, x86_amx %t, i32 0)
+ store volatile <16 x float> %r, ptr %out, align 64
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
+declare <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
similarity index 88%
rename from llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
rename to llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
index 2d94260701e40..711de8d3c9847 100644
--- a/llvm/test/CodeGen/X86/ACE/acev1-tile-basic.ll
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-basic.ll
@@ -1,8 +1,10 @@
-; ACE v1 Basic Tile Operations Test
-; Tests basic ACE tile operations: tilezero, outer products
-; Note: ACE v1 uses TILEMOVROW to extract tile data (not TILESTORED)
-;
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx512f,+avx512bf16 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+avx10.1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs | FileCheck %s
+
+; Test basic ACE tile operations: tilezero and outer products. ACE v1 alone, and
+; composed with AMX-TILE, must both work. ACE v1 extracts tile data with
+; TILEMOVROW rather than TILESTORED.
define void @test_acev1_basic(ptr %out, <32 x bfloat> %zmm_a, <32 x bfloat> %zmm_b) {
; CHECK-LABEL: test_acev1_basic:
diff --git a/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
new file mode 100644
index 0000000000000..52098739b8d8a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ACEV1/acev1-tile-spill.ll
@@ -0,0 +1,124 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1 -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefix=ACE \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+acev1,+amx-tile -verify-machineinstrs \
+; RUN: | FileCheck %s --check-prefix=ACE \
+; RUN: --implicit-check-not=tilestored --implicit-check-not=tileloadd
+
+; Test tile spilling. Nine simultaneously live tiles exceed the eight physical
+; TMM registers, so the tile register allocator must spill. ACE v1 configures
+; palette 2, which has no TILELOADD/TILESTORED, so a spill moves one row at a
+; time through a scratch ZMM. ACE v1 alone, and composed with AMX-TILE, must
+; both avoid TILELOADD/TILESTORED: the palette is 2 whenever ACE is enabled, and
+; those forms are unavailable there.
+;
+; tilemovrow prints the same mnemonic in both directions; the operand order
+; distinguishes them ($idx, %tmm, %zmm reads a row out, $idx, %zmm, %tmm writes
+; one in).
+
+; Nine live tiles accumulated with bf16 outer products.
+define void @spill_nine_tiles_bf16(ptr %out, <32 x bfloat> %a, <32 x bfloat> %b) nounwind {
+; ACE-LABEL: spill_nine_tiles_bf16:
+; ACE: ldtilecfg
+; Spill reads each of the 16 rows into a ZMM and stores it to the stack slot.
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; Reload loads each row back and writes it into the tile.
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
+entry:
+ %t0 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t3 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t4 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t5 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d0 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t0, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d1 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t1, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d2 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t2, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d3 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t3, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d4 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t4, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d5 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t5, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d6 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t6, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d7 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t7, <32 x bfloat> %a, <32 x bfloat> %b)
+ %d8 = tail call x86_amx @llvm.x86.top2bf16ps.internal(i16 16, i16 64, i16 64, x86_amx %t8, <32 x bfloat> %a, <32 x bfloat> %b)
+
+ ; Keep all nine results live to the end so the allocator has to spill.
+ %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ store volatile <16 x i32> %r0, ptr %out, align 64
+ %r1 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ store volatile <16 x i32> %r1, ptr %out, align 64
+ %r2 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %r2, ptr %out, align 64
+ %r3 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ store volatile <16 x i32> %r3, ptr %out, align 64
+ %r4 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ store volatile <16 x i32> %r4, ptr %out, align 64
+ %r5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ store volatile <16 x i32> %r5, ptr %out, align 64
+ %r6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ store volatile <16 x i32> %r6, ptr %out, align 64
+ %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ store volatile <16 x i32> %r7, ptr %out, align 64
+ %r8 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+ store volatile <16 x i32> %r8, ptr %out, align 64
+
+ ret void
+}
+
+; Same pressure, integer outer products, to check the spill path is independent
+; of the producing instruction.
+define void @spill_nine_tiles_int(ptr %out, <64 x i8> %a) nounwind {
+; ACE-LABEL: spill_nine_tiles_int:
+; ACE: ldtilecfg
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %tmm{{[0-7]}}, %zmm{{[0-9]+}}
+; ACE-COUNT-16: tilemovrow ${{[0-9]+}}, %zmm{{[0-9]+}}, %tmm{{[0-7]}}
+entry:
+ %t0 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t1 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t2 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t3 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t4 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t5 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t6 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t7 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+ %t8 = tail call x86_amx @llvm.x86.tilezero.internal(i16 16, i16 64)
+
+ %d0 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t0, <64 x i8> %a, <64 x i8> %a)
+ %d1 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t1, <64 x i8> %a, <64 x i8> %a)
+ %d2 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t2, <64 x i8> %a, <64 x i8> %a)
+ %d3 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t3, <64 x i8> %a, <64 x i8> %a)
+ %d4 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t4, <64 x i8> %a, <64 x i8> %a)
+ %d5 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t5, <64 x i8> %a, <64 x i8> %a)
+ %d6 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t6, <64 x i8> %a, <64 x i8> %a)
+ %d7 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t7, <64 x i8> %a, <64 x i8> %a)
+ %d8 = tail call x86_amx @llvm.x86.top4bssd.internal(i16 16, i16 64, i16 64, x86_amx %t8, <64 x i8> %a, <64 x i8> %a)
+
+ %r0 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d0, i32 0)
+ store volatile <16 x i32> %r0, ptr %out, align 64
+ %r1 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d1, i32 0)
+ store volatile <16 x i32> %r1, ptr %out, align 64
+ %r2 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d2, i32 0)
+ store volatile <16 x i32> %r2, ptr %out, align 64
+ %r3 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d3, i32 0)
+ store volatile <16 x i32> %r3, ptr %out, align 64
+ %r4 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d4, i32 0)
+ store volatile <16 x i32> %r4, ptr %out, align 64
+ %r5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d5, i32 0)
+ store volatile <16 x i32> %r5, ptr %out, align 64
+ %r6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d6, i32 0)
+ store volatile <16 x i32> %r6, ptr %out, align 64
+ %r7 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d7, i32 0)
+ store volatile <16 x i32> %r7, ptr %out, align 64
+ %r8 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 16, i16 64, x86_amx %d8, i32 0)
+ store volatile <16 x i32> %r8, ptr %out, align 64
+
+ ret void
+}
+
+declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
+declare x86_amx @llvm.x86.top2bf16ps.internal(i16, i16, i16, x86_amx, <32 x bfloat>, <32 x bfloat>)
+declare x86_amx @llvm.x86.top4bssd.internal(i16, i16, i16, x86_amx, <64 x i8>, <64 x i8>)
+declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32)
diff --git a/llvm/test/MC/Disassembler/X86/acev1.txt b/llvm/test/MC/Disassembler/X86/acev1.txt
new file mode 100644
index 0000000000000..08f3b4f34bad7
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/acev1.txt
@@ -0,0 +1,262 @@
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT
+# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL
+
+## Check that every byte sequence below is also reachable from the assembler.
+# RUN: llvm-mc --disassemble %s -triple=x86_64 | llvm-mc -triple=x86_64 -mattr=+acev1 -filetype=obj -o %t.o
+# RUN: llvm-objdump -d --no-print-imm-hex %t.o | FileCheck %s --check-prefixes=ATT
+
+## BSR management
+
+# ATT: bsrinit %bsr0
+# INTEL: bsrinit bsr0
+0xc4,0xe2,0xfb,0x49,0xc0
+
+## BSRMOVF - fill both BSR halves from two ZMMs
+
+# ATT: bsrmovf %zmm2, %zmm1, %bsr0
+# INTEL: bsrmovf bsr0, zmm1, zmm2
+0x62,0xf6,0xf4,0x48,0x95,0xc2
+
+# ATT: bsrmovf %zmm31, %zmm30, %bsr0
+# INTEL: bsrmovf bsr0, zmm30, zmm31
+0x62,0x96,0x8c,0x40,0x95,0xc7
+
+# ATT: bsrmovf (%rax), %zmm1, %bsr0
+# INTEL: bsrmovf bsr0, zmm1, zmmword ptr [rax]
+0x62,0xf6,0xf4,0x48,0x95,0x00
+
+# ATT: bsrmovf -64(%rsp), %zmm0, %bsr0
+# INTEL: bsrmovf bsr0, zmm0, zmmword ptr [rsp - 64]
+0x62,0xf6,0xfc,0x48,0x95,0x44,0x24,0xff
+
+# ATT: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+# INTEL: bsrmovf bsr0, zmm10, zmmword ptr [rdx + 4*rax + 64]
+0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01
+
+# ATT: bsrmovf 268435456(%rbp,%r14,8), %zmm1, %bsr0
+# INTEL: bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10
+
+## BSRMOVH - high half, set (ZMM to BSR) and get (BSR to ZMM)
+
+# ATT: bsrmovh %zmm1, %bsr0
+# INTEL: bsrmovh bsr0, zmm1
+0x62,0xf6,0xff,0x48,0x95,0xc1
+
+# ATT: bsrmovh %zmm31, %bsr0
+# INTEL: bsrmovh bsr0, zmm31
+0x62,0x96,0xff,0x48,0x95,0xc7
+
+# ATT: bsrmovh (%rbx), %bsr0
+# INTEL: bsrmovh bsr0, zmmword ptr [rbx]
+0x62,0xf6,0xff,0x48,0x95,0x03
+
+# ATT: bsrmovh 64(%rdx,%rax,4), %bsr0
+# INTEL: bsrmovh bsr0, zmmword ptr [rdx + 4*rax + 64]
+0x62,0xf6,0xff,0x48,0x95,0x44,0x82,0x01
+
+# ATT: bsrmovh %bsr0, %zmm1
+# INTEL: bsrmovh zmm1, bsr0
+0x62,0xf6,0x7f,0x48,0x95,0xc1
+
+# ATT: bsrmovh %bsr0, %zmm31
+# INTEL: bsrmovh zmm31, bsr0
+0x62,0x96,0x7f,0x48,0x95,0xc7
+
+# ATT: bsrmovh %bsr0, (%rcx)
+# INTEL: bsrmovh zmmword ptr [rcx], bsr0
+0x62,0xf6,0x7f,0x48,0x95,0x01
+
+# ATT: bsrmovh %bsr0, 268435456(%rbp,%r14,8)
+# INTEL: bsrmovh zmmword ptr [rbp + 8*r14 + 268435456], bsr0
+0x62,0xb6,0x7f,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10
+
+## BSRMOVL - low half, set (ZMM to BSR) and get (BSR to ZMM)
+
+# ATT: bsrmovl %zmm2, %bsr0
+# INTEL: bsrmovl bsr0, zmm2
+0x62,0xf6,0xfe,0x48,0x95,0xc2
+
+# ATT: bsrmovl %zmm30, %bsr0
+# INTEL: bsrmovl bsr0, zmm30
+0x62,0x96,0xfe,0x48,0x95,0xc6
+
+# ATT: bsrmovl (%rdx), %bsr0
+# INTEL: bsrmovl bsr0, zmmword ptr [rdx]
+0x62,0xf6,0xfe,0x48,0x95,0x02
+
+# ATT: bsrmovl 64(%rdx,%rax,4), %bsr0
+# INTEL: bsrmovl bsr0, zmmword ptr [rdx + 4*rax + 64]
+0x62,0xf6,0xfe,0x48,0x95,0x44,0x82,0x01
+
+# ATT: bsrmovl %bsr0, %zmm3
+# INTEL: bsrmovl zmm3, bsr0
+0x62,0xf6,0x7e,0x48,0x95,0xc3
+
+# ATT: bsrmovl %bsr0, %zmm30
+# INTEL: bsrmovl zmm30, bsr0
+0x62,0x96,0x7e,0x48,0x95,0xc6
+
+# ATT: bsrmovl %bsr0, (%rsi)
+# INTEL: bsrmovl zmmword ptr [rsi], bsr0
+0x62,0xf6,0x7e,0x48,0x95,0x06
+
+# ATT: bsrmovl %bsr0, 268435456(%rbp,%r14,8)
+# INTEL: bsrmovl zmmword ptr [rbp + 8*r14 + 268435456], bsr0
+0x62,0xb6,0x7e,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10
+
+## TILEMOVCOL - write a ZMM into a tile column (ACE only, EVEX.W1)
+
+# ATT: tilemovcol $0, %zmm0, %tmm0
+# INTEL: tilemovcol tmm0, zmm0, 0
+0x62,0xf3,0xfd,0x48,0x2f,0xc0,0x00
+
+# ATT: tilemovcol $5, %zmm2, %tmm1
+# INTEL: tilemovcol tmm1, zmm2, 5
+0x62,0xf3,0xfd,0x48,0x2f,0xca,0x05
+
+# ATT: tilemovcol $15, %zmm31, %tmm7
+# INTEL: tilemovcol tmm7, zmm31, 15
+0x62,0x93,0xfd,0x48,0x2f,0xff,0x0f
+
+# ATT: tilemovcol %ecx, %zmm2, %tmm1
+# INTEL: tilemovcol tmm1, zmm2, ecx
+0x62,0xf2,0xf5,0x48,0x4b,0xca
+
+# ATT: tilemovcol %r14d, %zmm15, %tmm7
+# INTEL: tilemovcol tmm7, zmm15, r14d
+0x62,0xd2,0x8d,0x48,0x4b,0xff
+
+# ATT: tilemovcol %edx, %zmm16, %tmm1
+# INTEL: tilemovcol tmm1, zmm16, edx
+0x62,0xb2,0xed,0x48,0x4b,0xc8
+
+# ATT: tilemovcol %edx, %zmm31, %tmm1
+# INTEL: tilemovcol tmm1, zmm31, edx
+0x62,0x92,0xed,0x48,0x4b,0xcf
+
+## TILEMOVROW write - ZMM into a tile row (ACE only, EVEX.W1)
+
+# ATT: tilemovrow $0, %zmm0, %tmm0
+# INTEL: tilemovrow tmm0, zmm0, 0
+0x62,0xf3,0xfd,0x48,0x07,0xc0,0x00
+
+# ATT: tilemovrow $3, %zmm2, %tmm1
+# INTEL: tilemovrow tmm1, zmm2, 3
+0x62,0xf3,0xfd,0x48,0x07,0xca,0x03
+
+# ATT: tilemovrow $15, %zmm31, %tmm7
+# INTEL: tilemovrow tmm7, zmm31, 15
+0x62,0x93,0xfd,0x48,0x07,0xff,0x0f
+
+# ATT: tilemovrow %edx, %zmm3, %tmm2
+# INTEL: tilemovrow tmm2, zmm3, edx
+0x62,0xf2,0xed,0x48,0x4a,0xd3
+
+# ATT: tilemovrow %r14d, %zmm15, %tmm7
+# INTEL: tilemovrow tmm7, zmm15, r14d
+0x62,0xd2,0x8d,0x48,0x4a,0xff
+
+# ATT: tilemovrow %edx, %zmm16, %tmm1
+# INTEL: tilemovrow tmm1, zmm16, edx
+0x62,0xb2,0xed,0x48,0x4a,0xc8
+
+# ATT: tilemovrow %edx, %zmm31, %tmm1
+# INTEL: tilemovrow tmm1, zmm31, edx
+0x62,0x92,0xed,0x48,0x4a,0xcf
+
+## TILEMOVROW read - tile row into a ZMM (EVEX.W0, same opcode as write)
+
+# ATT: tilemovrow $0, %tmm0, %zmm0
+# INTEL: tilemovrow zmm0, tmm0, 0
+0x62,0xf3,0x7d,0x48,0x07,0xc0,0x00
+
+# ATT: tilemovrow $3, %tmm1, %zmm2
+# INTEL: tilemovrow zmm2, tmm1, 3
+0x62,0xf3,0x7d,0x48,0x07,0xd1,0x03
+
+# ATT: tilemovrow $15, %tmm7, %zmm31
+# INTEL: tilemovrow zmm31, tmm7, 15
+0x62,0x63,0x7d,0x48,0x07,0xff,0x0f
+
+# ATT: tilemovrow %edx, %tmm2, %zmm3
+# INTEL: tilemovrow zmm3, tmm2, edx
+0x62,0xf2,0x6d,0x48,0x4a,0xda
+
+# ATT: tilemovrow %r14d, %tmm7, %zmm31
+# INTEL: tilemovrow zmm31, tmm7, r14d
+0x62,0x62,0x0d,0x48,0x4a,0xff
+
+## TOP2BF16PS - rank-2 BF16 outer product
+
+# ATT: top2bf16ps %zmm3, %zmm2, %tmm1
+# INTEL: top2bf16ps tmm1, zmm2, zmm3
+0x62,0xf2,0x66,0x48,0x5c,0xca
+
+# ATT: top2bf16ps %zmm31, %zmm30, %tmm7
+# INTEL: top2bf16ps tmm7, zmm30, zmm31
+0x62,0x92,0x06,0x40,0x5c,0xfe
+
+## TOP4 - rank-4 INT8 outer products
+
+# ATT: top4buud %zmm3, %zmm2, %tmm1
+# INTEL: top4buud tmm1, zmm2, zmm3
+0x62,0xf2,0x64,0x48,0x5e,0xca
+
+# ATT: top4buud %zmm31, %zmm30, %tmm7
+# INTEL: top4buud tmm7, zmm30, zmm31
+0x62,0x92,0x04,0x40,0x5e,0xfe
+
+# ATT: top4busd %zmm3, %zmm2, %tmm1
+# INTEL: top4busd tmm1, zmm2, zmm3
+0x62,0xf2,0x65,0x48,0x5e,0xca
+
+# ATT: top4busd %zmm31, %zmm30, %tmm7
+# INTEL: top4busd tmm7, zmm30, zmm31
+0x62,0x92,0x05,0x40,0x5e,0xfe
+
+# ATT: top4bssd %zmm3, %zmm2, %tmm1
+# INTEL: top4bssd tmm1, zmm2, zmm3
+0x62,0xf2,0x67,0x48,0x5e,0xca
+
+# ATT: top4bssd %zmm31, %zmm30, %tmm7
+# INTEL: top4bssd tmm7, zmm30, zmm31
+0x62,0x92,0x07,0x40,0x5e,0xfe
+
+# ATT: top4bsud %zmm3, %zmm2, %tmm1
+# INTEL: top4bsud tmm1, zmm2, zmm3
+0x62,0xf2,0x66,0x48,0x5e,0xca
+
+# ATT: top4bsud %zmm31, %zmm30, %tmm7
+# INTEL: top4bsud tmm7, zmm30, zmm31
+0x62,0x92,0x06,0x40,0x5e,0xfe
+
+## TOP4MX - rank-4 MX outer products, BSR scaling mode in imm8
+
+# ATT: top4mxbf8ps $0, %zmm0, %zmm0, %tmm0
+# INTEL: top4mxbf8ps tmm0, zmm0, zmm0, 0
+0x62,0xf3,0x7c,0x48,0x8d,0xc0,0x00
+
+# ATT: top4mxbf8ps $7, %zmm3, %zmm2, %tmm1
+# INTEL: top4mxbf8ps tmm1, zmm2, zmm3, 7
+0x62,0xf3,0x64,0x48,0x8d,0xca,0x07
+
+# ATT: top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
+# INTEL: top4mxhf8ps tmm1, zmm2, zmm3, 7
+0x62,0xf3,0x65,0x48,0x8d,0xca,0x07
+
+# ATT: top4mxbhf8ps $3, %zmm5, %zmm4, %tmm2
+# INTEL: top4mxbhf8ps tmm2, zmm4, zmm5, 3
+0x62,0xf3,0x57,0x48,0x8d,0xd4,0x03
+
+# ATT: top4mxhbf8ps $1, %zmm7, %zmm6, %tmm3
+# INTEL: top4mxhbf8ps tmm3, zmm6, zmm7, 1
+0x62,0xf3,0x46,0x48,0x8d,0xde,0x01
+
+# ATT: top4mxbssps $15, %zmm9, %zmm8, %tmm4
+# INTEL: top4mxbssps tmm4, zmm8, zmm9, 15
+0x62,0xd3,0x37,0x48,0x8f,0xe0,0x0f
+
+# ATT: top4mxbssps $255, %zmm31, %zmm30, %tmm7
+# INTEL: top4mxbssps tmm7, zmm30, zmm31, 255
+0x62,0x93,0x07,0x40,0x8f,0xfe,0xff
diff --git a/llvm/test/MC/X86/ACE/ace-att.s b/llvm/test/MC/X86/ACEV1/acev1-att.s
similarity index 63%
rename from llvm/test/MC/X86/ACE/ace-att.s
rename to llvm/test/MC/X86/ACEV1/acev1-att.s
index cf43cb899bb2c..a6f13e2c79ab3 100644
--- a/llvm/test/MC/X86/ACE/ace-att.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-att.s
@@ -12,6 +12,14 @@
// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
bsrmovf (%rax), %zmm1, %bsr0
+// CHECK: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
+ bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+
+// CHECK: bsrmovf 268435456(%rbp,%r14,8), %zmm1, %bsr0
+// CHECK: encoding: [0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10]
+ bsrmovf 268435456(%rbp,%r14,8), %zmm1, %bsr0
+
// CHECK: bsrmovh %zmm1, %bsr0
// CHECK: encoding: [0x62,0xf6,0xff,0x48,0x95,0xc1]
bsrmovh %zmm1, %bsr0
@@ -88,22 +96,50 @@
// CHECK: encoding: [0x62,0xf3,0x65,0x48,0x8d,0xca,0x07]
top4mxhf8ps $7, %zmm3, %zmm2, %tmm1
-// CHECK: top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
-// CHECK: encoding: [0x62,0xf3,0x5f,0x48,0x8d,0xd5,0x03]
- top4mxbhf8ps $3, %zmm4, %zmm5, %tmm2
+// CHECK: top4mxbhf8ps $3, %zmm5, %zmm4, %tmm2
+// CHECK: encoding: [0x62,0xf3,0x57,0x48,0x8d,0xd4,0x03]
+ top4mxbhf8ps $3, %zmm5, %zmm4, %tmm2
-// CHECK: top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
-// CHECK: encoding: [0x62,0xf3,0x4e,0x48,0x8d,0xdf,0x01]
- top4mxhbf8ps $1, %zmm6, %zmm7, %tmm3
+// CHECK: top4mxhbf8ps $1, %zmm7, %zmm6, %tmm3
+// CHECK: encoding: [0x62,0xf3,0x46,0x48,0x8d,0xde,0x01]
+ top4mxhbf8ps $1, %zmm7, %zmm6, %tmm3
-// CHECK: top4mxbssps $15, %zmm8, %zmm9, %tmm4
-// CHECK: encoding: [0x62,0xd3,0x3f,0x48,0x8f,0xe1,0x0f]
- top4mxbssps $15, %zmm8, %zmm9, %tmm4
+// CHECK: top4mxbssps $15, %zmm9, %zmm8, %tmm4
+// CHECK: encoding: [0x62,0xd3,0x37,0x48,0x8f,0xe0,0x0f]
+ top4mxbssps $15, %zmm9, %zmm8, %tmm4
// CHECK: top4buud %zmm31, %zmm30, %tmm7
-// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+// CHECK: encoding: [0x62,0x92,0x04,0x40,0x5e,0xfe]
top4buud %zmm31, %zmm30, %tmm7
-// CHECK: bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
-// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
- bsrmovf 64(%rdx,%rax,4), %zmm10, %bsr0
+// CHECK: tilemovrow %edx, %zmm31, %tmm1
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4a,0xcf]
+ tilemovrow %edx, %zmm31, %tmm1
+
+// CHECK: tilemovrow %edx, %zmm16, %tmm1
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4a,0xc8]
+ tilemovrow %edx, %zmm16, %tmm1
+
+// CHECK: tilemovcol %edx, %zmm31, %tmm1
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4b,0xcf]
+ tilemovcol %edx, %zmm31, %tmm1
+
+// CHECK: tilemovcol %edx, %zmm16, %tmm1
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4b,0xc8]
+ tilemovcol %edx, %zmm16, %tmm1
+
+// CHECK: top2bf16ps %zmm31, %zmm30, %tmm7
+// CHECK: encoding: [0x62,0x92,0x06,0x40,0x5c,0xfe]
+ top2bf16ps %zmm31, %zmm30, %tmm7
+
+// CHECK: top4busd %zmm15, %zmm31, %tmm1
+// CHECK: encoding: [0x62,0x92,0x05,0x48,0x5e,0xcf]
+ top4busd %zmm15, %zmm31, %tmm1
+
+// CHECK: top4bssd %zmm31, %zmm15, %tmm1
+// CHECK: encoding: [0x62,0xd2,0x07,0x40,0x5e,0xcf]
+ top4bssd %zmm31, %zmm15, %tmm1
+
+// CHECK: top4bsud %zmm0, %zmm16, %tmm1
+// CHECK: encoding: [0x62,0xb2,0x7e,0x48,0x5e,0xc8]
+ top4bsud %zmm0, %zmm16, %tmm1
diff --git a/llvm/test/MC/X86/ACE/ace-error.s b/llvm/test/MC/X86/ACEV1/acev1-error.s
similarity index 100%
rename from llvm/test/MC/X86/ACE/ace-error.s
rename to llvm/test/MC/X86/ACEV1/acev1-error.s
diff --git a/llvm/test/MC/X86/ACE/ace-intel.s b/llvm/test/MC/X86/ACEV1/acev1-intel.s
similarity index 74%
rename from llvm/test/MC/X86/ACE/ace-intel.s
rename to llvm/test/MC/X86/ACEV1/acev1-intel.s
index 81b5d8e0f7e24..daab4536709ec 100644
--- a/llvm/test/MC/X86/ACE/ace-intel.s
+++ b/llvm/test/MC/X86/ACEV1/acev1-intel.s
@@ -12,6 +12,10 @@
// CHECK: encoding: [0x62,0xf6,0xf4,0x48,0x95,0x00]
bsrmovf bsr0, zmm1, zmmword ptr [rax]
+// CHECK: bsrmovf bsr0, zmm10, zmmword ptr [rdx + 4*rax + 64]
+// CHECK: encoding: [0x62,0xf6,0xac,0x48,0x95,0x44,0x82,0x01]
+ bsrmovf bsr0, zmm10, zmmword ptr [rdx + 4*rax + 64]
+
// CHECK: bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
// CHECK: encoding: [0x62,0xb6,0xf4,0x48,0x95,0x84,0xf5,0x00,0x00,0x00,0x10]
bsrmovf bsr0, zmm1, zmmword ptr [rbp + 8*r14 + 268435456]
@@ -105,5 +109,37 @@
top4mxbssps tmm4, zmm8, zmm9, 15
// CHECK: top4buud tmm7, zmm30, zmm31
-// CHECK: encoding: [0x62,0xd2,0x04,0x40,0x5e,0xfe]
+// CHECK: encoding: [0x62,0x92,0x04,0x40,0x5e,0xfe]
top4buud tmm7, zmm30, zmm31
+
+// CHECK: tilemovrow tmm1, zmm31, edx
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4a,0xcf]
+ tilemovrow tmm1, zmm31, edx
+
+// CHECK: tilemovrow tmm1, zmm16, edx
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4a,0xc8]
+ tilemovrow tmm1, zmm16, edx
+
+// CHECK: tilemovcol tmm1, zmm31, edx
+// CHECK: encoding: [0x62,0x92,0xed,0x48,0x4b,0xcf]
+ tilemovcol tmm1, zmm31, edx
+
+// CHECK: tilemovcol tmm1, zmm16, edx
+// CHECK: encoding: [0x62,0xb2,0xed,0x48,0x4b,0xc8]
+ tilemovcol tmm1, zmm16, edx
+
+// CHECK: top2bf16ps tmm7, zmm30, zmm31
+// CHECK: encoding: [0x62,0x92,0x06,0x40,0x5c,0xfe]
+ top2bf16ps tmm7, zmm30, zmm31
+
+// CHECK: top4busd tmm1, zmm31, zmm15
+// CHECK: encoding: [0x62,0x92,0x05,0x48,0x5e,0xcf]
+ top4busd tmm1, zmm31, zmm15
+
+// CHECK: top4bssd tmm1, zmm15, zmm31
+// CHECK: encoding: [0x62,0xd2,0x07,0x40,0x5e,0xcf]
+ top4bssd tmm1, zmm15, zmm31
+
+// CHECK: top4bsud tmm1, zmm16, zmm0
+// CHECK: encoding: [0x62,0xb2,0x7e,0x48,0x5e,0xc8]
+ top4bsud tmm1, zmm16, zmm0
More information about the cfe-commits
mailing list