[llvm] 28a5690 - [X86][AMX-AVX512] Add *i intrinsics for immediate variants (#173545)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 31 05:56:20 PST 2025
Author: Phoebe Wang
Date: 2025-12-31T21:56:15+08:00
New Revision: 28a5690003781874e022763bd02e987d1f0af34c
URL: https://github.com/llvm/llvm-project/commit/28a5690003781874e022763bd02e987d1f0af34c
DIFF: https://github.com/llvm/llvm-project/commit/28a5690003781874e022763bd02e987d1f0af34c.diff
LOG: [X86][AMX-AVX512] Add *i intrinsics for immediate variants (#173545)
The immediate variants use the low 6-bit as row index, while register
variants use low 16-bit. We cannot select the immediate variants using
the same intrinsic. So let's add new intrinsics for them.
Added:
clang/test/CodeGen/X86/amx_amx512_errors.c
Modified:
clang/include/clang/Basic/BuiltinsX86_64.td
clang/lib/Headers/amxavx512intrin.h
clang/lib/Sema/SemaX86.cpp
clang/test/CodeGen/X86/amxavx512-builtins.c
llvm/include/llvm/IR/IntrinsicsX86.td
llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
llvm/lib/Target/X86/AsmParser/X86AsmParserCommon.h
llvm/lib/Target/X86/AsmParser/X86Operand.h
llvm/lib/Target/X86/X86InstrAMX.td
llvm/lib/Target/X86/X86InstrOperands.td
llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll
Removed:
################################################################################
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.td b/clang/include/clang/Basic/BuiltinsX86_64.td
index 2e68a5a016ef0..2bd62bd5e2663 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.td
+++ b/clang/include/clang/Basic/BuiltinsX86_64.td
@@ -304,6 +304,12 @@ let Features = "amx-avx512,avx10.2", Attributes = [NoThrow, RequiredVectorWidth<
def tcvtrowps2phh : X86Builtin<"_Vector<32, _Float16>(_Constant unsigned char, unsigned int)">;
def tcvtrowps2phl : X86Builtin<"_Vector<32, _Float16>(_Constant unsigned char, unsigned int)">;
def tilemovrow : X86Builtin<"_Vector<16, int>(_Constant unsigned char, unsigned int)">;
+ def tcvtrowd2psi : X86Builtin<"_Vector<16, float>(_Constant unsigned char, _Constant unsigned int)">;
+ def tcvtrowps2bf16hi : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, _Constant unsigned int)">;
+ def tcvtrowps2bf16li : X86Builtin<"_Vector<32, __bf16>(_Constant unsigned char, _Constant unsigned int)">;
+ def tcvtrowps2phhi : X86Builtin<"_Vector<32, _Float16>(_Constant unsigned char, _Constant unsigned int)">;
+ def tcvtrowps2phli : X86Builtin<"_Vector<32, _Float16>(_Constant unsigned char, _Constant unsigned int)">;
+ def tilemovrowi : X86Builtin<"_Vector<16, int>(_Constant unsigned char, _Constant unsigned int)">;
}
let Features = "amx-fp16", Attributes = [NoThrow] in {
diff --git a/clang/lib/Headers/amxavx512intrin.h b/clang/lib/Headers/amxavx512intrin.h
index 5f21c25b05ad1..18ef721cd1601 100644
--- a/clang/lib/Headers/amxavx512intrin.h
+++ b/clang/lib/Headers/amxavx512intrin.h
@@ -52,6 +52,40 @@
/// The row of the source tile
#define _tile_cvtrowd2ps(tsrc, row) __builtin_ia32_tcvtrowd2ps(tsrc, row)
+/// Moves a row from a tile register to a zmm destination register, converting
+/// the int32 source elements to fp32. The row of the tile is selected by a
+/// 8b immediate value.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m512i _tile_cvtrowd2psi(__tile tsrc, const unsigned int imm8);
+/// \endcode
+///
+/// \code{.operation}
+/// VL := 512
+/// VL_bytes := VL >> 3
+/// row_index := imm8 & 0x3f
+/// row_chunk := (imm8 >> 6) * VL_bytes
+/// FOR i := 0 TO (VL_bytes / 4) - 1
+/// IF i + row_chunk / 4 >= tsrc.colsb / 4
+/// dst.dword[i] := 0
+/// ELSE
+/// dst.f32[i] := CONVERT_INT32_TO_FP32(tsrc.row[row_index].dword[row_chunk/4+i], RNE)
+/// FI
+/// ENDFOR
+/// dst[MAX_VL-1:VL] := 0
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCVTROWD2PS instruction.
+///
+/// \param tsrc
+/// The source tile. Max size is 1024 Bytes.
+/// \param imm8
+/// The row of the source tile
+#define _tile_cvtrowd2psi(tsrc, imm8) __builtin_ia32_tcvtrowd2psi(tsrc, imm8)
+
/// Moves a row from a tile register to a zmm destination register, converting
/// the fp32 source elements to bf16. It places the resulting bf16 elements
/// in the high 16 bits within each dword. The row of the tile is selected
@@ -89,6 +123,43 @@
#define _tile_cvtrowps2bf16h(tsrc, row) \
__builtin_ia32_tcvtrowps2bf16h(tsrc, row)
+/// Moves a row from a tile register to a zmm destination register, converting
+/// the fp32 source elements to bf16. It places the resulting bf16 elements
+/// in the high 16 bits within each dword. The row of the tile is selected
+/// by a 8b immediate value.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m512i _tile_cvtrowps2bf16hi(__tile tsrc, const unsigned int imm8);
+/// \endcode
+///
+/// \code{.operation}
+/// VL := 512
+/// VL_bytes := VL >> 3
+/// row_index := imm8 & 0x3f
+/// row_chunk := (imm8 >> 6) * VL_bytes
+/// FOR i := 0 TO (VL_bytes / 4) - 1
+/// IF i + row_chunk / 4 >= tsrc.colsb / 4
+/// dst.dword[i] := 0
+/// ELSE
+/// dst.word[2*i+0] := 0
+/// dst.bf16[2*i+1] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
+/// FI
+/// ENDFOR
+/// dst[MAX_VL-1:VL] := 0
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCVTROWPS2BF16H instruction.
+///
+/// \param tsrc
+/// The source tile. Max size is 1024 Bytes.
+/// \param imm8
+/// The the row of the source tile.
+#define _tile_cvtrowps2bf16hi(tsrc, imm8) \
+ __builtin_ia32_tcvtrowps2bf16hi(tsrc, imm8)
+
/// Moves a row from a tile register to a zmm destination register, converting
/// the fp32 source elements to bf16. It places the resulting bf16 elements
/// in the low 16 bits within each dword. The row of the tile is selected
@@ -126,6 +197,43 @@
#define _tile_cvtrowps2bf16l(tsrc, row) \
__builtin_ia32_tcvtrowps2bf16l(tsrc, row)
+/// Moves a row from a tile register to a zmm destination register, converting
+/// the fp32 source elements to bf16. It places the resulting bf16 elements
+/// in the low 16 bits within each dword. The row of the tile is selected
+/// by a 8b immediate value.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m512i _tile_cvtrowps2bf16li(__tile tsrc, const unsigned int imm8);
+/// \endcode
+///
+/// \code{.operation}
+/// VL := 512
+/// VL_bytes := VL >> 3
+/// row_index := imm8 & 0x3f
+/// row_chunk := (imm8 >> 6) * VL_bytes
+/// FOR i := 0 TO (VL_bytes / 4) - 1
+/// IF i + row_chunk / 4 >= tsrc.colsb / 4
+/// dst.dword[i] := 0
+/// ELSE
+/// dst.word[2*i+1] := 0
+/// dst.bf16[2*i+0] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
+/// FI
+/// ENDFOR
+/// dst[MAX_VL-1:VL] := 0
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCVTROWPS2BF16L instruction.
+///
+/// \param tsrc
+/// The source tile. Max size is 1024 Bytes.
+/// \param imm8
+/// The the row of the source tile.
+#define _tile_cvtrowps2bf16li(tsrc, imm8) \
+ __builtin_ia32_tcvtrowps2bf16li(tsrc, imm8)
+
/// Moves a row from a tile register to a zmm destination register, converting
/// the fp32 source elements to fp16. It places the resulting fp16 elements
/// in the high 16 bits within each dword. The row of the tile is selected
@@ -162,6 +270,43 @@
/// The the row of the source tile.
#define _tile_cvtrowps2phh(tsrc, row) __builtin_ia32_tcvtrowps2phh(tsrc, row)
+/// Moves a row from a tile register to a zmm destination register, converting
+/// the fp32 source elements to fp16. It places the resulting fp16 elements
+/// in the high 16 bits within each dword. The row of the tile is selected
+/// by a 8b immediate value.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m512i _tile_cvtrowps2phhi(__tile tsrc, constunsigned int imm8);
+/// \endcode
+///
+/// \code{.operation}
+/// VL := 512
+/// VL_bytes := VL >> 3
+/// row_index := imm8 & 0x3f
+/// row_chunk := (imm8 >> 6) * VL_bytes
+/// FOR i := 0 TO (VL_bytes / 4) - 1
+/// IF i + row_chunk / 4 >= tsrc.colsb / 4
+/// dst.dword[i] := 0
+/// ELSE
+/// dst.word[2*i+0] := 0
+/// dst.fp16[2*i+1] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
+/// FI
+/// ENDFOR
+/// dst[MAX_VL-1:VL] := 0
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCVTROWPS2PHH instruction.
+///
+/// \param tsrc
+/// The source tile. Max size is 1024 Bytes.
+/// \param imm8
+/// The the row of the source tile.
+#define _tile_cvtrowps2phhi(tsrc, imm8) \
+ __builtin_ia32_tcvtrowps2phhi(tsrc, imm8)
+
/// Moves a row from a tile register to a zmm destination register, converting
/// the fp32 source elements to fp16. It places the resulting fp16 elements
/// in the low 16 bits within each dword. The row of the tile is selected
@@ -198,6 +343,43 @@
/// The the row of the source tile.
#define _tile_cvtrowps2phl(tsrc, row) __builtin_ia32_tcvtrowps2phl(tsrc, row)
+/// Moves a row from a tile register to a zmm destination register, converting
+/// the fp32 source elements to fp16. It places the resulting fp16 elements
+/// in the low 16 bits within each dword. The row of the tile is selected
+/// by a 8b immediate value.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// __m512i _tile_cvtrowps2phli(__tile tsrc, const unsigned int imm8);
+/// \endcode
+///
+/// \code{.operation}
+/// VL := 512
+/// VL_bytes := VL >> 3
+/// row_index := imm8 & 0x3f
+/// row_chunk := (imm8 >> 6) * VL_bytes
+/// FOR i := 0 TO (VL_bytes / 4) - 1
+/// IF i + row_chunk / 4 >= tsrc.colsb / 4
+/// dst.dword[i] := 0
+/// ELSE
+/// dst.word[2*i+1] := 0
+/// dst.fp16[2*i+0] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE)
+/// FI
+/// ENDFOR
+/// dst[MAX_VL-1:VL] := 0
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCVTROWPS2PHL instruction.
+///
+/// \param tsrc
+/// The source tile. Max size is 1024 Bytes.
+/// \param imm8
+/// The the row of the source tile.
+#define _tile_cvtrowps2phli(tsrc, imm8) \
+ __builtin_ia32_tcvtrowps2phli(tsrc, imm8)
+
/// Move one row of a tile data to a v16f32 data.
/// The row of the tile is selected by a 32b GPR.
///
@@ -230,6 +412,38 @@
/// \endcode
#define _tile_movrow(a, b) ((__m512i)__builtin_ia32_tilemovrow(a, b))
+/// Move one row of a tile data to a v16f32 data.
+/// The row of the tile is selected by a 8b immediate value.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m512 _tile_movrowi(__tile a, const unsigned b);
+/// \endcode
+///
+/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction.
+///
+/// \param a
+/// The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+/// The 2nd source r32. Size is 4 Bytes.
+/// \returns
+/// The destination v16f32 data. Size is 64 Bytes.
+///
+/// \code{.operation}
+/// VL := 512
+/// VL_bytes := VL>>3
+/// row_index := b&0x3f
+/// row_chunk := (b>>6) * VL_bytes
+/// FOR i := 0 TO (VL_bytes-1)
+/// IF (row_chunk + i >= a.colsb)
+/// dst.byte[i] := 0
+/// ELSE
+/// dst.byte[i] := a.row[row_index].byte[row_chunk+i]
+/// ENDFOR
+/// \endcode
+#define _tile_movrowi(a, b) ((__m512i)__builtin_ia32_tilemovrowi(a, b))
+
/// This is internal intrinsic. C/C++ user should avoid calling it directly.
static __inline__ __m512 __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowd2ps_internal(
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index 2f61bdd9a6540..39799e2203727 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -510,6 +510,15 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) {
case X86::BI__builtin_ia32_tdphf8ps:
case X86::BI__builtin_ia32_tmmultf32ps:
return CheckBuiltinTileRangeAndDuplicate(TheCall, {0, 1, 2});
+ case X86::BI__builtin_ia32_tcvtrowps2bf16hi:
+ case X86::BI__builtin_ia32_tcvtrowps2bf16li:
+ case X86::BI__builtin_ia32_tcvtrowps2phhi:
+ case X86::BI__builtin_ia32_tcvtrowps2phli:
+ case X86::BI__builtin_ia32_tcvtrowd2psi:
+ case X86::BI__builtin_ia32_tilemovrowi:
+ return CheckBuiltinTileArgumentsRange(TheCall, 0) ||
+ SemaRef.BuiltinConstantArgRange(TheCall, 1, 0, 255,
+ /*RangeIsError=*/false);
}
}
static bool isX86_32Builtin(unsigned BuiltinID) {
diff --git a/clang/test/CodeGen/X86/amx_amx512_errors.c b/clang/test/CodeGen/X86/amx_amx512_errors.c
new file mode 100644
index 0000000000000..5ea3d14b93f67
--- /dev/null
+++ b/clang/test/CodeGen/X86/amx_amx512_errors.c
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown \
+// RUN: -target-feature +amx-avx512 -target-feature +avx10.2 -verify
+
+#include <immintrin.h>
+#include <stddef.h>
+
+void test_tile_mmultf32ps() {
+ _tile_cvtrowd2psi(16, 2); // expected-error {{argument value 16 is outside the valid range [0, 7]}}
+ _tile_cvtrowd2psi(1, 260); // expected-error {{argument value 260 is outside the valid range [0, 255]}}
+}
+
diff --git a/clang/test/CodeGen/X86/amxavx512-builtins.c b/clang/test/CodeGen/X86/amxavx512-builtins.c
index dada53c2d6f83..073adab349764 100644
--- a/clang/test/CodeGen/X86/amxavx512-builtins.c
+++ b/clang/test/CodeGen/X86/amxavx512-builtins.c
@@ -39,3 +39,39 @@ __m512i test_tile_movrow(unsigned int A) {
// CHECK: %1 = call <16 x i32> @llvm.x86.tilemovrow(i8 1, i32 %{{.*}})
return _tile_movrow(1, A);
}
+
+__m512 test_tile_cvtrowd2psi(void) {
+ // CHECK-LABEL: @test_tile_cvtrowd2psi(
+ // CHECK: call <16 x float> @llvm.x86.tcvtrowd2psi(i8 1, i32 2)
+ return _tile_cvtrowd2psi(1, 2);
+}
+
+__m512bh test_tile_cvtrowps2bf16hi(void) {
+ // CHECK-LABEL: @test_tile_cvtrowps2bf16hi(
+ // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16hi(i8 1, i32 2)
+ return _tile_cvtrowps2bf16hi(1, 2);
+}
+
+__m512bh test_tile_cvtrowps2bf16li(void) {
+ // CHECK-LABEL: @test_tile_cvtrowps2bf16li(
+ // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2bf16li(i8 1, i32 2)
+ return _tile_cvtrowps2bf16li(1, 2);
+}
+
+__m512h test_tile_cvtrowps2phhi(void) {
+ // CHECK-LABEL: @test_tile_cvtrowps2phhi(
+ // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phhi(i8 1, i32 2)
+ return _tile_cvtrowps2phhi(1, 2);
+}
+
+__m512h test_tile_cvtrowps2phli(void) {
+ // CHECK-LABEL: @test_tile_cvtrowps2phli(
+ // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phli(i8 1, i32 2)
+ return _tile_cvtrowps2phli(1, 2);
+}
+
+__m512i test_tile_movrowi(void) {
+ // CHECK-LABEL: @test_tile_movrowi
+ // CHECK: %0 = call <16 x i32> @llvm.x86.tilemovrowi(i8 1, i32 2)
+ return _tile_movrowi(1, 2);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index e36187ea54d6f..b75a0485d6263 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5526,6 +5526,24 @@ let TargetPrefix = "x86" in {
def int_x86_tilemovrow : ClangBuiltin<"__builtin_ia32_tilemovrow">,
Intrinsic<[llvm_v16i32_ty], [llvm_i8_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>]>;
+ def int_x86_tcvtrowd2psi : ClangBuiltin<"__builtin_ia32_tcvtrowd2psi">,
+ Intrinsic<[llvm_v16f32_ty], [llvm_i8_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
+ def int_x86_tcvtrowps2bf16hi : ClangBuiltin<"__builtin_ia32_tcvtrowps2bf16hi">,
+ Intrinsic<[llvm_v32bf16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
+ def int_x86_tcvtrowps2bf16li : ClangBuiltin<"__builtin_ia32_tcvtrowps2bf16li">,
+ Intrinsic<[llvm_v32bf16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
+ def int_x86_tcvtrowps2phhi : ClangBuiltin<"__builtin_ia32_tcvtrowps2phhi">,
+ Intrinsic<[llvm_v32f16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
+ def int_x86_tcvtrowps2phli : ClangBuiltin<"__builtin_ia32_tcvtrowps2phli">,
+ Intrinsic<[llvm_v32f16_ty], [llvm_i8_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
+ def int_x86_tilemovrowi : ClangBuiltin<"__builtin_ia32_tilemovrowi">,
+ Intrinsic<[llvm_v16i32_ty], [llvm_i8_ty, llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>]>;
// AMX - internal intrinsics
def int_x86_ldtilecfg_internal :
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 90fc66e90cacc..d1dda4debe229 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -4374,6 +4374,13 @@ bool X86AsmParser::matchAndEmitATTInstruction(
return Error(ErrorLoc, "immediate must be an integer in range [0, 15]",
EmptyRange, MatchingInlineAsm);
}
+ case Match_InvalidImmUnsignedi6: {
+ SMLoc ErrorLoc = ((X86Operand &)*Operands[ErrorInfo]).getStartLoc();
+ if (ErrorLoc == SMLoc())
+ ErrorLoc = IDLoc;
+ return Error(ErrorLoc, "immediate must be an integer in range [0, 63]",
+ EmptyRange, MatchingInlineAsm);
+ }
case Match_MissingFeature:
return ErrorMissingFeature(IDLoc, MissingFeatures, MatchingInlineAsm);
case Match_InvalidOperand:
@@ -4738,6 +4745,14 @@ bool X86AsmParser::matchAndEmitIntelInstruction(
EmptyRange, MatchingInlineAsm);
}
+ if (llvm::count(Match, Match_InvalidImmUnsignedi6) == 1) {
+ SMLoc ErrorLoc = ((X86Operand &)*Operands[ErrorInfo]).getStartLoc();
+ if (ErrorLoc == SMLoc())
+ ErrorLoc = IDLoc;
+ return Error(ErrorLoc, "immediate must be an integer in range [0, 63]",
+ EmptyRange, MatchingInlineAsm);
+ }
+
// If all of these were an outright failure, report it in a useless way.
return Error(IDLoc, "unknown instruction mnemonic", EmptyRange,
MatchingInlineAsm);
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParserCommon.h b/llvm/lib/Target/X86/AsmParser/X86AsmParserCommon.h
index e9be28ca77b05..b39cb33604ea5 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParserCommon.h
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParserCommon.h
@@ -39,6 +39,10 @@ inline bool isImmUnsignedi4Value(uint64_t Value) {
return isUInt<4>(Value);
}
+inline bool isImmUnsignedi6Value(uint64_t Value) {
+ return isUInt<6>(Value);
+}
+
} // End of namespace llvm
#endif
diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h
index a31a7c3b4bd0e..acc5caa9c436f 100644
--- a/llvm/lib/Target/X86/AsmParser/X86Operand.h
+++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h
@@ -288,6 +288,15 @@ struct X86Operand final : public MCParsedAsmOperand {
return isImmUnsignedi4Value(CE->getValue());
}
+ bool isImmUnsignedi6() const {
+ if (!isImm()) return false;
+ // If this isn't a constant expr, reject it. The immediate byte is shared
+ // with a register encoding. We can't have it affected by a relocation.
+ const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+ if (!CE) return false;
+ return isImmUnsignedi6Value(CE->getValue());
+ }
+
bool isImmUnsignedi8() const {
if (!isImm()) return false;
// If this isn't a constant expr, just assume it fits and let relaxation
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
index 6b8b8f720ddd7..7b930eac1573f 100644
--- a/llvm/lib/Target/X86/X86InstrAMX.td
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -388,13 +388,13 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
let SchedRW = [WriteSystem] in {
let usesCustomInserter = 1 in {
def PTCVTROWD2PSrri : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
- [(set VR512:$dst, (int_x86_tcvtrowd2ps timm:$src1, imm:$src2))]>;
+ [(set VR512:$dst, (int_x86_tcvtrowd2psi timm:$src1, timm:$src2))]>;
def PTCVTROWD2PSrre : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2),
[(set VR512:$dst, (int_x86_tcvtrowd2ps timm:$src1, GR32:$src2))]>;
}
def PTCVTROWD2PSrriV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
[(set VR512: $dst,
(int_x86_tcvtrowd2ps_internal GR16:$src1, GR16:$src2,
TILE:$src3, imm:$src4))]>;
@@ -404,7 +404,7 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
(int_x86_tcvtrowd2ps_internal GR16:$src1, GR16:$src2,
TILE:$src3, GR32:$src4))]>;
def PTCVTROWPS2BF16HrriV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
[(set VR512: $dst,
(int_x86_tcvtrowps2bf16h_internal GR16:$src1, GR16:$src2,
TILE:$src3, imm:$src4))]>;
@@ -414,7 +414,7 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
(int_x86_tcvtrowps2bf16h_internal GR16:$src1, GR16:$src2,
TILE:$src3, GR32:$src4))]>;
def PTCVTROWPS2BF16LrriV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
[(set VR512: $dst,
(int_x86_tcvtrowps2bf16l_internal GR16:$src1, GR16:$src2,
TILE:$src3, imm:$src4))]>;
@@ -424,7 +424,7 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
(int_x86_tcvtrowps2bf16l_internal GR16:$src1, GR16:$src2,
TILE:$src3, GR32:$src4))]>;
def PTCVTROWPS2PHHrriV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
[(set VR512: $dst,
(int_x86_tcvtrowps2phh_internal GR16:$src1, GR16:$src2,
TILE:$src3, imm:$src4))]>;
@@ -434,7 +434,7 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
(int_x86_tcvtrowps2phh_internal GR16:$src1, GR16:$src2,
TILE:$src3, GR32:$src4))]>;
def PTCVTROWPS2PHLrriV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
[(set VR512: $dst,
(int_x86_tcvtrowps2phl_internal GR16:$src1, GR16:$src2,
TILE:$src3, imm:$src4))]>;
@@ -465,7 +465,7 @@ multiclass AMXAVX512_BASE<bits<8> Opcode1, bits<8> Opcode2, string Opstr,
(!cast<Intrinsic>("int_x86_"#Opstr) timm:$src1, GR32:$src2))]>;
def "P"#NAME#"rri" : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
[(set VR512:$dst,
- (!cast<Intrinsic>("int_x86_"#Opstr) timm:$src1, imm:$src2))]>;
+ (!cast<Intrinsic>("int_x86_"#Opstr#"i") timm:$src1, timm:$src2))]>;
}
}
}
@@ -496,13 +496,13 @@ let Predicates = [HasAMXAVX512, HasAVX10_2, In64BitMode] in {
let SchedRW = [WriteSystem] in {
let usesCustomInserter = 1 in {
def PTILEMOVROWrri : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2),
- [(set VR512:$dst, (int_x86_tilemovrow timm:$src1, imm:$src2))]>;
+ [(set VR512:$dst, (int_x86_tilemovrowi timm:$src1, timm:$src2))]>;
def PTILEMOVROWrre : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2),
[(set VR512:$dst, (int_x86_tilemovrow timm:$src1, GR32:$src2))]>;
}
def PTILEMOVROWrriV : PseudoI<(outs VR512:$dst),
- (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4),
+ (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u6imm:$src4),
[(set VR512: $dst,
(int_x86_tilemovrow_internal GR16:$src1, GR16:$src2,
TILE:$src3, imm:$src4))]>;
diff --git a/llvm/lib/Target/X86/X86InstrOperands.td b/llvm/lib/Target/X86/X86InstrOperands.td
index 69ce4f8552609..4373f39c360a8 100644
--- a/llvm/lib/Target/X86/X86InstrOperands.td
+++ b/llvm/lib/Target/X86/X86InstrOperands.td
@@ -415,6 +415,14 @@ def ImmUnsignedi4AsmOperand : AsmOperandClass {
let DiagnosticType = "InvalidImmUnsignedi4";
}
+// 6-bit immediate used by some AMX instructions
+// [0, 0x3F]
+def ImmUnsignedi6AsmOperand : AsmOperandClass {
+ let Name = "ImmUnsignedi6";
+ let RenderMethod = "addImmOperands";
+ let DiagnosticType = "InvalidImmUnsignedi6";
+}
+
// Unsigned immediate used by SSE/AVX instructions
// [0, 0xFF]
// [0xFFFFFFFFFFFFFF80, 0xFFFFFFFFFFFFFFFF]
@@ -475,6 +483,14 @@ def i16u8imm : Operand<i16> {
let OperandType = "OPERAND_IMMEDIATE";
}
+// 32-bit immediate but only 6-bits are significant and they are unsigned.
+// Used by some AMX instructions that use intrinsics.
+def i32u6imm : Operand<i32> {
+ let PrintMethod = "printU8Imm";
+ let ParserMatchClass = ImmUnsignedi6AsmOperand;
+ let OperandType = "OPERAND_IMMEDIATE";
+}
+
// 32-bit immediate but only 8-bits are significant and they are unsigned.
// Used by some SSE/AVX instructions that use intrinsics.
def i32u8imm : Operand<i32> {
diff --git a/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll b/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll
index 41208d6adb300..62cafc5a43c7b 100644
--- a/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll
@@ -15,7 +15,7 @@ define <16 x float> @test_tcvtrowd2psi() {
; CHECK: # %bb.0:
; CHECK-NEXT: tcvtrowd2ps $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x07,0xc1,0x7f]
; CHECK-NEXT: retq # encoding: [0xc3]
- %ret = call <16 x float> @llvm.x86.tcvtrowd2ps(i8 1, i32 127)
+ %ret = call <16 x float> @llvm.x86.tcvtrowd2psi(i8 1, i32 127)
ret <16 x float> %ret
}
declare <16 x float> @llvm.x86.tcvtrowd2ps(i8 %A, i32 %B)
@@ -34,7 +34,7 @@ define <32 x bfloat> @test_tcvtrowps2bf16hi() {
; CHECK: # %bb.0:
; CHECK-NEXT: tcvtrowps2bf16h $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x07,0xc1,0x7f]
; CHECK-NEXT: retq # encoding: [0xc3]
- %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2bf16h(i8 1, i32 127)
+ %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2bf16hi(i8 1, i32 127)
ret <32 x bfloat> %ret
}
declare <32 x bfloat> @llvm.x86.tcvtrowps2bf16h(i8 %A, i32 %B)
@@ -53,7 +53,7 @@ define <32 x bfloat> @test_tcvtrowps2bf16li() {
; CHECK: # %bb.0:
; CHECK-NEXT: tcvtrowps2bf16l $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x77,0xc1,0x7f]
; CHECK-NEXT: retq # encoding: [0xc3]
- %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2bf16l(i8 1, i32 127)
+ %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2bf16li(i8 1, i32 127)
ret <32 x bfloat> %ret
}
declare <32 x bfloat> @llvm.x86.tcvtrowps2bf16l(i8 %A, i32 %B)
@@ -72,7 +72,7 @@ define <32 x half> @test_tcvtrowps2phhi() {
; CHECK: # %bb.0:
; CHECK-NEXT: tcvtrowps2phh $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7c,0x48,0x07,0xc1,0x7f]
; CHECK-NEXT: retq # encoding: [0xc3]
- %ret = call <32 x half> @llvm.x86.tcvtrowps2phh(i8 1, i32 127)
+ %ret = call <32 x half> @llvm.x86.tcvtrowps2phhi(i8 1, i32 127)
ret <32 x half> %ret
}
declare <32 x half> @llvm.x86.tcvtrowps2phh(i8 %A, i32 %B)
@@ -91,7 +91,7 @@ define <32 x half> @test_tcvtrowps2phli() {
; CHECK: # %bb.0:
; CHECK-NEXT: tcvtrowps2phl $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x77,0xc1,0x7f]
; CHECK-NEXT: retq # encoding: [0xc3]
- %ret = call <32 x half> @llvm.x86.tcvtrowps2phl(i8 1, i32 127)
+ %ret = call <32 x half> @llvm.x86.tcvtrowps2phli(i8 1, i32 127)
ret <32 x half> %ret
}
declare <32 x half> @llvm.x86.tcvtrowps2phl(i8 %A, i32 %B)
@@ -110,7 +110,7 @@ define <16 x i32> @test_tilemovrowi() {
; CHECK: # %bb.0:
; CHECK-NEXT: tilemovrow $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x07,0xc1,0x7f]
; CHECK-NEXT: retq # encoding: [0xc3]
- %ret = call <16 x i32> @llvm.x86.tilemovrow(i8 1, i32 127)
+ %ret = call <16 x i32> @llvm.x86.tilemovrowi(i8 1, i32 127)
ret <16 x i32> %ret
}
declare <16 x i32> @llvm.x86.tilemovrow(i8 %A, i32 %B)
More information about the llvm-commits
mailing list