r360018 - Enable intrinsics of AVX512_BF16, which are supported for BFLOAT16 in Cooper Lake
via cfe-commits
cfe-commits at lists.llvm.org
Mon May 6 01:25:11 PDT 2019
Author: luoyuanke
Date: Mon May 6 01:25:11 2019
New Revision: 360018
URL: http://llvm.org/viewvc/llvm-project?rev=360018&view=rev
Log:
Enable intrinsics of AVX512_BF16, which are supported for BFLOAT16 in Cooper Lake
Summary:
1. Enable infrastructure of AVX512_BF16, which is supported for BFLOAT16 in Cooper Lake;
2. Enable intrinsics for VCVTNE2PS2BF16, VCVTNEPS2BF16 and DPBF16PS instructions, which are Vector Neural Network Instructions supporting BFLOAT16 inputs and conversion instructions from IEEE single precision.
For more details about BF16 intrinsic, please refer to the latest ISE document: https://software.intel.com/en-us/download/intel-architecture-instruction-set-extensions-programming-reference
Patch by LiuTianle
Reviewers: craig.topper, smaslov, LuoYuanke, wxiao3, annita.zhang, spatel, RKSimon
Reviewed By: craig.topper
Subscribers: mgorny, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D60552
Modified:
cfe/trunk/docs/ClangCommandLineReference.rst
cfe/trunk/include/clang/Basic/BuiltinsX86.def
cfe/trunk/include/clang/Driver/Options.td
cfe/trunk/lib/Basic/Targets/X86.cpp
cfe/trunk/lib/Basic/Targets/X86.h
cfe/trunk/lib/CodeGen/CGBuiltin.cpp
cfe/trunk/lib/Headers/CMakeLists.txt
cfe/trunk/lib/Headers/cpuid.h
cfe/trunk/lib/Headers/immintrin.h
cfe/trunk/test/CodeGen/attr-target-x86.c
cfe/trunk/test/Driver/x86-target-features.c
cfe/trunk/test/Preprocessor/x86_target_features.c
Modified: cfe/trunk/docs/ClangCommandLineReference.rst
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/docs/ClangCommandLineReference.rst?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/docs/ClangCommandLineReference.rst (original)
+++ cfe/trunk/docs/ClangCommandLineReference.rst Mon May 6 01:25:11 2019
@@ -2610,6 +2610,8 @@ X86
.. option:: -mavx512bitalg, -mno-avx512bitalg
+.. option:: -mavx512bf16, -mno-avx512bf16
+
.. option:: -mavx512bw, -mno-avx512bw
.. option:: -mavx512cd, -mno-avx512cd
Modified: cfe/trunk/include/clang/Basic/BuiltinsX86.def
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/BuiltinsX86.def?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/BuiltinsX86.def (original)
+++ cfe/trunk/include/clang/Basic/BuiltinsX86.def Mon May 6 01:25:11 2019
@@ -1831,6 +1831,24 @@ TARGET_BUILTIN(__builtin_ia32_cvtusi2ss3
TARGET_BUILTIN(__builtin_ia32_vpmultishiftqb512, "V64cV64cV64c", "ncV:512:", "avx512vbmi")
TARGET_BUILTIN(__builtin_ia32_vpmultishiftqb128, "V16cV16cV16c", "ncV:128:", "avx512vbmi,avx512vl")
TARGET_BUILTIN(__builtin_ia32_vpmultishiftqb256, "V32cV32cV32c", "ncV:256:", "avx512vbmi,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_cvtne2ps2bf16_128, "V8sV4fV4f", "ncV:128:",
+ "avx512bf16,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_cvtne2ps2bf16_256, "V16sV8fV8f", "ncV:256:",
+ "avx512bf16,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_cvtne2ps2bf16_512, "V32sV16fV16f", "ncV:512:",
+ "avx512bf16")
+TARGET_BUILTIN(__builtin_ia32_cvtneps2bf16_128_mask, "V8sV4fV8sUc", "ncV:128:",
+ "avx512bf16,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_cvtneps2bf16_256, "V8sV8f", "ncV:256:",
+ "avx512bf16,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_cvtneps2bf16_512, "V16sV16f", "ncV:512:",
+ "avx512bf16")
+TARGET_BUILTIN(__builtin_ia32_dpbf16ps_128, "V4fV4fV4iV4i", "ncV:128:",
+ "avx512bf16,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_dpbf16ps_256, "V8fV8fV8iV8i", "ncV:256:",
+ "avx512bf16,avx512vl")
+TARGET_BUILTIN(__builtin_ia32_dpbf16ps_512, "V16fV16fV16iV16i", "ncV:512:",
+ "avx512bf16")
// generic select intrinsics
TARGET_BUILTIN(__builtin_ia32_selectb_128, "V16cUsV16cV16c", "ncV:128:", "avx512bw,avx512vl")
Modified: cfe/trunk/include/clang/Driver/Options.td
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Driver/Options.td?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/include/clang/Driver/Options.td (original)
+++ cfe/trunk/include/clang/Driver/Options.td Mon May 6 01:25:11 2019
@@ -2854,6 +2854,8 @@ def mavx2 : Flag<["-"], "mavx2">, Group<
def mno_avx2 : Flag<["-"], "mno-avx2">, Group<m_x86_Features_Group>;
def mavx512f : Flag<["-"], "mavx512f">, Group<m_x86_Features_Group>;
def mno_avx512f : Flag<["-"], "mno-avx512f">, Group<m_x86_Features_Group>;
+def mavx512bf16 : Flag<["-"], "mavx512bf16">, Group<m_x86_Features_Group>;
+def mno_avx512bf16 : Flag<["-"], "mno-avx512bf16">, Group<m_x86_Features_Group>;
def mavx512bitalg : Flag<["-"], "mavx512bitalg">, Group<m_x86_Features_Group>;
def mno_avx512bitalg : Flag<["-"], "mno-avx512bitalg">, Group<m_x86_Features_Group>;
def mavx512bw : Flag<["-"], "mavx512bw">, Group<m_x86_Features_Group>;
Modified: cfe/trunk/lib/Basic/Targets/X86.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/Targets/X86.cpp?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/lib/Basic/Targets/X86.cpp (original)
+++ cfe/trunk/lib/Basic/Targets/X86.cpp Mon May 6 01:25:11 2019
@@ -521,6 +521,7 @@ void X86TargetInfo::setSSELevel(llvm::St
Features["avx512ifma"] = Features["avx512vpopcntdq"] =
Features["avx512bitalg"] = Features["avx512vnni"] =
Features["avx512vbmi2"] = false;
+ Features["avx512bf16"] = false;
break;
}
}
@@ -652,16 +653,22 @@ void X86TargetInfo::setFeatureEnabledImp
Name == "avx512dq" || Name == "avx512bw" || Name == "avx512vl" ||
Name == "avx512vbmi" || Name == "avx512ifma" ||
Name == "avx512vpopcntdq" || Name == "avx512bitalg" ||
+ Name == "avx512bf16" ||
Name == "avx512vnni" || Name == "avx512vbmi2") {
if (Enabled)
setSSELevel(Features, AVX512F, Enabled);
// Enable BWI instruction if VBMI/VBMI2/BITALG is being enabled.
if ((Name.startswith("avx512vbmi") || Name == "avx512bitalg") && Enabled)
Features["avx512bw"] = true;
+ if (Name == "avx512bf16" && Enabled)
+ Features["avx512bw"] = Features["avx512vl"] = true;
// Also disable VBMI/VBMI2/BITALG if BWI is being disabled.
if (Name == "avx512bw" && !Enabled)
Features["avx512vbmi"] = Features["avx512vbmi2"] =
+ Features["avx512bf16"] =
Features["avx512bitalg"] = false;
+ if (Name == "avx512vl" && !Enabled)
+ Features["avx512bf16"] = false;
} else if (Name == "fma") {
if (Enabled)
setSSELevel(Features, AVX, Enabled);
@@ -751,6 +758,8 @@ bool X86TargetInfo::handleTargetFeatures
HasAVX512VPOPCNTDQ = true;
} else if (Feature == "+avx512vnni") {
HasAVX512VNNI = true;
+ } else if (Feature == "+avx512bf16") {
+ HasAVX512BF16 = true;
} else if (Feature == "+avx512er") {
HasAVX512ER = true;
} else if (Feature == "+avx512pf") {
@@ -1141,6 +1150,8 @@ void X86TargetInfo::getTargetDefines(con
Builder.defineMacro("__AVX512VPOPCNTDQ__");
if (HasAVX512VNNI)
Builder.defineMacro("__AVX512VNNI__");
+ if (HasAVX512BF16)
+ Builder.defineMacro("__AVX512BF16__");
if (HasAVX512ER)
Builder.defineMacro("__AVX512ER__");
if (HasAVX512PF)
@@ -1305,6 +1316,7 @@ bool X86TargetInfo::isValidFeatureName(S
.Case("avx512cd", true)
.Case("avx512vpopcntdq", true)
.Case("avx512vnni", true)
+ .Case("avx512bf16", true)
.Case("avx512er", true)
.Case("avx512pf", true)
.Case("avx512dq", true)
@@ -1383,6 +1395,7 @@ bool X86TargetInfo::hasFeature(StringRef
.Case("avx512cd", HasAVX512CD)
.Case("avx512vpopcntdq", HasAVX512VPOPCNTDQ)
.Case("avx512vnni", HasAVX512VNNI)
+ .Case("avx512bf16", HasAVX512BF16)
.Case("avx512er", HasAVX512ER)
.Case("avx512pf", HasAVX512PF)
.Case("avx512dq", HasAVX512DQ)
Modified: cfe/trunk/lib/Basic/Targets/X86.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/Targets/X86.h?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/lib/Basic/Targets/X86.h (original)
+++ cfe/trunk/lib/Basic/Targets/X86.h Mon May 6 01:25:11 2019
@@ -68,6 +68,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetI
bool HasAVX512CD = false;
bool HasAVX512VPOPCNTDQ = false;
bool HasAVX512VNNI = false;
+ bool HasAVX512BF16 = false;
bool HasAVX512ER = false;
bool HasAVX512PF = false;
bool HasAVX512DQ = false;
Modified: cfe/trunk/lib/CodeGen/CGBuiltin.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/CodeGen/CGBuiltin.cpp?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/lib/CodeGen/CGBuiltin.cpp (original)
+++ cfe/trunk/lib/CodeGen/CGBuiltin.cpp Mon May 6 01:25:11 2019
@@ -11851,6 +11851,14 @@ Value *CodeGenFunction::EmitX86BuiltinEx
case X86::BI__builtin_ia32_cmpordsd:
return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
+// AVX512 bf16 intrinsics
+ case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
+ Ops[2] = getMaskVecValue(*this, Ops[2],
+ Ops[0]->getType()->getVectorNumElements());
+ Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
+ return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
+ }
+
case X86::BI__emul:
case X86::BI__emulu: {
llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
Modified: cfe/trunk/lib/Headers/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/CMakeLists.txt?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/lib/Headers/CMakeLists.txt (original)
+++ cfe/trunk/lib/Headers/CMakeLists.txt Mon May 6 01:25:11 2019
@@ -6,6 +6,7 @@ set(files
armintr.h
arm64intr.h
avx2intrin.h
+ avx512bf16intrin.h
avx512bwintrin.h
avx512bitalgintrin.h
avx512vlbitalgintrin.h
@@ -21,6 +22,7 @@ set(files
avx512vbmivlintrin.h
avx512vbmi2intrin.h
avx512vlvbmi2intrin.h
+ avx512vlbf16intrin.h
avx512vlbwintrin.h
avx512vlcdintrin.h
avx512vldqintrin.h
Modified: cfe/trunk/lib/Headers/cpuid.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/cpuid.h?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/lib/Headers/cpuid.h (original)
+++ cfe/trunk/lib/Headers/cpuid.h Mon May 6 01:25:11 2019
@@ -184,6 +184,9 @@
#define bit_PCONFIG 0x00040000
#define bit_IBT 0x00100000
+/* Features in %eax for leaf 7 sub-leaf 1 */
+#define bit_AVX512BF16 0x00000020
+
/* Features in %eax for leaf 13 sub-leaf 1 */
#define bit_XSAVEOPT 0x00000001
#define bit_XSAVEC 0x00000002
Modified: cfe/trunk/lib/Headers/immintrin.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/immintrin.h?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/lib/Headers/immintrin.h (original)
+++ cfe/trunk/lib/Headers/immintrin.h Mon May 6 01:25:11 2019
@@ -181,6 +181,15 @@
#include <avx512pfintrin.h>
#endif
+#if !defined(_MSC_VER) || __has_feature(modules) || defined(__AVX512BF16__)
+#include <avx512bf16intrin.h>
+#endif
+
+#if !defined(_MSC_VER) || __has_feature(modules) || \
+ (defined(__AVX512VL__) && defined(__AVX512BF16__))
+#include <avx512vlbf16intrin.h>
+#endif
+
#if !defined(_MSC_VER) || __has_feature(modules) || defined(__PKU__)
#include <pkuintrin.h>
#endif
Modified: cfe/trunk/test/CodeGen/attr-target-x86.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CodeGen/attr-target-x86.c?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/test/CodeGen/attr-target-x86.c (original)
+++ cfe/trunk/test/CodeGen/attr-target-x86.c Mon May 6 01:25:11 2019
@@ -50,9 +50,9 @@ int __attribute__((target("arch=lakemont
// CHECK: use_before_def{{.*}} #7
// CHECK: #0 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87"
// CHECK: #1 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt"
-// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-aes,-avx,-avx2,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vpopcntdq,-f16c,-fma,-fma4,-gfni,-pclmul,-sha,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
+// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-aes,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vpopcntdq,-f16c,-fma,-fma4,-gfni,-pclmul,-sha,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
// CHECK: #3 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87"
-// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-avx,-avx2,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vpopcntdq,-f16c,-fma,-fma4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
+// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-avx,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vpopcntdq,-f16c,-fma,-fma4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop,-xsave,-xsaveopt"
// CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-vaes"
// CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cx8,+x87,-3dnow,-3dnowa,-mmx"
// CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx"
Modified: cfe/trunk/test/Driver/x86-target-features.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Driver/x86-target-features.c?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/test/Driver/x86-target-features.c (original)
+++ cfe/trunk/test/Driver/x86-target-features.c Mon May 6 01:25:11 2019
@@ -178,3 +178,8 @@
// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-invpcid %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-INVPCID %s
// INVPCID: "-target-feature" "+invpcid"
// NO-INVPCID: "-target-feature" "-invpcid"
+
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mavx512bf16 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=AVX512BF16 %s
+// RUN: %clang -target i386-unknown-linux-gnu -march=i386 -mno-avx512bf16 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-AVX512BF16 %s
+// AVX512BF16: "-target-feature" "+avx512bf16"
+// NO-AVX512BF16: "-target-feature" "-avx512bf16"
Modified: cfe/trunk/test/Preprocessor/x86_target_features.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Preprocessor/x86_target_features.c?rev=360018&r1=360017&r2=360018&view=diff
==============================================================================
--- cfe/trunk/test/Preprocessor/x86_target_features.c (original)
+++ cfe/trunk/test/Preprocessor/x86_target_features.c Mon May 6 01:25:11 2019
@@ -443,3 +443,18 @@
// RUN: %clang -target i386-unknown-unknown -march=atom -mrdpid -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=RDPID %s
// RDPID: #define __RDPID__ 1
+
+// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512bf16 -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512BF16 %s
+
+// AVX512BF16: #define __AVX512BF16__ 1
+// AVX512BF16: #define __AVX512BW__ 1
+// AVX512BF16: #define __AVX512VL__ 1
+
+// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512bf16 -mno-avx512bw -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512BF16_NOAVX512BW %s
+
+// AVX512BF16_NOAVX512BW-NOT: #define __AVX512BF16__ 1
+
+// RUN: %clang -target i386-unknown-unknown -march=atom -mavx512bf16 -mno-avx512vl -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVX512BF16_NOAVX512VL %s
+
+// AVX512BF16_NOAVX512VL-NOT: #define __AVX512BF16__ 1
+
More information about the cfe-commits
mailing list