[clang] [clang-tools-extra] [compiler-rt] [llvm] [X86] AMD Zen 6 Initial enablement (PR #179150)
via cfe-commits
cfe-commits at lists.llvm.org
Sun Feb 1 15:36:47 PST 2026
https://github.com/ganeshgit created https://github.com/llvm/llvm-project/pull/179150
Added new ISA AVX512 BMM.
CPUID checks are updated for new subtarget.
Model numbers are checked for identifying zen6.
>From 7f63e8fb5e6f7beaaf8f830e644a14b3ecfb0cab Mon Sep 17 00:00:00 2001
From: Ganesh Gopalasubramanian <Ganesh.Gopalasubramanian at amd.com>
Date: Sun, 1 Feb 2026 22:06:40 +0000
Subject: [PATCH] [X86] AMD Zen 6 Initial enablement
---
.../find-all-symbols/STLPostfixHeaderMap.cpp | 2 +
.../clangd/index/CanonicalIncludes.cpp | 2 +
clang/docs/ReleaseNotes.rst | 15 ++
clang/include/clang/Basic/BuiltinsX86.td | 28 +++
clang/include/clang/Options/Options.td | 2 +
clang/lib/Basic/Targets/X86.cpp | 15 ++
clang/lib/Basic/Targets/X86.h | 1 +
clang/lib/CodeGen/TargetBuiltins/X86.cpp | 43 ++++
clang/lib/Headers/CMakeLists.txt | 2 +
clang/lib/Headers/avx512bmmintrin.h | 63 +++++
clang/lib/Headers/avx512bmmvlintrin.h | 85 +++++++
clang/lib/Headers/immintrin.h | 4 +
clang/test/CodeGen/attr-target-x86.c | 4 +-
clang/test/CodeGen/target-builtin-noerror.c | 1 +
clang/test/Driver/x86-march.c | 4 +
clang/test/Frontend/x86-target-cpu.c | 1 +
clang/test/Misc/target-invalid-cpu-note/x86.c | 4 +
.../Preprocessor/predefined-arch-macros.c | 151 ++++++++++++
compiler-rt/lib/builtins/cpu_model/x86.c | 23 +-
llvm/include/llvm/IR/IntrinsicsX86.td | 27 +++
.../llvm/Support/GenericLoopInfoImpl.h | 5 +-
.../llvm/TargetParser/X86TargetParser.def | 2 +
.../llvm/TargetParser/X86TargetParser.h | 1 +
.../SelectionDAG/ScheduleDAGRRList.cpp | 3 +-
llvm/lib/Target/X86/X86.td | 15 ++
llvm/lib/Target/X86/X86ISelLowering.cpp | 11 +-
llvm/lib/Target/X86/X86ISelLowering.h | 5 +
llvm/lib/Target/X86/X86InstrAVX512.td | 62 +++++
llvm/lib/Target/X86/X86InstrFragmentsSIMD.td | 8 +-
llvm/lib/Target/X86/X86InstrPredicates.td | 2 +
llvm/lib/Target/X86/X86IntrinsicsInfo.h | 12 +
llvm/lib/Target/X86/X86PfmCounters.td | 1 +
llvm/lib/Target/X86/X86ScheduleZnver4.td | 32 +--
llvm/lib/TargetParser/Host.cpp | 9 +-
llvm/lib/TargetParser/X86TargetParser.cpp | 6 +
.../X86/avx512bmm-vbitrevb-bitreverse.ll | 88 +++++++
.../X86/avx512bmm-vbitrevb-intrinsics-mem.ll | 141 +++++++++++
.../X86/avx512bmm-vbitrevb-intrinsics.ll | 222 ++++++++++++++++++
.../CodeGen/X86/avx512bmm-vbmac-intrinsics.ll | 63 +++++
.../CodeGen/X86/bypass-slow-division-64.ll | 1 +
llvm/test/CodeGen/X86/cmp16.ll | 1 +
llvm/test/CodeGen/X86/cpus-amd.ll | 1 +
llvm/test/CodeGen/X86/rdpru.ll | 1 +
llvm/test/CodeGen/X86/shuffle-as-shifts.ll | 1 +
llvm/test/CodeGen/X86/slow-unaligned-mem.ll | 1 +
llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll | 1 +
.../X86/tuning-shuffle-permilpd-avx512.ll | 1 +
.../X86/tuning-shuffle-permilps-avx512.ll | 1 +
.../X86/tuning-shuffle-unpckpd-avx512.ll | 1 +
.../X86/tuning-shuffle-unpckps-avx512.ll | 1 +
.../X86/vector-shuffle-fast-per-lane.ll | 1 +
llvm/test/CodeGen/X86/vpdpwssd.ll | 1 +
.../CodeGen/X86/x86-64-double-shifts-var.ll | 1 +
llvm/test/MC/X86/x86_long_nop.s | 2 +
llvm/test/TableGen/x86-fold-tables.inc | 33 +++
.../Transforms/LoopUnroll/X86/call-remark.ll | 1 +
.../Transforms/SLPVectorizer/X86/pr63668.ll | 1 +
llvm/utils/TableGen/X86FoldTablesEmitter.cpp | 3 +-
.../gn/secondary/clang/lib/Headers/BUILD.gn | 2 +
59 files changed, 1185 insertions(+), 36 deletions(-)
create mode 100644 clang/lib/Headers/avx512bmmintrin.h
create mode 100644 clang/lib/Headers/avx512bmmvlintrin.h
create mode 100644 llvm/test/CodeGen/X86/avx512bmm-vbitrevb-bitreverse.ll
create mode 100644 llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics-mem.ll
create mode 100644 llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics.ll
create mode 100644 llvm/test/CodeGen/X86/avx512bmm-vbmac-intrinsics.ll
diff --git a/clang-tools-extra/clang-include-fixer/find-all-symbols/STLPostfixHeaderMap.cpp b/clang-tools-extra/clang-include-fixer/find-all-symbols/STLPostfixHeaderMap.cpp
index d43a3ea327a39..8a5e3290ed9ef 100644
--- a/clang-tools-extra/clang-include-fixer/find-all-symbols/STLPostfixHeaderMap.cpp
+++ b/clang-tools-extra/clang-include-fixer/find-all-symbols/STLPostfixHeaderMap.cpp
@@ -39,6 +39,8 @@ const HeaderMapCollector::RegexHeaderMap *getSTLPostfixHeaderMap() {
{"include/ammintrin.h$", "<ammintrin.h>"},
{"include/avx2intrin.h$", "<immintrin.h>"},
{"include/avx512bwintrin.h$", "<immintrin.h>"},
+ {"include/avx512bmmintrin.h$", "<immintrin.h>"},
+ {"include/avx512bmmvlintrin.h$", "<immintrin.h>"},
{"include/avx512cdintrin.h$", "<immintrin.h>"},
{"include/avx512dqintrin.h$", "<immintrin.h>"},
{"include/avx512erintrin.h$", "<immintrin.h>"},
diff --git a/clang-tools-extra/clangd/index/CanonicalIncludes.cpp b/clang-tools-extra/clangd/index/CanonicalIncludes.cpp
index cbef64f351341..3a646e47a9b16 100644
--- a/clang-tools-extra/clangd/index/CanonicalIncludes.cpp
+++ b/clang-tools-extra/clangd/index/CanonicalIncludes.cpp
@@ -42,6 +42,8 @@ const std::pair<llvm::StringRef, llvm::StringRef> IncludeMappings[] = {
{"include/ammintrin.h", "<ammintrin.h>"},
{"include/avx2intrin.h", "<immintrin.h>"},
{"include/avx512bwintrin.h", "<immintrin.h>"},
+ {"include/avx512bmmintrin.h", "<immintrin.h>"},
+ {"include/avx512bmmvlintrin.h", "<immintrin.h>"},
{"include/avx512cdintrin.h", "<immintrin.h>"},
{"include/avx512dqintrin.h", "<immintrin.h>"},
{"include/avx512erintrin.h", "<immintrin.h>"},
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 3a3d76112a02b..a6ac25634a567 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -265,6 +265,21 @@ NVPTX Support
X86 Support
^^^^^^^^^^^
+- `march=znver6` is now supported.
+- Support ISA of ``AVX512BMM``.
+ * Support intrinsic of ``_mm512_vbmacor16x16x16_epi16``.
+ * Support intrinsic of ``_mm512_vbmacxor16x16x16_epi16``.
+ * Support intrinsic of ``_mm512_mask_vbitrev_epi8``.
+ * Support intrinsic of ``_mm512_maskz_vbitrev_epi8``.
+ * Support intrinsic of ``_mm512_vbitrev_epi8``.
+ * Support intrinsic of ``_mm256_vbmacor16x16x16_epi16``.
+ * Support intrinsic of ``_mm256_vbmacxor16x16x16_epi16``.
+ * Support intrinsic of ``_mm128_mask_vbitrev_epu8``.
+ * Support intrinsic of ``_mm256_mask_vbitrev_epu8``.
+ * Support intrinsic of ``_mm128_maskz_vbitrev_epu8``.
+ * Support intrinsic of ``_mm256_maskz_vbitrev_epu8``.
+ * Support intrinsic of ``_mm128_vbitrev_epu8``.
+ * Support intrinsic of ``_mm256_vbitrev_epu8``.
Arm and AArch64 Support
^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/BuiltinsX86.td b/clang/include/clang/Basic/BuiltinsX86.td
index 23eac47eb5e4c..7d1b513532f34 100644
--- a/clang/include/clang/Basic/BuiltinsX86.td
+++ b/clang/include/clang/Basic/BuiltinsX86.td
@@ -5055,3 +5055,31 @@ let Features = "avx10.2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>
let Features = "avx10.2", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
def vgetmantbf16512_mask : X86Builtin<"_Vector<32, __bf16>(_Vector<32, __bf16>, _Constant int, _Vector<32, __bf16>, unsigned int)">;
}
+
+let Features = "avx10.2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+ def vsqrtbf16 : X86Builtin<"_Vector<8, __bf16>(_Vector<8, __bf16>)">;
+}
+
+let Features = "avx10.2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+ def vsqrtbf16256 : X86Builtin<"_Vector<16, __bf16>(_Vector<16, __bf16>)">;
+}
+
+let Features = "avx10.2", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+ def vsqrtbf16512 : X86Builtin<"_Vector<32, __bf16>(_Vector<32, __bf16>)">;
+}
+
+let Features = "avx512bmm", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
+ def bmacor16x16x16_v32hi : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>, _Vector<32, short>)">;
+ def bmacxor16x16x16_v32hi : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>, _Vector<32, short>)">;
+ def bitrev512 : X86Builtin<"_Vector<64, char>(_Vector<64, char>)">;
+}
+
+let Features = "avx512bmm,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
+ def bmacor16x16x16_v16hi : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>, _Vector<16, short>)">;
+ def bmacxor16x16x16_v16hi : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>, _Vector<16, short>)">;
+ def bitrev256 : X86Builtin<"_Vector<32, char>(_Vector<32, char>)">;
+}
+
+let Features = "avx512bmm,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
+ def bitrev128 : X86Builtin<"_Vector<16, char>(_Vector<16, char>)">;
+}
diff --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index 421208a812bbc..658ee6f7ebe60 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -6889,6 +6889,8 @@ def mavx512bf16 : Flag<["-"], "mavx512bf16">, Group<m_x86_Features_Group>;
def mno_avx512bf16 : Flag<["-"], "mno-avx512bf16">, Group<m_x86_Features_Group>;
def mavx512bitalg : Flag<["-"], "mavx512bitalg">, Group<m_x86_Features_Group>;
def mno_avx512bitalg : Flag<["-"], "mno-avx512bitalg">, Group<m_x86_Features_Group>;
+def mavx512bmm : Flag<["-"], "mavx512bmm">, Group<m_x86_Features_Group>;
+def mno_avx512bmm : Flag<["-"], "mno-avx512bmm">, Group<m_x86_Features_Group>;
def mavx512bw : Flag<["-"], "mavx512bw">, Group<m_x86_Features_Group>;
def mno_avx512bw : Flag<["-"], "mno-avx512bw">, Group<m_x86_Features_Group>;
def mavx512cd : Flag<["-"], "mavx512cd">, Group<m_x86_Features_Group>;
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index f00d435937b92..2d65e5f6a9ba7 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -296,6 +296,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
HasAVX512DQ = true;
} else if (Feature == "+avx512bitalg") {
HasAVX512BITALG = true;
+ } else if (Feature == "+avx512bmm") {
+ HasAVX512BMM = true;
} else if (Feature == "+avx512bw") {
HasAVX512BW = true;
} else if (Feature == "+avx512vl") {
@@ -308,6 +310,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
HasAVX512IFMA = true;
} else if (Feature == "+avx512vp2intersect") {
HasAVX512VP2INTERSECT = true;
+ } else if (Feature == "avx512bmm") {
+ HasAVX512BMM = true;
} else if (Feature == "+sha") {
HasSHA = true;
} else if (Feature == "+sha512") {
@@ -716,6 +720,9 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
case CK_ZNVER5:
defineCPUMacros(Builder, "znver5");
break;
+ case CK_ZNVER6:
+ defineCPUMacros(Builder, "znver6");
+ break;
case CK_Geode:
defineCPUMacros(Builder, "geode");
break;
@@ -833,6 +840,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__AVX512DQ__");
if (HasAVX512BITALG)
Builder.defineMacro("__AVX512BITALG__");
+ if (HasAVX512BMM)
+ Builder.defineMacro("__AVX512BMM__");
if (HasAVX512BW)
Builder.defineMacro("__AVX512BW__");
if (HasAVX512VL) {
@@ -846,6 +855,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__AVX512IFMA__");
if (HasAVX512VP2INTERSECT)
Builder.defineMacro("__AVX512VP2INTERSECT__");
+ if (HasAVX512BMM)
+ Builder.defineMacro("__AVX512BMM__");
if (HasSHA)
Builder.defineMacro("__SHA__");
if (HasSHA512)
@@ -1076,6 +1087,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const {
.Case("avx512fp16", true)
.Case("avx512dq", true)
.Case("avx512bitalg", true)
+ .Case("avx512bmm", true)
.Case("avx512bw", true)
.Case("avx512vl", true)
.Case("avx512vbmi", true)
@@ -1196,6 +1208,8 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
.Case("avx512fp16", HasAVX512FP16)
.Case("avx512dq", HasAVX512DQ)
.Case("avx512bitalg", HasAVX512BITALG)
+ .Case("avx512bmm", HasAVX512BMM)
+ .Case("avx512bmm", HasAVX512BMM)
.Case("avx512bw", HasAVX512BW)
.Case("avx512vl", HasAVX512VL)
.Case("avx512vbmi", HasAVX512VBMI)
@@ -1641,6 +1655,7 @@ std::optional<unsigned> X86TargetInfo::getCPUCacheLineSize() const {
case CK_ZNVER3:
case CK_ZNVER4:
case CK_ZNVER5:
+ case CK_ZNVER6:
// Deprecated
case CK_x86_64:
case CK_x86_64_v2:
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
index 922e32906cd04..6bd55f9fbf4bb 100644
--- a/clang/lib/Basic/Targets/X86.h
+++ b/clang/lib/Basic/Targets/X86.h
@@ -104,6 +104,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo {
bool HasAVX512BF16 = false;
bool HasAVX512DQ = false;
bool HasAVX512BITALG = false;
+ bool HasAVX512BMM = false;
bool HasAVX512BW = false;
bool HasAVX512VL = false;
bool HasAVX512VBMI = false;
diff --git a/clang/lib/CodeGen/TargetBuiltins/X86.cpp b/clang/lib/CodeGen/TargetBuiltins/X86.cpp
index 9645ed87b8ef3..2c4e1f0cc8b17 100644
--- a/clang/lib/CodeGen/TargetBuiltins/X86.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/X86.cpp
@@ -2678,6 +2678,49 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
return EmitX86MaskedCompareResult(*this, Shufbit, NumElts, MaskIn);
}
+ case X86::BI__builtin_ia32_bitrev128:
+ case X86::BI__builtin_ia32_bitrev256:
+ case X86::BI__builtin_ia32_bitrev512: {
+ Intrinsic::ID ID;
+ switch (BuiltinID) {
+ default: llvm_unreachable("Unsupported intrinsic!");
+ case X86::BI__builtin_ia32_bitrev128:
+ ID = Intrinsic::x86_avx512_vbitrevb_128;
+ break;
+ case X86::BI__builtin_ia32_bitrev256:
+ ID = Intrinsic::x86_avx512_vbitrevb_256;
+ break;
+ case X86::BI__builtin_ia32_bitrev512:
+ ID = Intrinsic::x86_avx512_vbitrevb_512;
+ break;
+ }
+
+ return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
+ }
+
+ case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
+ case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
+ case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
+ case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi: {
+ Intrinsic::ID ID;
+ switch (BuiltinID) {
+ default: llvm_unreachable("Unsupported intrinsic!");
+ case X86::BI__builtin_ia32_bmacor16x16x16_v16hi:
+ ID = Intrinsic::x86_avx512_vbmacor_v16hi;
+ break;
+ case X86::BI__builtin_ia32_bmacor16x16x16_v32hi:
+ ID = Intrinsic::x86_avx512_vbmacor_v32hi;
+ break;
+ case X86::BI__builtin_ia32_bmacxor16x16x16_v16hi:
+ ID = Intrinsic::x86_avx512_vbmacxor_v16hi;
+ break;
+ case X86::BI__builtin_ia32_bmacxor16x16x16_v32hi:
+ ID = Intrinsic::x86_avx512_vbmacxor_v32hi;
+ break;
+ }
+
+ return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
+ }
// packed comparison intrinsics
case X86::BI__builtin_ia32_cmpeqps:
case X86::BI__builtin_ia32_cmpeqpd:
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index c92b370b88d2d..3686080c1d6cf 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -186,6 +186,8 @@ set(x86_files
avx2intrin.h
avx512bf16intrin.h
avx512bitalgintrin.h
+ avx512bmmintrin.h
+ avx512bmmvlintrin.h
avx512bwintrin.h
avx512cdintrin.h
avx512dqintrin.h
diff --git a/clang/lib/Headers/avx512bmmintrin.h b/clang/lib/Headers/avx512bmmintrin.h
new file mode 100644
index 0000000000000..d39106f9c2276
--- /dev/null
+++ b/clang/lib/Headers/avx512bmmintrin.h
@@ -0,0 +1,63 @@
+/*===-------- avx512bmmintrin.h - AVX512BMM intrinsics *------------------===
+ *
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===---------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <avx512bmmintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef _AVX512BMMINTRIN_H
+#define _AVX512BMMINTRIN_H
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS \
+ __attribute__((__always_inline__, __nodebug__, __target__("avx512bmm"), \
+ __min_vector_width__(512)))
+
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
+
+static __inline __m512i __DEFAULT_FN_ATTRS _mm512_bmacor16x16x16(__m512i __A,
+ __m512i __B,
+ __m512i __C) {
+ return (__m512i)__builtin_ia32_bmacor16x16x16_v32hi(
+ (__v32hi)__A, (__v32hi)__B, (__v32hi)__C);
+}
+
+static __inline __m512i __DEFAULT_FN_ATTRS _mm512_bmacxor16x16x16(__m512i __A,
+ __m512i __B,
+ __m512i __C) {
+ return (__m512i)__builtin_ia32_bmacxor16x16x16_v32hi(
+ (__v32hi)__A, (__v32hi)__B, (__v32hi)__C);
+}
+
+static __inline __m512i __DEFAULT_FN_ATTRS _mm512_bitrev_epi8(__m512i __A) {
+ return (__m512i)__builtin_ia32_bitrev512((__v64qi)__A);
+}
+
+static __inline __m512i __DEFAULT_FN_ATTRS
+_mm512_mask_bitrev_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
+ return (__m512i)__builtin_ia32_selectb_512(
+ (__mmask64)__U, (__v64qi)_mm512_bitrev_epi8(__A), (__v64qi)__B);
+}
+
+static __inline __m512i __DEFAULT_FN_ATTRS
+_mm512_maskz_bitrev_epi8(__mmask64 __U, __m512i __A) {
+ return (__m512i)__builtin_ia32_selectb_512((__mmask64)__U,
+ (__v64qi)_mm512_bitrev_epi8(__A),
+ (__v64qi)_mm512_setzero_si512());
+}
+
+#undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
+
+#endif
diff --git a/clang/lib/Headers/avx512bmmvlintrin.h b/clang/lib/Headers/avx512bmmvlintrin.h
new file mode 100644
index 0000000000000..86a1f2ab410d0
--- /dev/null
+++ b/clang/lib/Headers/avx512bmmvlintrin.h
@@ -0,0 +1,85 @@
+/*===------------- avx512bmvlintrin.h - BMM intrinsics ------------------===
+ *
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef __IMMINTRIN_H
+#error \
+ "Never use <avx512bmmvlintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#ifndef __BMMVLINTRIN_H
+#define __BMMVLINTRIN_H
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS128 \
+ __attribute__((__always_inline__, __nodebug__, \
+ __target__("avx512bmm,avx512vl"), __min_vector_width__(128)))
+#define __DEFAULT_FN_ATTRS256 \
+ __attribute__((__always_inline__, __nodebug__, \
+ __target__("avx512bmm,avx512vl"), __min_vector_width__(256)))
+
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
+static __inline __m256i __DEFAULT_FN_ATTRS256
+_mm256_bmacor16x16x16(__m256i __A, __m256i __B, __m256i __C) {
+ return (__m256i)__builtin_ia32_bmacor16x16x16_v16hi(
+ (__v16hi)__A, (__v16hi)__B, (__v16hi)__C);
+}
+
+static __inline __m256i __DEFAULT_FN_ATTRS256
+_mm256_bmacxor16x16x16(__m256i __A, __m256i __B, __m256i __C) {
+ return (__m256i)__builtin_ia32_bmacxor16x16x16_v16hi(
+ (__v16hi)__A, (__v16hi)__B, (__v16hi)__C);
+}
+
+static __inline __m128i __DEFAULT_FN_ATTRS128 _mm128_bitrev_epi8(__m128i __A) {
+ return (__m128i)__builtin_ia32_bitrev128((__v16qi)__A);
+}
+
+static __inline __m256i __DEFAULT_FN_ATTRS256 _mm256_bitrev_epi8(__m256i __A) {
+ return (__m256i)__builtin_ia32_bitrev256((__v32qi)__A);
+}
+
+static __inline __m128i __DEFAULT_FN_ATTRS128
+_mm128_mask_bitrev_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
+ return (__m128i)__builtin_ia32_selectb_128(
+ (__mmask16)__U, (__v16qi)_mm128_bitrev_epi8(__A), (__v16qi)__B);
+}
+
+static __inline __m256i __DEFAULT_FN_ATTRS256
+_mm256_mask_bitrev_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
+ return (__m256i)__builtin_ia32_selectb_256(
+ (__mmask32)__U, (__v32qi)_mm256_bitrev_epi8(__A), (__v32qi)__B);
+}
+
+static __inline __m128i __DEFAULT_FN_ATTRS128
+_mm128_maskz_bitrev_epi8(__mmask16 __U, __m128i __A) {
+ return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
+ (__v16qi)_mm128_bitrev_epi8(__A),
+ (__v16qi)_mm_setzero_si128());
+}
+
+static __inline __m256i __DEFAULT_FN_ATTRS256
+_mm256_maskz_bitrev_epi8(__mmask32 __U, __m256i __A) {
+ return (__m256i)__builtin_ia32_selectb_256((__mmask32)__U,
+ (__v32qi)_mm256_bitrev_epi8(__A),
+ (__v32qi)_mm256_setzero_si256());
+}
+
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS128
+#undef __DEFAULT_FN_ATTRS256
+
+#endif
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 19064a4ff5cea..00107c44c3a55 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -58,6 +58,10 @@
#include <avx512bitalgintrin.h>
+#include <avx512bmmintrin.h>
+
+#include <avx512bmmvlintrin.h>
+
#include <avx512cdintrin.h>
#include <avx512vpopcntdqintrin.h>
diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c
index 474fa93629d89..6a110ce38605b 100644
--- a/clang/test/CodeGen/attr-target-x86.c
+++ b/clang/test/CodeGen/attr-target-x86.c
@@ -33,7 +33,7 @@ __attribute__((target("fpmath=387")))
void f_fpmath_387(void) {}
// CHECK-NOT: tune-cpu
-// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
+// CHECK: [[f_no_sse2]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse2")))
void f_no_sse2(void) {}
@@ -41,7 +41,7 @@ void f_no_sse2(void) {}
__attribute__((target("sse4")))
void f_sse4(void) {}
-// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
+// CHECK: [[f_no_sse4]] = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1,-avx10.2,-avx2,-avx512bf16,-avx512bitalg,-avx512bmm,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686"
__attribute__((target("no-sse4")))
void f_no_sse4(void) {}
diff --git a/clang/test/CodeGen/target-builtin-noerror.c b/clang/test/CodeGen/target-builtin-noerror.c
index 47d5ae51d643a..a65a07d81b8c0 100644
--- a/clang/test/CodeGen/target-builtin-noerror.c
+++ b/clang/test/CodeGen/target-builtin-noerror.c
@@ -209,5 +209,6 @@ void verifycpustrings(void) {
(void)__builtin_cpu_is("znver3");
(void)__builtin_cpu_is("znver4");
(void)__builtin_cpu_is("znver5");
+ (void)__builtin_cpu_is("znver6");
(void)__builtin_cpu_is("diamondrapids");
}
diff --git a/clang/test/Driver/x86-march.c b/clang/test/Driver/x86-march.c
index 15f82547892c2..6a3ef5be67d8a 100644
--- a/clang/test/Driver/x86-march.c
+++ b/clang/test/Driver/x86-march.c
@@ -258,6 +258,10 @@
// RUN: %clang -target x86_64-unknown-unknown -c -### %s -march=znver5 2>&1 \
// RUN: | FileCheck %s -check-prefix=znver5
// znver5: "-target-cpu" "znver5"
+//
+// RUN: %clang -target x86_64-unknown-unknown -c -### %s -march=znver6 2>&1 \
+// RUN: | FileCheck %s -check-prefix=znver6
+// znver6: "-target-cpu" "znver6"
// RUN: %clang -target x86_64 -c -### %s -march=x86-64 2>&1 | FileCheck %s --check-prefix=x86-64
// x86-64: "-target-cpu" "x86-64"
diff --git a/clang/test/Frontend/x86-target-cpu.c b/clang/test/Frontend/x86-target-cpu.c
index f2885a040c370..7dc7f5474687e 100644
--- a/clang/test/Frontend/x86-target-cpu.c
+++ b/clang/test/Frontend/x86-target-cpu.c
@@ -39,5 +39,6 @@
// RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu znver3 -verify %s
// RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu znver4 -verify %s
// RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu znver5 -verify %s
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -target-cpu znver6 -verify %s
//
// expected-no-diagnostics
diff --git a/clang/test/Misc/target-invalid-cpu-note/x86.c b/clang/test/Misc/target-invalid-cpu-note/x86.c
index 4a70e9bff3fef..766bd679796f5 100644
--- a/clang/test/Misc/target-invalid-cpu-note/x86.c
+++ b/clang/test/Misc/target-invalid-cpu-note/x86.c
@@ -103,6 +103,7 @@
// X86-SAME: {{^}}, znver3
// X86-SAME: {{^}}, znver4
// X86-SAME: {{^}}, znver5
+// X86-SAME: {{^}}, znver6
// X86-SAME: {{^}}, x86-64
// X86-SAME: {{^}}, x86-64-v2
// X86-SAME: {{^}}, x86-64-v3
@@ -183,6 +184,7 @@
// X86_64-SAME: {{^}}, znver3
// X86_64-SAME: {{^}}, znver4
// X86_64-SAME: {{^}}, znver5
+// X86_64-SAME: {{^}}, znver6
// X86_64-SAME: {{^}}, x86-64
// X86_64-SAME: {{^}}, x86-64-v2
// X86_64-SAME: {{^}}, x86-64-v3
@@ -290,6 +292,7 @@
// TUNE_X86-SAME: {{^}}, znver3
// TUNE_X86-SAME: {{^}}, znver4
// TUNE_X86-SAME: {{^}}, znver5
+// TUNE_X86-SAME: {{^}}, znver6
// TUNE_X86-SAME: {{^}}, x86-64
// TUNE_X86-SAME: {{^}}, geode
// TUNE_X86-SAME: {{$}}
@@ -395,6 +398,7 @@
// TUNE_X86_64-SAME: {{^}}, znver3
// TUNE_X86_64-SAME: {{^}}, znver4
// TUNE_X86_64-SAME: {{^}}, znver5
+// TUNE_X86_64-SAME: {{^}}, znver6
// TUNE_X86_64-SAME: {{^}}, x86-64
// TUNE_X86_64-SAME: {{^}}, geode
// TUNE_X86_64-SAME: {{$}}
diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index 1e38b4d3ba350..be94eb064cf91 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -4133,6 +4133,157 @@
// CHECK_ZNVER5_M64: #define __znver5 1
// CHECK_ZNVER5_M64: #define __znver5__ 1
+// RUN: %clang -march=znver6 -m32 -E -dM %s -o - 2>&1 \
+// RUN: -target i386-unknown-linux \
+// RUN: | FileCheck -match-full-lines %s -check-prefix=CHECK_ZNVER6_M32
+// CHECK_ZNVER6_M32-NOT: #define __3dNOW_A__ 1
+// CHECK_ZNVER6_M32-NOT: #define __3dNOW__ 1
+// CHECK_ZNVER6_M32: #define __ADX__ 1
+// CHECK_ZNVER6_M32: #define __AES__ 1
+// CHECK_ZNVER6_M32: #define __AVX2__ 1
+// CHECK_ZNVER6_M32: #define __AVX512BF16__ 1
+// CHECK_ZNVER6_M32: #define __AVX512BITALG__ 1
+// CHECK_ZNVER6_M32: #define __AVX512BW__ 1
+// CHECK_ZNVER6_M32: #define __AVX512CD__ 1
+// CHECK_ZNVER6_M32: #define __AVX512DQ__ 1
+// CHECK_ZNVER6_M32: #define __AVX512FP16__ 1
+// CHECK_ZNVER6_M32: #define __AVX512F__ 1
+// CHECK_ZNVER6_M32: #define __AVX512IFMA__ 1
+// CHECK_ZNVER6_M32: #define __AVX512VBMI2__ 1
+// CHECK_ZNVER6_M32: #define __AVX512VBMI__ 1
+// CHECK_ZNVER6_M32: #define __AVX512VL__ 1
+// CHECK_ZNVER6_M32: #define __AVX512VNNI__ 1
+// CHECK_ZNVER6_M32: #define __AVX512VP2INTERSECT__ 1
+// CHECK_ZNVER6_M32: #define __AVX512VPOPCNTDQ__ 1
+// CHECK_ZNVER6_M32: #define __AVXIFMA__ 1
+// CHECK_ZNVER6_M32: #define __AVXNECONVERT__ 1
+// CHECK_ZNVER6_M32: #define __AVXVNNIINT8__ 1
+// CHECK_ZNVER6_M32: #define __AVXVNNI__ 1
+// CHECK_ZNVER6_M32: #define __AVX__ 1
+// CHECK_ZNVER6_M32: #define __BMI2__ 1
+// CHECK_ZNVER6_M32: #define __BMI__ 1
+// CHECK_ZNVER6_M32: #define __CLFLUSHOPT__ 1
+// CHECK_ZNVER6_M32: #define __CLWB__ 1
+// CHECK_ZNVER6_M32: #define __CLZERO__ 1
+// CHECK_ZNVER6_M32: #define __F16C__ 1
+// CHECK_ZNVER6_M32-NOT: #define __FMA4__ 1
+// CHECK_ZNVER6_M32: #define __FMA__ 1
+// CHECK_ZNVER6_M32: #define __FSGSBASE__ 1
+// CHECK_ZNVER6_M32: #define __GFNI__ 1
+// CHECK_ZNVER6_M32: #define __LZCNT__ 1
+// CHECK_ZNVER6_M32: #define __MMX__ 1
+// CHECK_ZNVER6_M32: #define __MOVDIR64B__ 1
+// CHECK_ZNVER6_M32: #define __MOVDIRI__ 1
+// CHECK_ZNVER6_M32: #define __PCLMUL__ 1
+// CHECK_ZNVER6_M32: #define __PKU__ 1
+// CHECK_ZNVER6_M32: #define __POPCNT__ 1
+// CHECK_ZNVER6_M32: #define __PREFETCHI__ 1
+// CHECK_ZNVER6_M32: #define __PRFCHW__ 1
+// CHECK_ZNVER6_M32: #define __RDPID__ 1
+// CHECK_ZNVER6_M32: #define __RDPRU__ 1
+// CHECK_ZNVER6_M32: #define __RDRND__ 1
+// CHECK_ZNVER6_M32: #define __RDSEED__ 1
+// CHECK_ZNVER6_M32: #define __SHA__ 1
+// CHECK_ZNVER6_M32: #define __SSE2_MATH__ 1
+// CHECK_ZNVER6_M32: #define __SSE2__ 1
+// CHECK_ZNVER6_M32: #define __SSE3__ 1
+// CHECK_ZNVER6_M32: #define __SSE4A__ 1
+// CHECK_ZNVER6_M32: #define __SSE4_1__ 1
+// CHECK_ZNVER6_M32: #define __SSE4_2__ 1
+// CHECK_ZNVER6_M32: #define __SSE_MATH__ 1
+// CHECK_ZNVER6_M32: #define __SSE__ 1
+// CHECK_ZNVER6_M32: #define __SSSE3__ 1
+// CHECK_ZNVER6_M32-NOT: #define __TBM__ 1
+// CHECK_ZNVER6_M32: #define __WBNOINVD__ 1
+// CHECK_ZNVER6_M32-NOT: #define __XOP__ 1
+// CHECK_ZNVER6_M32: #define __XSAVEC__ 1
+// CHECK_ZNVER6_M32: #define __XSAVEOPT__ 1
+// CHECK_ZNVER6_M32: #define __XSAVES__ 1
+// CHECK_ZNVER6_M32: #define __XSAVE__ 1
+// CHECK_ZNVER6_M32: #define __i386 1
+// CHECK_ZNVER6_M32: #define __i386__ 1
+// CHECK_ZNVER6_M32: #define __tune_znver6__ 1
+// CHECK_ZNVER6_M32: #define __znver6 1
+// CHECK_ZNVER6_M32: #define __znver6__ 1
+
+// RUN: %clang -march=znver6 -m64 -E -dM %s -o - 2>&1 \
+// RUN: -target i386-unknown-linux \
+// RUN: | FileCheck -match-full-lines %s -check-prefix=CHECK_ZNVER6_M64
+// CHECK_ZNVER6_M64-NOT: #define __3dNOW_A__ 1
+// CHECK_ZNVER6_M64-NOT: #define __3dNOW__ 1
+// CHECK_ZNVER6_M64: #define __ADX__ 1
+// CHECK_ZNVER6_M64: #define __AES__ 1
+// CHECK_ZNVER6_M64: #define __AVX2__ 1
+// CHECK_ZNVER6_M64: #define __AVX512BF16__ 1
+// CHECK_ZNVER6_M64: #define __AVX512BITALG__ 1
+// CHECK_ZNVER6_M64: #define __AVX512BW__ 1
+// CHECK_ZNVER6_M64: #define __AVX512CD__ 1
+// CHECK_ZNVER6_M64: #define __AVX512DQ__ 1
+// CHECK_ZNVER6_M64: #define __AVX512FP16__ 1
+// CHECK_ZNVER6_M64: #define __AVX512F__ 1
+// CHECK_ZNVER6_M64: #define __AVX512IFMA__ 1
+// CHECK_ZNVER6_M64: #define __AVX512VBMI2__ 1
+// CHECK_ZNVER6_M64: #define __AVX512VBMI__ 1
+// CHECK_ZNVER6_M64: #define __AVX512VL__ 1
+// CHECK_ZNVER6_M64: #define __AVX512VNNI__ 1
+// CHECK_ZNVER6_M64: #define __AVX512VP2INTERSECT__ 1
+// CHECK_ZNVER6_M64: #define __AVX512VPOPCNTDQ__ 1
+// CHECK_ZNVER6_M64: #define __AVXIFMA__ 1
+// CHECK_ZNVER6_M64: #define __AVXNECONVERT__ 1
+// CHECK_ZNVER6_M64: #define __AVXVNNIINT8__ 1
+// CHECK_ZNVER6_M64: #define __AVXVNNI__ 1
+// CHECK_ZNVER6_M64: #define __AVX__ 1
+// CHECK_ZNVER6_M64: #define __BMI2__ 1
+// CHECK_ZNVER6_M64: #define __BMI__ 1
+// CHECK_ZNVER6_M64: #define __CLFLUSHOPT__ 1
+// CHECK_ZNVER6_M64: #define __CLWB__ 1
+// CHECK_ZNVER6_M64: #define __CLZERO__ 1
+// CHECK_ZNVER6_M64: #define __F16C__ 1
+// CHECK_ZNVER6_M64-NOT: #define __FMA4__ 1
+// CHECK_ZNVER6_M64: #define __FMA__ 1
+// CHECK_ZNVER6_M64: #define __FSGSBASE__ 1
+// CHECK_ZNVER6_M64: #define __GFNI__ 1
+// CHECK_ZNVER6_M64: #define __LZCNT__ 1
+// CHECK_ZNVER6_M64: #define __MMX__ 1
+// CHECK_ZNVER6_M64: #define __MOVDIR64B__ 1
+// CHECK_ZNVER6_M64: #define __MOVDIRI__ 1
+// CHECK_ZNVER6_M64: #define __PCLMUL__ 1
+// CHECK_ZNVER6_M64: #define __PKU__ 1
+// CHECK_ZNVER6_M64: #define __POPCNT__ 1
+// CHECK_ZNVER6_M64: #define __PREFETCHI__ 1
+// CHECK_ZNVER6_M64: #define __PRFCHW__ 1
+// CHECK_ZNVER6_M64: #define __RDPID__ 1
+// CHECK_ZNVER6_M64: #define __RDPRU__ 1
+// CHECK_ZNVER6_M64: #define __RDRND__ 1
+// CHECK_ZNVER6_M64: #define __RDSEED__ 1
+// CHECK_ZNVER6_M64: #define __SHA__ 1
+// CHECK_ZNVER6_M64: #define __SSE2_MATH__ 1
+// CHECK_ZNVER6_M64: #define __SSE2__ 1
+// CHECK_ZNVER6_M64: #define __SSE3__ 1
+// CHECK_ZNVER6_M64: #define __SSE4A__ 1
+// CHECK_ZNVER6_M64: #define __SSE4_1__ 1
+// CHECK_ZNVER6_M64: #define __SSE4_2__ 1
+// CHECK_ZNVER6_M64: #define __SSE_MATH__ 1
+// CHECK_ZNVER6_M64: #define __SSE__ 1
+// CHECK_ZNVER6_M64: #define __SSSE3__ 1
+// CHECK_ZNVER6_M64-NOT: #define __TBM__ 1
+// CHECK_ZNVER6_M64: #define __VAES__ 1
+// CHECK_ZNVER6_M64: #define __VPCLMULQDQ__ 1
+// CHECK_ZNVER6_M64: #define __WBNOINVD__ 1
+// CHECK_ZNVER6_M64-NOT: #define __XOP__ 1
+// CHECK_ZNVER6_M64: #define __XSAVEC__ 1
+// CHECK_ZNVER6_M64: #define __XSAVEOPT__ 1
+// CHECK_ZNVER6_M64: #define __XSAVES__ 1
+// CHECK_ZNVER6_M64: #define __XSAVE__ 1
+// CHECK_ZNVER6_M64: #define __amd64 1
+// CHECK_ZNVER6_M64: #define __amd64__ 1
+// CHECK_ZNVER6_M64: #define __tune_znver6__ 1
+// CHECK_ZNVER6_M64: #define __x86_64 1
+// CHECK_ZNVER6_M64: #define __x86_64__ 1
+// CHECK_ZNVER6_M64: #define __znver6 1
+// CHECK_ZNVER6_M64: #define __znver6__ 1
+
+
// End X86/GCC/Linux tests ------------------
// Begin PPC/GCC/Linux tests ----------------
diff --git a/compiler-rt/lib/builtins/cpu_model/x86.c b/compiler-rt/lib/builtins/cpu_model/x86.c
index 55eb2b0958450..405dc65efb56a 100644
--- a/compiler-rt/lib/builtins/cpu_model/x86.c
+++ b/compiler-rt/lib/builtins/cpu_model/x86.c
@@ -27,6 +27,7 @@
#if (defined(__GNUC__) || defined(__clang__)) && !defined(_MSC_VER)
#include <cpuid.h>
+#include <stdio.h>
#endif
#ifdef _MSC_VER
@@ -105,6 +106,7 @@ enum ProcessorSubtypes {
INTEL_COREI7_ARROWLAKE_S,
INTEL_COREI7_PANTHERLAKE,
AMDFAM1AH_ZNVER5,
+ AMDFAM1AH_ZNVER6,
INTEL_COREI7_DIAMONDRAPIDS,
INTEL_COREI7_NOVALAKE,
CPU_SUBTYPE_MAX
@@ -231,6 +233,7 @@ enum ProcessorFeatures {
FEATURE_AMX_FP8 = 120,
FEATURE_MOVRS,
FEATURE_AMX_MOVRS,
+ FEATURE_AVX512BMM,
CPU_FEATURE_MAX
};
@@ -837,20 +840,18 @@ getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
case 26:
CPU = "znver5";
Type = AMDFAM1AH;
- if (Model <= 0x77) {
- // Models 00h-0Fh (Breithorn).
- // Models 10h-1Fh (Breithorn-Dense).
- // Models 20h-2Fh (Strix 1).
- // Models 30h-37h (Strix 2).
- // Models 38h-3Fh (Strix 3).
- // Models 40h-4Fh (Granite Ridge).
- // Models 50h-5Fh (Weisshorn).
- // Models 60h-6Fh (Krackan1).
- // Models 70h-77h (Sarlak).
+ if (Model <= 0x4f || (Model >= 0x60 && Model <= 0x77) ||
+ (Model >= 0xd0 && Model <= 0xd7)) {
CPU = "znver5";
Subtype = AMDFAM1AH_ZNVER5;
break; // "znver5"
}
+ if ((Model >= 0x50 && Model <= 0x5f) || (Model >= 0x80 && Model <= 0xcf) ||
+ (Model >= 0xd8 && Model <= 0xe7)) {
+ CPU = "znver6";
+ Subtype = AMDFAM1AH_ZNVER6;
+ break; // "znver6"
+ }
break;
default:
break; // Unknown AMD CPU.
@@ -1142,6 +1143,8 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
// AMD cpuid bit for prefetchi is different from Intel
if (HasExtLeaf21 && ((EAX >> 20) & 1))
setFeature(FEATURE_PREFETCHI);
+ if (HasExtLeaf21 && ((EAX >> 23) & 1))
+ setFeature(FEATURE_AVX512BMM);
bool HasLeaf14 = MaxLevel >= 0x14 &&
!getX86CpuIDAndInfoEx(0x14, 0x0, &EAX, &EBX, &ECX, &EDX);
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index b75a0485d6263..a0597556147cc 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -7341,4 +7341,31 @@ def int_x86_movrsdi : ClangBuiltin<"__builtin_ia32_movrsdi">,
[IntrReadMem]>;
def int_x86_prefetchrs : ClangBuiltin<"__builtin_ia32_prefetchrs">,
Intrinsic<[], [llvm_ptr_ty], []>;
+
+//===----------------------------------------------------------------------===//
+// BMM intrinsics
+def int_x86_avx512_vbitrevb_128 :
+ DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty],
+ [IntrNoMem]>;
+def int_x86_avx512_vbitrevb_256 :
+ DefaultAttrsIntrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty],
+ [IntrNoMem]>;
+def int_x86_avx512_vbitrevb_512 :
+ DefaultAttrsIntrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty],
+ [IntrNoMem]>;
+
+def int_x86_avx512_vbmacor_v16hi :
+ DefaultAttrsIntrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty],
+ [IntrNoMem]>;
+def int_x86_avx512_vbmacor_v32hi :
+ DefaultAttrsIntrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty],
+ [IntrNoMem]>;
+
+def int_x86_avx512_vbmacxor_v16hi :
+ DefaultAttrsIntrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, llvm_v16i16_ty],
+ [IntrNoMem]>;
+def int_x86_avx512_vbmacxor_v32hi :
+ DefaultAttrsIntrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, llvm_v32i16_ty],
+ [IntrNoMem]>;
}
+//===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Support/GenericLoopInfoImpl.h b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
index c830f0a67a448..89df62926a5bc 100644
--- a/llvm/include/llvm/Support/GenericLoopInfoImpl.h
+++ b/llvm/include/llvm/Support/GenericLoopInfoImpl.h
@@ -551,8 +551,9 @@ void PopulateLoopsDFS<BlockT, LoopT>::insertIntoLoop(BlockT *Block) {
// For convenience, Blocks and Subloops are inserted in postorder. Reverse
// the lists, except for the loop header, which is always at the beginning.
Subloop->reverseBlock(1);
- std::reverse(Subloop->getSubLoopsVector().begin(),
- Subloop->getSubLoopsVector().end());
+ auto &SubloopsVec = Subloop->getSubLoopsVector();
+ if (!SubloopsVec.empty())
+ std::reverse(SubloopsVec.begin(), SubloopsVec.end());
Subloop = Subloop->getParentLoop();
}
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def
index 09592bcea27f4..084c1a5b05b21 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.def
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.def
@@ -107,6 +107,7 @@ X86_CPU_SUBTYPE(INTEL_COREI7_ARROWLAKE, "arrowlake")
X86_CPU_SUBTYPE(INTEL_COREI7_ARROWLAKE_S, "arrowlake-s")
X86_CPU_SUBTYPE(INTEL_COREI7_PANTHERLAKE, "pantherlake")
X86_CPU_SUBTYPE(AMDFAM1AH_ZNVER5, "znver5")
+X86_CPU_SUBTYPE(AMDFAM1AH_ZNVER6, "znver6")
X86_CPU_SUBTYPE(INTEL_COREI7_DIAMONDRAPIDS, "diamondrapids")
X86_CPU_SUBTYPE(INTEL_COREI7_NOVALAKE, "novalake")
@@ -247,6 +248,7 @@ X86_FEATURE_COMPAT(AMX_TF32, "amx-tf32", 0, 118)
X86_FEATURE_COMPAT(AMX_FP8, "amx-fp8", 0, 120)
X86_FEATURE_COMPAT(MOVRS, "movrs", 0, 121)
X86_FEATURE_COMPAT(AMX_MOVRS, "amx-movrs", 0, 122)
+X86_FEATURE_COMPAT(AVX512BMM, "avx512bmm", 0, 123)
// Features we don't multiversion on.
X86_FEATURE (NF, "nf")
diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.h b/llvm/include/llvm/TargetParser/X86TargetParser.h
index 46061f9d1fc7d..31d13ce29f7fc 100644
--- a/llvm/include/llvm/TargetParser/X86TargetParser.h
+++ b/llvm/include/llvm/TargetParser/X86TargetParser.h
@@ -146,6 +146,7 @@ enum CPUKind {
CK_ZNVER3,
CK_ZNVER4,
CK_ZNVER5,
+ CK_ZNVER6,
CK_x86_64,
CK_x86_64_v2,
CK_x86_64_v3,
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 12fc26d949581..a9335d18f046c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -1637,7 +1637,8 @@ void ScheduleDAGRRList::ListScheduleBottomUp() {
}
// Reverse the order if it is bottom up.
- std::reverse(Sequence.begin(), Sequence.end());
+ if (!Sequence.empty())
+ std::reverse(Sequence.begin(), Sequence.end());
#ifndef NDEBUG
VerifyScheduledSequence(/*isBottomUp=*/true);
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 1b9a6ee2b4ef4..446214b8b77cd 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -145,6 +145,9 @@ def FeatureVBMI : SubtargetFeature<"avx512vbmi", "HasVBMI", "true",
def FeatureVBMI2 : SubtargetFeature<"avx512vbmi2", "HasVBMI2", "true",
"Enable AVX-512 further Vector Byte Manipulation Instructions",
[FeatureBWI]>;
+def FeatureBMM : SubtargetFeature<"avx512bmm", "HasBMM", "true",
+ "Enable AVX512 Bit Manipulation Instructions",[FeatureVLX]>;
+
def FeatureAVXIFMA : SubtargetFeature<"avxifma", "HasAVXIFMA", "true",
"Enable AVX-IFMA",
[FeatureAVX2]>;
@@ -1631,6 +1634,16 @@ def ProcessorFeatures {
];
list<SubtargetFeature> ZN5Features =
!listconcat(ZN4Features, ZN5AdditionalFeatures);
+
+ list<SubtargetFeature> ZN6Tuning = ZN5Tuning;
+ list<SubtargetFeature> ZN6AdditionalFeatures = [FeatureFP16,
+ FeatureAVXVNNIINT8,
+ FeatureAVXNECONVERT,
+ FeatureAVXIFMA,
+ FeatureBMM
+ ];
+ list<SubtargetFeature> ZN6Features =
+ !listconcat(ZN5Features, ZN6AdditionalFeatures);
}
//===----------------------------------------------------------------------===//
@@ -1993,6 +2006,8 @@ def : ProcModel<"znver4", Znver4Model, ProcessorFeatures.ZN4Features,
ProcessorFeatures.ZN4Tuning>;
def : ProcModel<"znver5", Znver4Model, ProcessorFeatures.ZN5Features,
ProcessorFeatures.ZN5Tuning>;
+def : ProcModel<"znver6", Znver4Model, ProcessorFeatures.ZN6Features,
+ ProcessorFeatures.ZN6Tuning>;
def : Proc<"geode", [FeatureX87, FeatureCX8, FeatureMMX, FeaturePRFCHW],
[TuningSlowUAMem16, TuningInsertVZEROUPPER]>;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index dff237f303103..87d00999ce438 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -33190,6 +33190,12 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
unsigned NumElts = VT.getVectorNumElements();
+ // If we have BMM, we can use VPBITREVB to reverse the bits in each byte.
+ // BMM implies VLX support, so this works for 128/256/512-bit vectors.
+ if (Subtarget.hasBMM() && VT.getSizeInBits() >= 128) {
+ return DAG.getNode(X86ISD::VPBITREVB, DL, VT, In);
+ }
+
// If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits.
if (Subtarget.hasGFNI()) {
SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT);
@@ -35922,8 +35928,9 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
NODE_NAME_CASE(CVTTP2UIS)
NODE_NAME_CASE(MCVTTP2UIS)
NODE_NAME_CASE(POP_FROM_X87_REG)
- NODE_NAME_CASE(TC_RETURN_GLOBALADDR)
- NODE_NAME_CASE(CALL_GLOBALADDR)
+ NODE_NAME_CASE(VPBITREVB)
+ NODE_NAME_CASE(VBMACOR)
+ NODE_NAME_CASE(VBMACXOR)
}
return nullptr;
#undef NODE_NAME_CASE
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 30faca54e13f6..4ae8da84d6c89 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -997,6 +997,11 @@ namespace llvm {
AESENCWIDE256KL,
AESDECWIDE256KL,
+ // BMM Instructions
+ VPBITREVB,
+ VBMACOR,
+ VBMACXOR,
+
/// Compare and Add if Condition is Met. Compare value in operand 2 with
/// value in memory of operand 1. If condition of operand 4 is met, add
/// value operand 3 to m32 and write new value in operand 1. Operand 2 is
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 0392100f8c6c3..ce73e7fb508d9 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -11320,6 +11320,41 @@ multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
Sched<[sched.Folded]>;
}
+// Variant of avx512_unary_rm that requires aligned memory operands
+multiclass avx512_unary_rm_aligned<bits<8> opc, string OpcodeStr, SDNode OpNode,
+ X86FoldableSchedWrite sched, X86VectorVTInfo _> {
+ let ExeDomain = _.ExeDomain in {
+ defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
+ (ins _.RC:$src1), OpcodeStr,
+ "$src1", "$src1",
+ (_.VT (OpNode (_.VT _.RC:$src1)))>, EVEX, AVX5128IBase,
+ Sched<[sched]>;
+
+ let mayLoad = 1 in
+ defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
+ (ins _.MemOp:$src1), OpcodeStr,
+ "$src1", "$src1",
+ (_.VT (OpNode (_.VT (bitconvert (_.AlignedLdFrag addr:$src1)))))>,
+ EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>,
+ Sched<[sched.Folded]>;
+ }
+}
+
+multiclass avx512_unary_rm_vl_aligned<bits<8> opc, string OpcodeStr, SDNode OpNode,
+ X86SchedWriteWidths sched,
+ AVX512VLVectorVTInfo VTInfo, Predicate prd> {
+ let Predicates = [prd] in
+ defm Z : avx512_unary_rm_aligned<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,
+ EVEX_V512;
+
+ let Predicates = [prd, HasVLX] in {
+ defm Z256 : avx512_unary_rm_aligned<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,
+ EVEX_V256;
+ defm Z128 : avx512_unary_rm_aligned<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,
+ EVEX_V128;
+ }
+}
+
multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
X86SchedWriteWidths sched,
AVX512VLVectorVTInfo VTInfo, Predicate prd> {
@@ -13764,3 +13799,30 @@ let Uses = [MXCSR] in {
defm VFCMULCSHZ : avx512_cfmbinop_sh_common<0xD7, "vfcmulcsh", x86vfcmulcSh, x86vfcmulcShRnd, 0>,
T_MAP6, XD, EVEX_CD8<32, CD8VT1>, EVEX_V128, VEX_LIG, EVEX, VVVV;
}
+
+// VPBITREVB - BMM bit reverse instructions
+// Basic instruction patterns for BMM bit manipulation
+// Note: VPBITREVB requires aligned memory operands
+defm VPBITREVB : avx512_unary_rm_vl_aligned<0x81, "vpbitrevb", x86vpbitrevb, SchedWriteVecALU,
+ avx512vl_i8_info, HasBMM>, T_MAP6, XS;
+
+defm : avx512_unary_lowering<"VPBITREVB", x86vpbitrevb, avx512vl_i8_info, HasBMM>;
+
+// VBMACOR, VBMACXOR - BMM matrix multiplication instructions
+// VBMACOR: EVEX.256.NP.MAP6.W0 80 /r, EVEX.512.NP.MAP6.W0 80 /r
+let Predicates = [HasBMM, HasVLX] in
+defm VBMACORZ256 : VNNI_rmb<0x80, "vbmacor16x16x16", x86vbmacor, SchedWriteVecIMul.YMM, v16i16x_info, 0>,
+ EVEX_V256, T_MAP6;
+
+let Predicates = [HasBMM] in
+defm VBMACORZ : VNNI_rmb<0x80, "vbmacor16x16x16", x86vbmacor, SchedWriteVecIMul.ZMM, v32i16_info, 0>,
+ EVEX_V512, T_MAP6;
+
+// VBMACXOR: EVEX.256.NP.MAP6.W1 80 /r, EVEX.512.NP.MAP6.W1 80 /r
+let Predicates = [HasBMM, HasVLX] in
+defm VBMACXORZ256 : VNNI_rmb<0x80, "vbmacxor16x16x16", x86vbmacxor, SchedWriteVecIMul.YMM, v16i16x_info, 0>,
+ EVEX_V256, T_MAP6, REX_W;
+
+let Predicates = [HasBMM] in
+defm VBMACXORZ : VNNI_rmb<0x80, "vbmacxor16x16x16", x86vbmacxor, SchedWriteVecIMul.ZMM, v32i16_info, 0>,
+ EVEX_V512, T_MAP6, REX_W;
\ No newline at end of file
diff --git a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index a2b09c62af958..72a99a2ca5011 100644
--- a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -245,7 +245,7 @@ def X86cmpmmSAE : SDNode<"X86ISD::CMPMM_SAE", X86MaskCmpMaskCC>;
def X86cmpms : SDNode<"X86ISD::FSETCCM", X86CmpMaskCCScalar>;
def X86cmpmsSAE : SDNode<"X86ISD::FSETCCM_SAE", X86CmpMaskCCScalar>;
-def X86phminpos: SDNode<"X86ISD::PHMINPOS",
+def X86phminpos: SDNode<"X86ISD::PHMINPOS",
SDTypeProfile<1, 1, [SDTCisVT<0, v8i16>, SDTCisVT<1, v8i16>]>>;
def X86vshiftuniform : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
@@ -1518,3 +1518,9 @@ def X86vpmaddwd_su : PatFrag<(ops node:$lhs, node:$rhs),
return N->hasOneUse();
}]>;
+// Bit reverse bytes operation
+def x86vpbitrevb : SDNode<"X86ISD::VPBITREVB", SDTShuff1Op>;
+
+// BMM matrix multiplication operations
+def x86vbmacor : SDNode<"X86ISD::VBMACOR", SDTVnni>;
+def x86vbmacxor : SDNode<"X86ISD::VBMACXOR", SDTVnni>;
\ No newline at end of file
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 21e6bacbacee2..e9819778dbe4f 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -85,6 +85,7 @@ def HasVPOPCNTDQ : Predicate<"Subtarget->hasVPOPCNTDQ()">;
def HasDQI : Predicate<"Subtarget->hasDQI()">;
def NoDQI : Predicate<"!Subtarget->hasDQI()">;
def HasBWI : Predicate<"Subtarget->hasBWI()">;
+def HasBMM : Predicate<"Subtarget->hasBMM()">;
def NoBWI : Predicate<"!Subtarget->hasBWI()">;
def HasVLX : Predicate<"Subtarget->hasVLX()">;
def NoVLX : Predicate<"!Subtarget->hasVLX()">;
@@ -175,6 +176,7 @@ def HasENQCMD : Predicate<"Subtarget->hasENQCMD()">;
def HasAMXFP16 : Predicate<"Subtarget->hasAMXFP16()">;
def HasCMPCCXADD : Predicate<"Subtarget->hasCMPCCXADD()">;
def HasAVXNECONVERT : Predicate<"Subtarget->hasAVXNECONVERT()">;
+def HasAVXBMM : Predicate<"Subtarget->hasAVXBMM()">;
def HasKL : Predicate<"Subtarget->hasKL()">;
def HasRAOINT : Predicate<"Subtarget->hasRAOINT()">;
def HasWIDEKL : Predicate<"Subtarget->hasWIDEKL()">;
diff --git a/llvm/lib/Target/X86/X86IntrinsicsInfo.h b/llvm/lib/Target/X86/X86IntrinsicsInfo.h
index c0c98c1f35491..9fff406772ff3 100644
--- a/llvm/lib/Target/X86/X86IntrinsicsInfo.h
+++ b/llvm/lib/Target/X86/X86IntrinsicsInfo.h
@@ -1389,6 +1389,18 @@ static const IntrinsicData IntrinsicsWithoutChain[] = {
X86ISD::FSUB_RND),
X86_INTRINSIC_DATA(avx512_uitofp_round, INTR_TYPE_1OP, ISD::UINT_TO_FP,
X86ISD::UINT_TO_FP_RND),
+ X86_INTRINSIC_DATA(avx512_vbitrevb_128, INTR_TYPE_1OP, X86ISD::VPBITREVB,
+ 0),
+ X86_INTRINSIC_DATA(avx512_vbitrevb_256, INTR_TYPE_1OP, X86ISD::VPBITREVB,
+ 0),
+ X86_INTRINSIC_DATA(avx512_vbitrevb_512, INTR_TYPE_1OP, X86ISD::VPBITREVB,
+ 0),
+ X86_INTRINSIC_DATA(avx512_vbmacor_v16hi, INTR_TYPE_3OP, X86ISD::VBMACOR, 0),
+ X86_INTRINSIC_DATA(avx512_vbmacor_v32hi, INTR_TYPE_3OP, X86ISD::VBMACOR, 0),
+ X86_INTRINSIC_DATA(avx512_vbmacxor_v16hi, INTR_TYPE_3OP, X86ISD::VBMACXOR,
+ 0),
+ X86_INTRINSIC_DATA(avx512_vbmacxor_v32hi, INTR_TYPE_3OP, X86ISD::VBMACXOR,
+ 0),
X86_INTRINSIC_DATA(avx512_vcomi_sd, COMI_RM, X86ISD::COMI, X86ISD::UCOMI),
X86_INTRINSIC_DATA(avx512_vcomi_ss, COMI_RM, X86ISD::COMI, X86ISD::UCOMI),
X86_INTRINSIC_DATA(avx512_vcvtsd2si32, INTR_TYPE_1OP, X86ISD::CVTS2SI,
diff --git a/llvm/lib/Target/X86/X86PfmCounters.td b/llvm/lib/Target/X86/X86PfmCounters.td
index b299633446410..9d2a1ce01c273 100644
--- a/llvm/lib/Target/X86/X86PfmCounters.td
+++ b/llvm/lib/Target/X86/X86PfmCounters.td
@@ -387,3 +387,4 @@ def ZnVer4PfmCounters : ProcPfmCounters {
}
def : PfmCountersBinding<"znver4", ZnVer4PfmCounters>;
def : PfmCountersBinding<"znver5", ZnVer4PfmCounters>;
+def : PfmCountersBinding<"znver6", ZnVer4PfmCounters>;
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td
index ac4d31de8dbfe..bffcf35d6b65d 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver4.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td
@@ -9,7 +9,7 @@
// This file defines the machine model for Znver4 to support instruction
// scheduling and other instruction cost heuristics.
// Based on:
-// * AMD Software Optimization Guide for the AMD Family 19h (Zen4)
+// * AMD Software Optimization Guide for the AMD Family 19h (Zen4)
// Microarchitecture
// https://www.amd.com/system/files/TechDocs/57647.zip
//===----------------------------------------------------------------------===//
@@ -1550,7 +1550,7 @@ def Zn4WriteVFIXUPIMMPDZrr_VRANGESDrr : SchedWriteRes<[Zn4FPFMisc01]> {
let NumMicroOps = 1;
}
def : InstRW<[Zn4WriteVFIXUPIMMPDZrr_VRANGESDrr], (instregex
- "VFIXUPIMM(S|P)(S|D)(Z|Z128|Z256?)rrik", "VFIXUPIMM(S|P)(S|D)(Z?|Z128?|Z256?)rrikz",
+ "VFIXUPIMM(S|P)(S|D)(Z|Z128|Z256?)rrik", "VFIXUPIMM(S|P)(S|D)(Z?|Z128?|Z256?)rrikz",
"VFIXUPIMM(S|P)(S|D)(Z128|Z256?)rri", "VRANGE(S|P)(S|D)(Z?|Z128?|Z256?)rri(b?)",
"VRANGE(S|P)(S|D)(Z|Z128|Z256?)rri(b?)k","VRANGE(S|P)(S|D)(Z?|Z128?|Z256?)rri(b?)kz"
)>;
@@ -1824,20 +1824,20 @@ def Zn4VecALUZSlow: SchedWriteRes<[Zn4FPFMisc01]> {
let ReleaseAtCycles = [2];
let NumMicroOps = 1;
}
-def : InstRW<[Zn4VecALUZSlow], (instrs
- VPABSBZ128rr, VPABSBZ128rrk, VPABSBZ128rrkz, VPABSDZ128rr,
- VPABSDZ128rrk, VPABSDZ128rrkz, VPABSQZ128rr, VPABSQZ128rrk,
- VPABSQZ128rrkz, VPABSWZ128rr, VPABSWZ128rrk, VPABSWZ128rrkz,
- VPADDSBZ128rr, VPADDSBZ128rrk, VPADDSBZ128rrkz, VPADDSWZ128rr,
- VPADDSWZ128rrk, VPADDSWZ128rrkz,VPADDUSBZ128rr, VPADDUSBZ128rrk,
- VPADDUSBZ128rrkz, VPADDUSWZ128rr, VPADDUSWZ128rrk, VPADDUSWZ128rrkz,
- VPAVGBZ128rr, VPAVGBZ128rrk, VPAVGBZ128rrkz, VPAVGWZ128rr,
- VPAVGWZ128rrk, VPAVGWZ128rrkz, VPOPCNTBZ128rr, VPOPCNTBZ128rrk,
- VPOPCNTBZ128rrkz, VPOPCNTDZ128rr, VPOPCNTDZ128rrk, VPOPCNTDZ128rrkz,
- VPOPCNTQZ128rr, VPOPCNTQZ128rrk,VPOPCNTQZ128rrkz, VPOPCNTWZ128rr,
- VPOPCNTWZ128rrk, VPOPCNTWZ128rrkz,VPSUBSBZ128rr, VPSUBSBZ128rrk,
- VPSUBSBZ128rrkz, VPSUBSWZ128rr, VPSUBSWZ128rrk, VPSUBSWZ128rrkz,
- VPSUBUSBZ128rr, VPSUBUSBZ128rrk, VPSUBUSBZ128rrkz,VPSUBUSWZ128rr,
+def : InstRW<[Zn4VecALUZSlow], (instrs
+ VPABSBZ128rr, VPABSBZ128rrk, VPABSBZ128rrkz, VPABSDZ128rr,
+ VPABSDZ128rrk, VPABSDZ128rrkz, VPABSQZ128rr, VPABSQZ128rrk,
+ VPABSQZ128rrkz, VPABSWZ128rr, VPABSWZ128rrk, VPABSWZ128rrkz,
+ VPADDSBZ128rr, VPADDSBZ128rrk, VPADDSBZ128rrkz, VPADDSWZ128rr,
+ VPADDSWZ128rrk, VPADDSWZ128rrkz,VPADDUSBZ128rr, VPADDUSBZ128rrk,
+ VPADDUSBZ128rrkz, VPADDUSWZ128rr, VPADDUSWZ128rrk, VPADDUSWZ128rrkz,
+ VPAVGBZ128rr, VPAVGBZ128rrk, VPAVGBZ128rrkz, VPAVGWZ128rr,
+ VPAVGWZ128rrk, VPAVGWZ128rrkz, VPOPCNTBZ128rr, VPOPCNTBZ128rrk,
+ VPOPCNTBZ128rrkz, VPOPCNTDZ128rr, VPOPCNTDZ128rrk, VPOPCNTDZ128rrkz,
+ VPOPCNTQZ128rr, VPOPCNTQZ128rrk,VPOPCNTQZ128rrkz, VPOPCNTWZ128rr,
+ VPOPCNTWZ128rrk, VPOPCNTWZ128rrkz,VPSUBSBZ128rr, VPSUBSBZ128rrk,
+ VPSUBSBZ128rrkz, VPSUBSWZ128rr, VPSUBSWZ128rrk, VPSUBSWZ128rrkz,
+ VPSUBUSBZ128rr, VPSUBUSBZ128rrk, VPSUBUSBZ128rrkz,VPSUBUSWZ128rr,
VPSUBUSWZ128rrk, VPSUBUSWZ128rrkz
)>;
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 5d4fa2c88153c..407fec8a5eb69 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -1340,6 +1340,12 @@ static const char *getAMDProcessorTypeAndSubtype(unsigned Family,
*Subtype = X86::AMDFAM1AH_ZNVER5;
break; // "znver5"
}
+ if ((Model >= 0x50 && Model <= 0x5f) || (Model >= 0x80 && Model <= 0xcf) ||
+ (Model >= 0xd8 && Model <= 0xe7)) {
+ CPU = "znver6";
+ *Subtype = X86::AMDFAM1AH_ZNVER6;
+ break; // "znver6"
+ }
break;
default:
@@ -1349,7 +1355,7 @@ static const char *getAMDProcessorTypeAndSubtype(unsigned Family,
return CPU;
}
-#undef testFeature
+#undef testFeaturemake
static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
unsigned *Features) {
@@ -2079,6 +2085,7 @@ StringMap<bool> sys::getHostCPUFeatures() {
!getX86CpuIDAndInfo(0x80000021, &EAX, &EBX, &ECX, &EDX);
// AMD cpuid bit for prefetchi is different from Intel
Features["prefetchi"] = HasExtLeaf21 && ((EAX >> 20) & 1);
+ Features["avx512bmm"] = HasExtLeaf21 && ((EAX >> 23) & 1) && HasAVX512Save;
bool HasLeaf7 =
MaxLevel >= 7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 2810849e4af9e..6ce1f9cbd8531 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -255,6 +255,10 @@ static constexpr FeatureBitset FeaturesZNVER5 =
FeaturesZNVER4 | FeatureAVXVNNI | FeatureMOVDIRI | FeatureMOVDIR64B |
FeatureAVX512VP2INTERSECT | FeaturePREFETCHI | FeatureAVXVNNI;
+static constexpr FeatureBitset FeaturesZNVER6 =
+ FeaturesZNVER5 | FeatureAVXVNNIINT8 | FeatureAVX512FP16 | FeatureAVXIFMA |
+ FeatureAVXNECONVERT | FeatureAVX512BMM;
+
// D151696 tranplanted Mangling and OnlyForCPUDispatchSpecific from
// X86TargetParser.def to here. They are assigned by following ways:
// 1. Copy the mangling from the original CPU_SPEICIFC MACROs. If no, assign
@@ -440,6 +444,7 @@ constexpr ProcInfo Processors[] = {
{ {"znver3"}, CK_ZNVER3, FEATURE_AVX2, FeaturesZNVER3, '\0', false },
{ {"znver4"}, CK_ZNVER4, FEATURE_AVX512VBMI2, FeaturesZNVER4, '\0', false },
{ {"znver5"}, CK_ZNVER5, FEATURE_AVX512VP2INTERSECT, FeaturesZNVER5, '\0', false },
+ { {"znver6"}, CK_ZNVER6, FEATURE_AVX512FP16, FeaturesZNVER6, '\0', false },
// Generic 64-bit processor.
{ {"x86-64"}, CK_x86_64, FEATURE_SSE2 , FeaturesX86_64, '\0', false },
{ {"x86-64-v2"}, CK_x86_64_v2, FEATURE_SSE4_2 , FeaturesX86_64_V2, '\0', false },
@@ -597,6 +602,7 @@ constexpr FeatureBitset ImpliedFeaturesAVX512VPOPCNTDQ = FeatureAVX512F;
constexpr FeatureBitset ImpliedFeaturesAVX512VBMI = FeatureAVX512BW;
constexpr FeatureBitset ImpliedFeaturesAVX512VBMI2 = FeatureAVX512BW;
constexpr FeatureBitset ImpliedFeaturesAVX512VP2INTERSECT = FeatureAVX512F;
+constexpr FeatureBitset ImpliedFeaturesAVX512BMM = FeatureAVX512BW;
// FIXME: These two aren't really implemented and just exist in the feature
// list for __builtin_cpu_supports. So omit their dependencies.
diff --git a/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-bitreverse.ll b/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-bitreverse.ll
new file mode 100644
index 0000000000000..dd8b92eefdc69
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-bitreverse.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bmm,+avx512vl,+avx512bw --show-mc-encoding | FileCheck %s
+
+; Test vbitrevb instruction generation from bitreverse intrinsic
+; This test verifies that the bitreverse intrinsic generates vbitrevb instructions
+; when AVX512BMM is available. This tests code converted from C (bitrev3.c).
+
+; Test 512-bit vector bit reversal with aligned memory load
+define <64 x i8> @bitrev_zmm_aligned_load(ptr %ptr) {
+; CHECK-LABEL: bitrev_zmm_aligned_load:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb (%rdi), %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x81,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <64 x i8>, ptr %ptr, align 64
+ %1 = tail call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %0)
+ ret <64 x i8> %1
+}
+
+; Test 256-bit with aligned memory load (AVX512VL)
+define <32 x i8> @bitrev_ymm_aligned_load(ptr %ptr) {
+; CHECK-LABEL: bitrev_ymm_aligned_load:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb (%rdi), %ymm0 # encoding: [0x62,0xf6,0x7e,0x28,0x81,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <32 x i8>, ptr %ptr, align 32
+ %1 = tail call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %0)
+ ret <32 x i8> %1
+}
+
+; Test 128-bit with aligned memory load (AVX512VL + AVX512BW)
+define <16 x i8> @bitrev_xmm_aligned_load(ptr %ptr) {
+; CHECK-LABEL: bitrev_xmm_aligned_load:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb (%rdi), %xmm0 # encoding: [0x62,0xf6,0x7e,0x08,0x81,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <16 x i8>, ptr %ptr, align 16
+ %1 = tail call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %0)
+ ret <16 x i8> %1
+}
+
+; Test 512-bit with unaligned memory load
+; Unaligned loads must use vmovdqu + vpbitrevb register form
+define <64 x i8> @bitrev_zmm_unaligned_load(ptr %ptr) {
+; CHECK-LABEL: bitrev_zmm_unaligned_load:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vmovdqu64 (%rdi), %zmm0 # encoding: [0x62,0xf1,0xfe,0x48,0x6f,0x07]
+; CHECK-NEXT: vpbitrevb %zmm0, %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x81,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <64 x i8>, ptr %ptr, align 1
+ %1 = tail call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %0)
+ ret <64 x i8> %1
+}
+
+; Test 256-bit with unaligned memory load
+; Unaligned loads must use vmovdqu + vpbitrevb register form
+define <32 x i8> @bitrev_ymm_unaligned_load(ptr %ptr) {
+; CHECK-LABEL: bitrev_ymm_unaligned_load:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vmovdqu (%rdi), %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfe,0x6f,0x07]
+; CHECK-NEXT: vpbitrevb %ymm0, %ymm0 # encoding: [0x62,0xf6,0x7e,0x28,0x81,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <32 x i8>, ptr %ptr, align 1
+ %1 = tail call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %0)
+ ret <32 x i8> %1
+}
+
+; Test 128-bit with unaligned memory load
+; Unaligned loads must use vmovdqu + vpbitrevb register form
+define <16 x i8> @bitrev_xmm_unaligned_load(ptr %ptr) {
+; CHECK-LABEL: bitrev_xmm_unaligned_load:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vmovdqu (%rdi), %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x6f,0x07]
+; CHECK-NEXT: vpbitrevb %xmm0, %xmm0 # encoding: [0x62,0xf6,0x7e,0x08,0x81,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <16 x i8>, ptr %ptr, align 1
+ %1 = tail call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %0)
+ ret <16 x i8> %1
+}
+
+declare <64 x i8> @llvm.bitreverse.v64i8(<64 x i8>)
+declare <32 x i8> @llvm.bitreverse.v32i8(<32 x i8>)
+declare <16 x i8> @llvm.bitreverse.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics-mem.ll b/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics-mem.ll
new file mode 100644
index 0000000000000..92cf5ff4885aa
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics-mem.ll
@@ -0,0 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bmm,+avx512vl,+avx512bw --show-mc-encoding | FileCheck %s
+
+
+define <2 x i64> @test_mm128_vbitrevb_epi8_mem(ptr %ptr) {
+; CHECK-LABEL: test_mm128_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb (%rdi), %xmm0 # encoding: [0x62,0xf6,0x7e,0x08,0x81,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <16 x i8>, ptr %ptr, align 16
+ %1 = tail call <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8> %0)
+ %2 = bitcast <16 x i8> %1 to <2 x i64>
+ ret <2 x i64> %2
+}
+
+define <2 x i64> @test_mm128_mask_vbitrevb_epi8_mem(<2 x i64> %src, i16 %mask, ptr %ptr) {
+; CHECK-LABEL: test_mm128_mask_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT: vpbitrevb (%rsi), %xmm0 {%k1} # encoding: [0x62,0xf6,0x7e,0x09,0x81,0x06]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <16 x i8>, ptr %ptr, align 16
+ %1 = tail call <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8> %0)
+ %2 = bitcast <2 x i64> %src to <16 x i8>
+ %3 = bitcast i16 %mask to <16 x i1>
+ %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2
+ %5 = bitcast <16 x i8> %4 to <2 x i64>
+ ret <2 x i64> %5
+}
+
+define <2 x i64> @test_mm128_maskz_vbitrevb_epi8_mem(i16 %mask, ptr %ptr) {
+; CHECK-LABEL: test_mm128_maskz_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT: vpbitrevb (%rsi), %xmm0 {%k1} {z} # encoding: [0x62,0xf6,0x7e,0x89,0x81,0x06]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <16 x i8>, ptr %ptr, align 16
+ %1 = tail call <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8> %0)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+ %4 = bitcast <16 x i8> %3 to <2 x i64>
+ ret <2 x i64> %4
+}
+
+define <4 x i64> @test_mm256_vbitrevb_epi8_mem(ptr %ptr) {
+; CHECK-LABEL: test_mm256_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb (%rdi), %ymm0 # encoding: [0x62,0xf6,0x7e,0x28,0x81,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <32 x i8>, ptr %ptr, align 32
+ %1 = tail call <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8> %0)
+ %2 = bitcast <32 x i8> %1 to <4 x i64>
+ ret <4 x i64> %2
+}
+
+define <4 x i64> @test_mm256_mask_vbitrevb_epi8_mem(<4 x i64> %src, i32 %mask, ptr %ptr) {
+; CHECK-LABEL: test_mm256_mask_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT: vpbitrevb (%rsi), %ymm0 {%k1} # encoding: [0x62,0xf6,0x7e,0x29,0x81,0x06]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <32 x i8>, ptr %ptr, align 32
+ %1 = tail call <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8> %0)
+ %2 = bitcast <4 x i64> %src to <32 x i8>
+ %3 = bitcast i32 %mask to <32 x i1>
+ %4 = select <32 x i1> %3, <32 x i8> %1, <32 x i8> %2
+ %5 = bitcast <32 x i8> %4 to <4 x i64>
+ ret <4 x i64> %5
+}
+
+define <4 x i64> @test_mm256_maskz_vbitrevb_epi8_mem(i32 %mask, ptr %ptr) {
+; CHECK-LABEL: test_mm256_maskz_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT: vpbitrevb (%rsi), %ymm0 {%k1} {z} # encoding: [0x62,0xf6,0x7e,0xa9,0x81,0x06]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <32 x i8>, ptr %ptr, align 32
+ %1 = tail call <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8> %0)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+ %4 = bitcast <32 x i8> %3 to <4 x i64>
+ ret <4 x i64> %4
+}
+
+define <8 x i64> @test_mm512_vbitrevb_epi8_mem(ptr %ptr) {
+; CHECK-LABEL: test_mm512_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb (%rdi), %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x81,0x07]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <64 x i8>, ptr %ptr, align 64
+ %1 = tail call <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8> %0)
+ %2 = bitcast <64 x i8> %1 to <8 x i64>
+ ret <8 x i64> %2
+}
+
+define <8 x i64> @test_mm512_mask_vbitrevb_epi8_mem(<8 x i64> %src, i64 %mask, ptr %ptr) {
+; CHECK-LABEL: test_mm512_mask_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; CHECK-NEXT: vpbitrevb (%rsi), %zmm0 {%k1} # encoding: [0x62,0xf6,0x7e,0x49,0x81,0x06]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <64 x i8>, ptr %ptr, align 64
+ %1 = tail call <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8> %0)
+ %2 = bitcast <8 x i64> %src to <64 x i8>
+ %3 = bitcast i64 %mask to <64 x i1>
+ %4 = select <64 x i1> %3, <64 x i8> %1, <64 x i8> %2
+ %5 = bitcast <64 x i8> %4 to <8 x i64>
+ ret <8 x i64> %5
+}
+
+define <8 x i64> @test_mm512_maskz_vbitrevb_epi8_mem(i64 %mask, ptr %ptr) {
+; CHECK-LABEL: test_mm512_maskz_vbitrevb_epi8_mem:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; CHECK-NEXT: vpbitrevb (%rsi), %zmm0 {%k1} {z} # encoding: [0x62,0xf6,0x7e,0xc9,0x81,0x06]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = load <64 x i8>, ptr %ptr, align 64
+ %1 = tail call <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8> %0)
+ %2 = bitcast i64 %mask to <64 x i1>
+ %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> zeroinitializer
+ %4 = bitcast <64 x i8> %3 to <8 x i64>
+ ret <8 x i64> %4
+}
+
+declare <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8>)
+
+declare <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8>)
+
+declare <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8>)
+
+
+
diff --git a/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics.ll
new file mode 100644
index 0000000000000..ac182e5d267ac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512bmm-vbitrevb-intrinsics.ll
@@ -0,0 +1,222 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bmm,+avx512vl --show-mc-encoding | FileCheck %s
+
+define <2 x i64> @test_mm128_vbitrev_epi8(<2 x i64> %a) {
+; CHECK-LABEL: test_mm128_vbitrev_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb %xmm0, %xmm0 # encoding: [0x62,0xf6,0x7e,0x08,0x81,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <2 x i64> %a to <16 x i8>
+ %1 = tail call <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8> %0)
+ %2 = bitcast <16 x i8> %1 to <2 x i64>
+ ret <2 x i64> %2
+}
+
+define <4 x i64> @test_mm256_vbitrev_epi8(<4 x i64> %a) {
+; CHECK-LABEL: test_mm256_vbitrev_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb %ymm0, %ymm0 # encoding: [0x62,0xf6,0x7e,0x28,0x81,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <4 x i64> %a to <32 x i8>
+ %1 = tail call <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8> %0)
+ %2 = bitcast <32 x i8> %1 to <4 x i64>
+ ret <4 x i64> %2
+}
+
+define <8 x i64> @test_mm512_vbitrev_epi8(<8 x i64> %a) {
+; CHECK-LABEL: test_mm512_vbitrev_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb %zmm0, %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x81,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <8 x i64> %a to <64 x i8>
+ %1 = tail call <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8> %0)
+ %2 = bitcast <64 x i8> %1 to <8 x i64>
+ ret <8 x i64> %2
+}
+
+define <4 x float> @test_mm128_mask_vbitrevb_epi8(<2 x i64> %a, i64 %mask, <2 x i64> %b) {
+; CHECK-LABEL: test_mm128_mask_vbitrevb_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb %xmm1, %xmm1 # encoding: [0x62,0xf6,0x7e,0x08,0x81,0xc9]
+; CHECK-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT: vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z} # encoding: [0x62,0xf3,0x6d,0xc9,0x25,0xd2,0xff]
+; CHECK-NEXT: # zmm2 {%k1} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm2, %xmm2 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xd2]
+; CHECK-NEXT: vpternlogq $216, %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf3,0xf5,0x08,0x25,0xc2,0xd8]
+; CHECK-NEXT: # xmm0 = xmm0 ^ (xmm2 & (xmm0 ^ xmm1))
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %conv = trunc i64 %mask to i16
+ %0 = bitcast <2 x i64> %b to <16 x i8>
+ %1 = tail call <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8> %0)
+ %2 = bitcast <2 x i64> %a to <16 x i8>
+ %3 = bitcast i16 %conv to <16 x i1>
+ %4 = select <16 x i1> %3, <16 x i8> %1, <16 x i8> %2
+ %5 = bitcast <16 x i8> %4 to <4 x float>
+ ret <4 x float> %5
+}
+
+define <8 x float> @test_mm256_mask_vbitrevb_epi8(<4 x i64> %a, i64 %mask, <4 x i64> %b) {
+; CHECK-LABEL: test_mm256_mask_vbitrevb_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT: movl %edi, %eax # encoding: [0x89,0xf8]
+; CHECK-NEXT: shrl $16, %eax # encoding: [0xc1,0xe8,0x10]
+; CHECK-NEXT: vpbitrevb %ymm1, %ymm1 # encoding: [0x62,0xf6,0x7e,0x28,0x81,0xc9]
+; CHECK-NEXT: kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
+; CHECK-NEXT: vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z} # encoding: [0x62,0xf3,0x6d,0xc9,0x25,0xd2,0xff]
+; CHECK-NEXT: # zmm2 {%k1} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm2, %xmm2 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xd2]
+; CHECK-NEXT: vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k2} {z} # encoding: [0x62,0xf3,0x65,0xca,0x25,0xdb,0xff]
+; CHECK-NEXT: # zmm3 {%k2} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm3, %xmm3 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xdb]
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x6d,0x38,0xd3,0x01]
+; CHECK-NEXT: vpternlogq $216, %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf3,0xf5,0x28,0x25,0xc2,0xd8]
+; CHECK-NEXT: # ymm0 = ymm0 ^ (ymm2 & (ymm0 ^ ymm1))
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %conv = trunc i64 %mask to i32
+ %0 = bitcast <4 x i64> %b to <32 x i8>
+ %1 = tail call <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8> %0)
+ %2 = bitcast <4 x i64> %a to <32 x i8>
+ %3 = bitcast i32 %conv to <32 x i1>
+ %4 = select <32 x i1> %3, <32 x i8> %1, <32 x i8> %2
+ %5 = bitcast <32 x i8> %4 to <8 x float>
+ ret <8 x float> %5
+}
+
+define <8 x i64> @test_mm512_mask_vbitrevb_epi8(<8 x i64> %a, i64 %mask, <8 x i64> %b) {
+; CHECK-LABEL: test_mm512_mask_vbitrevb_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq %rdi, %rax # encoding: [0x48,0x89,0xf8]
+; CHECK-NEXT: movl %edi, %ecx # encoding: [0x89,0xf9]
+; CHECK-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT: shrq $32, %rdi # encoding: [0x48,0xc1,0xef,0x20]
+; CHECK-NEXT: shrq $48, %rax # encoding: [0x48,0xc1,0xe8,0x30]
+; CHECK-NEXT: shrl $16, %ecx # encoding: [0xc1,0xe9,0x10]
+; CHECK-NEXT: vpbitrevb %zmm1, %zmm1 # encoding: [0x62,0xf6,0x7e,0x48,0x81,0xc9]
+; CHECK-NEXT: kmovw %ecx, %k2 # encoding: [0xc5,0xf8,0x92,0xd1]
+; CHECK-NEXT: kmovw %eax, %k3 # encoding: [0xc5,0xf8,0x92,0xd8]
+; CHECK-NEXT: kmovw %edi, %k4 # encoding: [0xc5,0xf8,0x92,0xe7]
+; CHECK-NEXT: vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k4} {z} # encoding: [0x62,0xf3,0x6d,0xcc,0x25,0xd2,0xff]
+; CHECK-NEXT: # zmm2 {%k4} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm2, %xmm2 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xd2]
+; CHECK-NEXT: vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k3} {z} # encoding: [0x62,0xf3,0x65,0xcb,0x25,0xdb,0xff]
+; CHECK-NEXT: # zmm3 {%k3} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm3, %xmm3 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xdb]
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x6d,0x38,0xd3,0x01]
+; CHECK-NEXT: vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k1} {z} # encoding: [0x62,0xf3,0x65,0xc9,0x25,0xdb,0xff]
+; CHECK-NEXT: # zmm3 {%k1} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm3, %xmm3 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xdb]
+; CHECK-NEXT: vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k2} {z} # encoding: [0x62,0xf3,0x5d,0xca,0x25,0xe4,0xff]
+; CHECK-NEXT: # zmm4 {%k2} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm4, %xmm4 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xe4]
+; CHECK-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x65,0x38,0xdc,0x01]
+; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2 # encoding: [0x62,0xf3,0xe5,0x48,0x3a,0xd2,0x01]
+; CHECK-NEXT: vpternlogq $216, %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf3,0xf5,0x48,0x25,0xc2,0xd8]
+; CHECK-NEXT: # zmm0 = zmm0 ^ (zmm2 & (zmm0 ^ zmm1))
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <8 x i64> %b to <64 x i8>
+ %1 = tail call <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8> %0)
+ %2 = bitcast <8 x i64> %a to <64 x i8>
+ %3 = bitcast i64 %mask to <64 x i1>
+ %4 = select <64 x i1> %3, <64 x i8> %1, <64 x i8> %2
+ %5 = bitcast <64 x i8> %4 to <8 x i64>
+ ret <8 x i64> %5
+}
+
+define <4 x float> @test_mm128_maskz_vbitrevb_epi8(i64 %mask, <2 x i64> %b) {
+; CHECK-LABEL: test_mm128_maskz_vbitrevb_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vpbitrevb %xmm0, %xmm0 # encoding: [0x62,0xf6,0x7e,0x08,0x81,0xc0]
+; CHECK-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xc9,0x25,0xc9,0xff]
+; CHECK-NEXT: # zmm1 {%k1} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm1, %xmm1 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc9]
+; CHECK-NEXT: vpand %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xdb,0xc0]
+; CHECK-NEXT: vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %conv = trunc i64 %mask to i16
+ %0 = bitcast <2 x i64> %b to <16 x i8>
+ %1 = tail call <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8> %0)
+ %2 = bitcast i16 %conv to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+ %4 = bitcast <16 x i8> %3 to <4 x float>
+ ret <4 x float> %4
+}
+
+define <8 x float> @test_mm256_maskz_vbitrevb_epi8(i64 %mask, <4 x i64> %b) {
+; CHECK-LABEL: test_mm256_maskz_vbitrevb_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT: movl %edi, %eax # encoding: [0x89,0xf8]
+; CHECK-NEXT: shrl $16, %eax # encoding: [0xc1,0xe8,0x10]
+; CHECK-NEXT: vpbitrevb %ymm0, %ymm0 # encoding: [0x62,0xf6,0x7e,0x28,0x81,0xc0]
+; CHECK-NEXT: kmovw %eax, %k2 # encoding: [0xc5,0xf8,0x92,0xd0]
+; CHECK-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z} # encoding: [0x62,0xf3,0x75,0xc9,0x25,0xc9,0xff]
+; CHECK-NEXT: # zmm1 {%k1} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm1, %xmm1 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc9]
+; CHECK-NEXT: vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k2} {z} # encoding: [0x62,0xf3,0x6d,0xca,0x25,0xd2,0xff]
+; CHECK-NEXT: # zmm2 {%k2} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm2, %xmm2 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xd2]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x75,0x38,0xca,0x01]
+; CHECK-NEXT: vpand %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xdb,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %conv = trunc i64 %mask to i32
+ %0 = bitcast <4 x i64> %b to <32 x i8>
+ %1 = tail call <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8> %0)
+ %2 = bitcast i32 %conv to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+ %4 = bitcast <32 x i8> %3 to <8 x float>
+ ret <8 x float> %4
+}
+
+define <8 x i64> @test_mm512_maskz_vbitrevb_epi8(i64 %mask, <8 x i64> %b) {
+; CHECK-LABEL: test_mm512_maskz_vbitrevb_epi8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movq %rdi, %rax # encoding: [0x48,0x89,0xf8]
+; CHECK-NEXT: movl %edi, %ecx # encoding: [0x89,0xf9]
+; CHECK-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; CHECK-NEXT: shrq $32, %rdi # encoding: [0x48,0xc1,0xef,0x20]
+; CHECK-NEXT: shrq $48, %rax # encoding: [0x48,0xc1,0xe8,0x30]
+; CHECK-NEXT: shrl $16, %ecx # encoding: [0xc1,0xe9,0x10]
+; CHECK-NEXT: vpbitrevb %zmm0, %zmm0 # encoding: [0x62,0xf6,0x7e,0x48,0x81,0xc0]
+; CHECK-NEXT: kmovw %ecx, %k2 # encoding: [0xc5,0xf8,0x92,0xd1]
+; CHECK-NEXT: kmovw %eax, %k3 # encoding: [0xc5,0xf8,0x92,0xd8]
+; CHECK-NEXT: kmovw %edi, %k4 # encoding: [0xc5,0xf8,0x92,0xe7]
+; CHECK-NEXT: vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k4} {z} # encoding: [0x62,0xf3,0x75,0xcc,0x25,0xc9,0xff]
+; CHECK-NEXT: # zmm1 {%k4} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm1, %xmm1 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xc9]
+; CHECK-NEXT: vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k3} {z} # encoding: [0x62,0xf3,0x6d,0xcb,0x25,0xd2,0xff]
+; CHECK-NEXT: # zmm2 {%k3} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm2, %xmm2 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xd2]
+; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x75,0x38,0xca,0x01]
+; CHECK-NEXT: vpternlogd $255, %zmm2, %zmm2, %zmm2 {%k1} {z} # encoding: [0x62,0xf3,0x6d,0xc9,0x25,0xd2,0xff]
+; CHECK-NEXT: # zmm2 {%k1} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm2, %xmm2 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xd2]
+; CHECK-NEXT: vpternlogd $255, %zmm3, %zmm3, %zmm3 {%k2} {z} # encoding: [0x62,0xf3,0x65,0xca,0x25,0xdb,0xff]
+; CHECK-NEXT: # zmm3 {%k2} {z} = -1
+; CHECK-NEXT: vpmovdb %zmm3, %xmm3 # encoding: [0x62,0xf2,0x7e,0x48,0x31,0xdb]
+; CHECK-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x6d,0x38,0xd3,0x01]
+; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1 # encoding: [0x62,0xf3,0xed,0x48,0x3a,0xc9,0x01]
+; CHECK-NEXT: vpandq %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0xdb,0xc0]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <8 x i64> %b to <64 x i8>
+ %1 = tail call <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8> %0)
+ %2 = bitcast i64 %mask to <64 x i1>
+ %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> zeroinitializer
+ %4 = bitcast <64 x i8> %3 to <8 x i64>
+ ret <8 x i64> %4
+}
+
+declare <16 x i8> @llvm.x86.avx512.vbitrevb.128(<16 x i8>)
+declare <32 x i8> @llvm.x86.avx512.vbitrevb.256(<32 x i8>)
+declare <64 x i8> @llvm.x86.avx512.vbitrevb.512(<64 x i8>)
diff --git a/llvm/test/CodeGen/X86/avx512bmm-vbmac-intrinsics.ll b/llvm/test/CodeGen/X86/avx512bmm-vbmac-intrinsics.ll
new file mode 100644
index 0000000000000..231ef1a5a351d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512bmm-vbmac-intrinsics.ll
@@ -0,0 +1,63 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bmm,+avx512vl --show-mc-encoding | FileCheck %s
+
+define <4 x i64> @test_mm256_vbmacor(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
+; CHECK-LABEL: test_mm256_vbmacor:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vbmacor16x16x16 %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf6,0x74,0x28,0x80,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <4 x i64> %a to <16 x i16>
+ %1 = bitcast <4 x i64> %b to <16 x i16>
+ %2 = bitcast <4 x i64> %c to <16 x i16>
+ %3 = tail call <16 x i16> @llvm.x86.avx512.vbmacor.v16hi(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2)
+ %4 = bitcast <16 x i16> %3 to <4 x i64>
+ ret <4 x i64> %4
+}
+
+define <4 x i64> @test_mm256_vbmacxor(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) {
+; CHECK-LABEL: test_mm256_vbmacxor:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vbmacxor16x16x16 %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf6,0xf4,0x28,0x80,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <4 x i64> %a to <16 x i16>
+ %1 = bitcast <4 x i64> %b to <16 x i16>
+ %2 = bitcast <4 x i64> %c to <16 x i16>
+ %3 = tail call <16 x i16> @llvm.x86.avx512.vbmacxor.v16hi(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2)
+ %4 = bitcast <16 x i16> %3 to <4 x i64>
+ ret <4 x i64> %4
+}
+
+define <8 x i64> @test_mm512_vbmacor(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
+; CHECK-LABEL: test_mm512_vbmacor:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vbmacor16x16x16 %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf6,0x74,0x48,0x80,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <8 x i64> %a to <32 x i16>
+ %1 = bitcast <8 x i64> %b to <32 x i16>
+ %2 = bitcast <8 x i64> %c to <32 x i16>
+ %3 = tail call <32 x i16> @llvm.x86.avx512.vbmacor.v32hi(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)
+ %4 = bitcast <32 x i16> %3 to <8 x i64>
+ ret <8 x i64> %4
+}
+
+define <8 x i64> @test_mm512_vbmacxor(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c) {
+; CHECK-LABEL: test_mm512_vbmacxor:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: vbmacxor16x16x16 %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf6,0xf4,0x48,0x80,0xc2]
+; CHECK-NEXT: retq # encoding: [0xc3]
+entry:
+ %0 = bitcast <8 x i64> %a to <32 x i16>
+ %1 = bitcast <8 x i64> %b to <32 x i16>
+ %2 = bitcast <8 x i64> %c to <32 x i16>
+ %3 = tail call <32 x i16> @llvm.x86.avx512.vbmacxor.v32hi(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)
+ %4 = bitcast <32 x i16> %3 to <8 x i64>
+ ret <8 x i64> %4
+}
+
+declare <16 x i16> @llvm.x86.avx512.vbmacor.v16hi(<16 x i16>, <16 x i16>, <16 x i16>)
+declare <16 x i16> @llvm.x86.avx512.vbmacxor.v16hi(<16 x i16>, <16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.x86.avx512.vbmacor.v32hi(<32 x i16>, <32 x i16>, <32 x i16>)
+declare <32 x i16> @llvm.x86.avx512.vbmacxor.v32hi(<32 x i16>, <32 x i16>, <32 x i16>)
diff --git a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
index b0ca0069a526b..821b7b8e4144f 100644
--- a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
+++ b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
@@ -24,6 +24,7 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,SLOW-DIVQ
; Additional tests for 64-bit divide bypass
diff --git a/llvm/test/CodeGen/X86/cmp16.ll b/llvm/test/CodeGen/X86/cmp16.ll
index 8c14a78d9e113..ff6ee68074088 100644
--- a/llvm/test/CodeGen/X86/cmp16.ll
+++ b/llvm/test/CodeGen/X86/cmp16.ll
@@ -14,6 +14,7 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=X64,X64-FAST
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=X64,X64-FAST
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver6 | FileCheck %s --check-prefixes=X64,X64-FAST
define i1 @cmp16_reg_eq_reg(i16 %a0, i16 %a1) {
; X86-GENERIC-LABEL: cmp16_reg_eq_reg:
diff --git a/llvm/test/CodeGen/X86/cpus-amd.ll b/llvm/test/CodeGen/X86/cpus-amd.ll
index 33b2cf3731478..33cbc71b41ecd 100644
--- a/llvm/test/CodeGen/X86/cpus-amd.ll
+++ b/llvm/test/CodeGen/X86/cpus-amd.ll
@@ -30,6 +30,7 @@
; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=znver3 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=znver4 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=znver5 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
+; RUN: llc < %s -o /dev/null -mtriple=x86_64-unknown-unknown -mcpu=znver6 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ERROR --allow-empty
define void @foo() {
ret void
diff --git a/llvm/test/CodeGen/X86/rdpru.ll b/llvm/test/CodeGen/X86/rdpru.ll
index be79a4499a338..067ae31142c39 100644
--- a/llvm/test/CodeGen/X86/rdpru.ll
+++ b/llvm/test/CodeGen/X86/rdpru.ll
@@ -7,6 +7,7 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 -fast-isel | FileCheck %s --check-prefix=X64
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 -fast-isel | FileCheck %s --check-prefix=X64
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 -fast-isel | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver6 -fast-isel | FileCheck %s --check-prefix=X64
define void @rdpru_asm() {
; X86-LABEL: rdpru_asm:
diff --git a/llvm/test/CodeGen/X86/shuffle-as-shifts.ll b/llvm/test/CodeGen/X86/shuffle-as-shifts.ll
index 4b8f78d36c3f5..021f8d6fb971d 100644
--- a/llvm/test/CodeGen/X86/shuffle-as-shifts.ll
+++ b/llvm/test/CodeGen/X86/shuffle-as-shifts.ll
@@ -4,6 +4,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-V4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
define <4 x i32> @shuf_rot_v4i32_1032(<4 x i32> %x) {
diff --git a/llvm/test/CodeGen/X86/slow-unaligned-mem.ll b/llvm/test/CodeGen/X86/slow-unaligned-mem.ll
index ceef3fb4bb188..a215b60055dd5 100644
--- a/llvm/test/CodeGen/X86/slow-unaligned-mem.ll
+++ b/llvm/test/CodeGen/X86/slow-unaligned-mem.ll
@@ -51,6 +51,7 @@
; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=znver3 2>&1 | FileCheck %s --check-prefixes=FAST,FAST-AVX256
; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=znver4 2>&1 | FileCheck %s --check-prefixes=FAST,FAST-AVX512
; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=znver5 2>&1 | FileCheck %s --check-prefixes=FAST,FAST-AVX512
+; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=znver6 2>&1 | FileCheck %s --check-prefixes=FAST,FAST-AVX512
; Other chips with slow unaligned memory accesses
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll b/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll
index 74b51ac21dc1f..9d2708674c3ff 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath-tune.ll
@@ -7,6 +7,7 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=FAST-SCALAR,FAST-VECTOR
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=FAST-SCALAR,FAST-VECTOR
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s --check-prefixes=FAST-SCALAR,FAST-VECTOR
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver6 | FileCheck %s --check-prefixes=FAST-SCALAR,FAST-VECTOR
; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=X86-64
define float @f32_no_daz(float %f) #0 {
diff --git a/llvm/test/CodeGen/X86/tuning-shuffle-permilpd-avx512.ll b/llvm/test/CodeGen/X86/tuning-shuffle-permilpd-avx512.ll
index 162ab71fc00d4..e2c8b6df6e744 100644
--- a/llvm/test/CodeGen/X86/tuning-shuffle-permilpd-avx512.ll
+++ b/llvm/test/CodeGen/X86/tuning-shuffle-permilpd-avx512.ll
@@ -5,6 +5,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw,+avx512dq | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
define <8 x double> @transform_VPERMILPSZrr(<8 x double> %a) nounwind {
; CHECK-LABEL: transform_VPERMILPSZrr:
diff --git a/llvm/test/CodeGen/X86/tuning-shuffle-permilps-avx512.ll b/llvm/test/CodeGen/X86/tuning-shuffle-permilps-avx512.ll
index cd97946da248f..53bad74552f8a 100644
--- a/llvm/test/CodeGen/X86/tuning-shuffle-permilps-avx512.ll
+++ b/llvm/test/CodeGen/X86/tuning-shuffle-permilps-avx512.ll
@@ -5,6 +5,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw,+avx512dq | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
define <16 x float> @transform_VPERMILPSZrr(<16 x float> %a) nounwind {
; CHECK-LABEL: transform_VPERMILPSZrr:
diff --git a/llvm/test/CodeGen/X86/tuning-shuffle-unpckpd-avx512.ll b/llvm/test/CodeGen/X86/tuning-shuffle-unpckpd-avx512.ll
index 5ea991f85523e..39a072eeeea4c 100644
--- a/llvm/test/CodeGen/X86/tuning-shuffle-unpckpd-avx512.ll
+++ b/llvm/test/CodeGen/X86/tuning-shuffle-unpckpd-avx512.ll
@@ -6,6 +6,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw,+avx512dq | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
define <16 x float> @transform_VUNPCKLPDZrr(<16 x float> %a, <16 x float> %b) nounwind {
diff --git a/llvm/test/CodeGen/X86/tuning-shuffle-unpckps-avx512.ll b/llvm/test/CodeGen/X86/tuning-shuffle-unpckps-avx512.ll
index 96155f0300d2d..f8b9dac4c7ba8 100644
--- a/llvm/test/CodeGen/X86/tuning-shuffle-unpckps-avx512.ll
+++ b/llvm/test/CodeGen/X86/tuning-shuffle-unpckps-avx512.ll
@@ -6,6 +6,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl,+avx512bw,+avx512dq | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,CHECK-ZNVER4
define <16 x float> @transform_VUNPCKLPSZrr(<16 x float> %a, <16 x float> %b) nounwind {
; CHECK-LABEL: transform_VUNPCKLPSZrr:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-fast-per-lane.ll b/llvm/test/CodeGen/X86/vector-shuffle-fast-per-lane.ll
index 4021b1bf292bb..5bf936c6e5cec 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-fast-per-lane.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-fast-per-lane.ll
@@ -9,6 +9,7 @@
; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=znver3 | FileCheck %s --check-prefixes=FAST
; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=FAST
; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=FAST
; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=haswell | FileCheck %s --check-prefixes=FAST
; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=skx | FileCheck %s --check-prefixes=FAST
diff --git a/llvm/test/CodeGen/X86/vpdpwssd.ll b/llvm/test/CodeGen/X86/vpdpwssd.ll
index 2ac2b48af4ce7..ea97800505bc2 100644
--- a/llvm/test/CodeGen/X86/vpdpwssd.ll
+++ b/llvm/test/CodeGen/X86/vpdpwssd.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver4 | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver5 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver6 | FileCheck %s --check-prefixes=CHECK,AVX-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512-VNNI
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+fast-dpwssd | FileCheck %s --check-prefixes=CHECK,AVX512VL-VNNI
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
index c5e879c0135f4..bb1a4e5fcb75b 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
@@ -18,6 +18,7 @@
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=BMI2-FAST
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=BMI2-FAST
; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s --check-prefixes=BMI2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver6 | FileCheck %s --check-prefixes=BMI2-FAST
; Verify that for the X86_64 processors that are known to have poor latency
; double precision shift instructions we do not generate 'shld' or 'shrd'
diff --git a/llvm/test/MC/X86/x86_long_nop.s b/llvm/test/MC/X86/x86_long_nop.s
index b79403bb5f1ec..2c5fe3acde26c 100644
--- a/llvm/test/MC/X86/x86_long_nop.s
+++ b/llvm/test/MC/X86/x86_long_nop.s
@@ -21,6 +21,8 @@
# RUN: llvm-mc -filetype=obj -arch=x86 -triple=i686-pc-linux-gnu %s -mcpu=znver4 | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=LNOP15
# RUN: llvm-mc -filetype=obj -arch=x86 -triple=x86_64-pc-linux-gnu -mcpu=znver5 %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=LNOP15
# RUN: llvm-mc -filetype=obj -arch=x86 -triple=i686-pc-linux-gnu %s -mcpu=znver5 | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=LNOP15
+# RUN: llvm-mc -filetype=obj -arch=x86 -triple=x86_64-pc-linux-gnu -mcpu=znver6 %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=LNOP15
+# RUN: llvm-mc -filetype=obj -arch=x86 -triple=i686-pc-linux-gnu %s -mcpu=znver6 | llvm-objdump -d --no-show-raw-insn - | FileCheck %s --check-prefix=LNOP15
# RUN: llvm-mc -filetype=obj -arch=x86 -triple=i686-pc-linux-gnu -mcpu=nehalem %s | llvm-objdump -d --no-show-raw-insn - | FileCheck --check-prefix=LNOP10 %s
# RUN: llvm-mc -filetype=obj -arch=x86 -triple=i686-pc-linux-gnu -mcpu=westmere %s | llvm-objdump -d --no-show-raw-insn - | FileCheck --check-prefix=LNOP10 %s
# RUN: llvm-mc -filetype=obj -arch=x86 -triple=i686-pc-linux-gnu -mcpu=sandybridge %s | llvm-objdump -d --no-show-raw-insn - | FileCheck --check-prefix=LNOP15 %s
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index bafc98a69ddae..013d5ec2e71e7 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -1633,6 +1633,9 @@ static const X86FoldTableEntry Table1[] = {
{X86::VPABSWZ256rr, X86::VPABSWZ256rm, 0},
{X86::VPABSWZrr, X86::VPABSWZrm, 0},
{X86::VPABSWrr, X86::VPABSWrm, 0},
+ {X86::VPBITREVBZ128rr, X86::VPBITREVBZ128rm, TB_ALIGN_16},
+ {X86::VPBITREVBZ256rr, X86::VPBITREVBZ256rm, TB_ALIGN_32},
+ {X86::VPBITREVBZrr, X86::VPBITREVBZrm, TB_ALIGN_64},
{X86::VPBROADCASTBYrr, X86::VPBROADCASTBYrm, TB_NO_REVERSE},
{X86::VPBROADCASTBZ128rr, X86::VPBROADCASTBZ128rm, TB_NO_REVERSE},
{X86::VPBROADCASTBZ256rr, X86::VPBROADCASTBZ256rm, TB_NO_REVERSE},
@@ -3310,6 +3313,9 @@ static const X86FoldTableEntry Table2[] = {
{X86::VPAVGWZ256rr, X86::VPAVGWZ256rm, 0},
{X86::VPAVGWZrr, X86::VPAVGWZrm, 0},
{X86::VPAVGWrr, X86::VPAVGWrm, 0},
+ {X86::VPBITREVBZ128rrkz, X86::VPBITREVBZ128rmkz, TB_ALIGN_16},
+ {X86::VPBITREVBZ256rrkz, X86::VPBITREVBZ256rmkz, TB_ALIGN_32},
+ {X86::VPBITREVBZrrkz, X86::VPBITREVBZrmkz, TB_ALIGN_64},
{X86::VPBLENDDYrri, X86::VPBLENDDYrmi, 0},
{X86::VPBLENDDrri, X86::VPBLENDDrmi, 0},
{X86::VPBLENDMBZ128rr, X86::VPBLENDMBZ128rm, 0},
@@ -4266,6 +4272,10 @@ static const X86FoldTableEntry Table3[] = {
{X86::VBLENDMPSZ128rrk, X86::VBLENDMPSZ128rmk, 0},
{X86::VBLENDMPSZ256rrk, X86::VBLENDMPSZ256rmk, 0},
{X86::VBLENDMPSZrrk, X86::VBLENDMPSZrmk, 0},
+ {X86::VBMACORZ256rr, X86::VBMACORZ256rm, 0},
+ {X86::VBMACORZrr, X86::VBMACORZrm, 0},
+ {X86::VBMACXORZ256rr, X86::VBMACXORZ256rm, 0},
+ {X86::VBMACXORZrr, X86::VBMACXORZrm, 0},
{X86::VBROADCASTF32X2Z256rrk, X86::VBROADCASTF32X2Z256rmk, TB_NO_REVERSE},
{X86::VBROADCASTF32X2Zrrk, X86::VBROADCASTF32X2Zrmk, TB_NO_REVERSE},
{X86::VBROADCASTI32X2Z128rrk, X86::VBROADCASTI32X2Z128rmk, TB_NO_REVERSE},
@@ -5284,6 +5294,9 @@ static const X86FoldTableEntry Table3[] = {
{X86::VPAVGWZ128rrkz, X86::VPAVGWZ128rmkz, 0},
{X86::VPAVGWZ256rrkz, X86::VPAVGWZ256rmkz, 0},
{X86::VPAVGWZrrkz, X86::VPAVGWZrmkz, 0},
+ {X86::VPBITREVBZ128rrk, X86::VPBITREVBZ128rmk, TB_ALIGN_16},
+ {X86::VPBITREVBZ256rrk, X86::VPBITREVBZ256rmk, TB_ALIGN_32},
+ {X86::VPBITREVBZrrk, X86::VPBITREVBZrmk, TB_ALIGN_64},
{X86::VPBLENDMBZ128rrk, X86::VPBLENDMBZ128rmk, 0},
{X86::VPBLENDMBZ256rrk, X86::VPBLENDMBZ256rmk, 0},
{X86::VPBLENDMBZrrk, X86::VPBLENDMBZrmk, 0},
@@ -6110,6 +6123,14 @@ static const X86FoldTableEntry Table4[] = {
{X86::VANDPSZ128rrk, X86::VANDPSZ128rmk, 0},
{X86::VANDPSZ256rrk, X86::VANDPSZ256rmk, 0},
{X86::VANDPSZrrk, X86::VANDPSZrmk, 0},
+ {X86::VBMACORZ256rrk, X86::VBMACORZ256rmk, 0},
+ {X86::VBMACORZ256rrkz, X86::VBMACORZ256rmkz, 0},
+ {X86::VBMACORZrrk, X86::VBMACORZrmk, 0},
+ {X86::VBMACORZrrkz, X86::VBMACORZrmkz, 0},
+ {X86::VBMACXORZ256rrk, X86::VBMACXORZ256rmk, 0},
+ {X86::VBMACXORZ256rrkz, X86::VBMACXORZ256rmkz, 0},
+ {X86::VBMACXORZrrk, X86::VBMACXORZrmk, 0},
+ {X86::VBMACXORZrrkz, X86::VBMACXORZrmkz, 0},
{X86::VCVT2PH2BF8SZ128rrk, X86::VCVT2PH2BF8SZ128rmk, 0},
{X86::VCVT2PH2BF8SZ256rrk, X86::VCVT2PH2BF8SZ256rmk, 0},
{X86::VCVT2PH2BF8SZrrk, X86::VCVT2PH2BF8SZrmk, 0},
@@ -8674,6 +8695,10 @@ static const X86FoldTableEntry BroadcastTable3[] = {
{X86::VBLENDMPSZ128rrk, X86::VBLENDMPSZ128rmbk, TB_BCAST_SS},
{X86::VBLENDMPSZ256rrk, X86::VBLENDMPSZ256rmbk, TB_BCAST_SS},
{X86::VBLENDMPSZrrk, X86::VBLENDMPSZrmbk, TB_BCAST_SS},
+ {X86::VBMACORZ256rr, X86::VBMACORZ256rmb, TB_BCAST_W},
+ {X86::VBMACORZrr, X86::VBMACORZrmb, TB_BCAST_W},
+ {X86::VBMACXORZ256rr, X86::VBMACXORZ256rmb, TB_BCAST_W},
+ {X86::VBMACXORZrr, X86::VBMACXORZrmb, TB_BCAST_W},
{X86::VCMPBF16Z128rrik, X86::VCMPBF16Z128rmbik, TB_BCAST_SH},
{X86::VCMPBF16Z256rrik, X86::VCMPBF16Z256rmbik, TB_BCAST_SH},
{X86::VCMPBF16Zrrik, X86::VCMPBF16Zrmbik, TB_BCAST_SH},
@@ -9786,6 +9811,14 @@ static const X86FoldTableEntry BroadcastTable4[] = {
{X86::VANDPSZ128rrk, X86::VANDPSZ128rmbk, TB_BCAST_SS},
{X86::VANDPSZ256rrk, X86::VANDPSZ256rmbk, TB_BCAST_SS},
{X86::VANDPSZrrk, X86::VANDPSZrmbk, TB_BCAST_SS},
+ {X86::VBMACORZ256rrk, X86::VBMACORZ256rmbk, TB_BCAST_W},
+ {X86::VBMACORZ256rrkz, X86::VBMACORZ256rmbkz, TB_BCAST_W},
+ {X86::VBMACORZrrk, X86::VBMACORZrmbk, TB_BCAST_W},
+ {X86::VBMACORZrrkz, X86::VBMACORZrmbkz, TB_BCAST_W},
+ {X86::VBMACXORZ256rrk, X86::VBMACXORZ256rmbk, TB_BCAST_W},
+ {X86::VBMACXORZ256rrkz, X86::VBMACXORZ256rmbkz, TB_BCAST_W},
+ {X86::VBMACXORZrrk, X86::VBMACXORZrmbk, TB_BCAST_W},
+ {X86::VBMACXORZrrkz, X86::VBMACXORZrmbkz, TB_BCAST_W},
{X86::VCVT2PH2BF8SZ128rrk, X86::VCVT2PH2BF8SZ128rmbk, TB_BCAST_SH},
{X86::VCVT2PH2BF8SZ256rrk, X86::VCVT2PH2BF8SZ256rmbk, TB_BCAST_SH},
{X86::VCVT2PH2BF8SZrrk, X86::VCVT2PH2BF8SZrmbk, TB_BCAST_SH},
diff --git a/llvm/test/Transforms/LoopUnroll/X86/call-remark.ll b/llvm/test/Transforms/LoopUnroll/X86/call-remark.ll
index b0f4385b7913d..f9768141e5d9c 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/call-remark.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/call-remark.ll
@@ -2,6 +2,7 @@
; RUN: opt -passes=debugify,loop-unroll -mcpu=znver3 -pass-remarks=TTI -pass-remarks-analysis=TTI < %s -S 2>&1 | FileCheck --check-prefixes=ALL,TTI %s
; RUN: opt -passes=debugify,loop-unroll -mcpu=znver4 -pass-remarks=loop-unroll -pass-remarks-analysis=loop-unroll < %s -S 2>&1 | FileCheck --check-prefixes=ALL,UNROLL %s
; RUN: opt -passes=debugify,loop-unroll -mcpu=znver5 -pass-remarks=loop-unroll -pass-remarks-analysis=loop-unroll < %s -S 2>&1 | FileCheck --check-prefixes=ALL,UNROLL %s
+; RUN: opt -passes=debugify,loop-unroll -mcpu=znver6 -pass-remarks=loop-unroll -pass-remarks-analysis=loop-unroll < %s -S 2>&1 | FileCheck --check-prefixes=ALL,UNROLL %s
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr63668.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr63668.ll
index 037e073de9d59..a1ee268392d0e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr63668.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr63668.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 -S < %s | FileCheck %s
; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=znver5 -S < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=znver6 -S < %s | FileCheck %s
define internal i32 @testfunc() {
; CHECK-LABEL: define internal i32 @testfunc
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index cbb7f89bee679..31481eb40f249 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -34,7 +34,8 @@ struct ManualMapEntry {
// List of instructions requiring explicitly aligned memory.
static constexpr const char *ExplicitAlign[] = {
- "MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", "MOVNTPD", "MOVNTDQ", "MOVNTDQA"};
+ "MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS",
+ "MOVNTPD", "MOVNTDQ", "MOVNTDQA", "VPBITREVB"};
// List of instructions NOT requiring explicit memory alignment.
static constexpr const char *ExplicitUnalign[] = {
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 3087744f694c7..c65bec65b6c66 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -162,6 +162,8 @@ copy("Headers") {
"avx512bf16intrin.h",
"avx512bitalgintrin.h",
"avx512bwintrin.h",
+ "avx512bmmintrin.h"
+ "avx512bmmvlintrin.h"
"avx512cdintrin.h",
"avx512dqintrin.h",
"avx512fintrin.h",
More information about the cfe-commits
mailing list