[llvm-branch-commits] [llvm] [4/7][PISA] Add PISA GlobalISel lowering, legalization and combiners (PR #214373)
Michal Paszkowski via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 22 00:30:27 PDT 2026
https://github.com/michalpaszkowski updated https://github.com/llvm/llvm-project/pull/214373
>From df719f2fa35d9f875343b1bbff3a44baba3b09d9 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Wed, 5 Aug 2026 17:10:33 -0700
Subject: [PATCH 01/13] [PISA] Add PISA GlobalISel lowering, legalization and
combiners
Add the PISA GlobalISel lowering path: call lowering, the legalizer rules,
register-bank information and the pre/post-legalization combiners, together
with the machine-function info, frame lowering, and the target-machine
pipeline through register-bank selection.
Adds GlobalISel LIT tests for legalization and the combiners. Instruction
selection is added in the following change.
---
llvm/include/llvm/AsmParser/LLToken.h | 1 +
.../llvm/CodeGen/TargetFrameLowering.h | 1 +
llvm/include/llvm/TargetParser/CMakeLists.txt | 3 +
.../llvm/TargetParser/PISATargetParser.def | 18 +
.../llvm/TargetParser/PISATargetParser.h | 116 +
llvm/lib/AsmParser/LLLexer.cpp | 1 +
llvm/lib/AsmParser/LLParser.cpp | 1 +
llvm/lib/IR/AsmWriter.cpp | 1 +
llvm/lib/Target/PISA/CMakeLists.txt | 14 +
llvm/lib/Target/PISA/PISA.h | 63 +-
llvm/lib/Target/PISA/PISACallLowering.cpp | 758 ++++
llvm/lib/Target/PISA/PISACallLowering.h | 44 +
llvm/lib/Target/PISA/PISAFrameLowering.h | 10 +
llvm/lib/Target/PISA/PISAISelLowering.cpp | 712 ++++
llvm/lib/Target/PISA/PISAISelLowering.h | 112 +
llvm/lib/Target/PISA/PISALegalizerInfo.cpp | 3513 +++++++++++++++++
llvm/lib/Target/PISA/PISALegalizerInfo.h | 30 +
.../Target/PISA/PISAMachineFunctionInfo.cpp | 17 +
.../lib/Target/PISA/PISAMachineFunctionInfo.h | 39 +
.../Target/PISA/PISAPostLegalizerCombiner.cpp | 2005 ++++++++++
.../Target/PISA/PISAPreLegalizerCombiner.cpp | 1664 ++++++++
llvm/lib/Target/PISA/PISARegisterBankInfo.cpp | 53 +
llvm/lib/Target/PISA/PISARegisterBankInfo.h | 37 +
llvm/lib/Target/PISA/PISASubtarget.cpp | 24 +-
llvm/lib/Target/PISA/PISASubtarget.h | 46 +-
llvm/lib/Target/PISA/PISATargetMachine.cpp | 134 +-
llvm/lib/Target/PISA/PISATargetMachine.h | 30 +-
llvm/lib/Target/PISA/PISATargetObjectFile.h | 59 +
llvm/lib/TargetParser/CMakeLists.txt | 1 +
llvm/lib/TargetParser/PISATargetParser.cpp | 24 +
...ild-vector-with-constants-trunc-double.mir | 45 +
.../PISA/GlobalISel/combine-abs-iredsmax.mir | 48 +
.../GlobalISel/combine-cmp-and-all-ones.mir | 89 +
.../combine-extract-build-vector.mir | 88 +
.../PISA/GlobalISel/combine-fadd-constant.mir | 18 +
...ine-getdef-ignoring-bitcasts-novectors.mir | 74 +
.../combine-local-id-range-trunc-zext.mir | 92 +
.../combine-p2i-to-i2p-addrspace.mir | 52 +
.../combine-select-trunc-one-zero.mir | 160 +
.../GlobalISel/combine-trunc-bool-trunc.mir | 91 +
.../GlobalISel/combine-truncated-shift.mir | 56 +
.../PISA/GlobalISel/fix-illegal-shift-amt.mir | 63 +
.../legalize-load-store-subbyte-scalar.mir | 76 +
.../PISA/GlobalISel/legalize-threeway-cmp.mir | 90 +
.../PISA/GlobalISel/legalizer-16-32-vec.mir | 182 +
.../legalizer-concat-vectors-clamp-elts.mir | 40 +
.../PISA/GlobalISel/legalizer-constant.mir | 158 +
.../legalizer-extract-subvector.mir | 15 +
...alizer-extract-vector-elt-illegal-size.mir | 52 +
.../PISA/GlobalISel/legalizer-fldexp.mir | 411 ++
.../PISA/GlobalISel/legalizer-freeze-i96.mir | 48 +
.../PISA/GlobalISel/legalizer-frem.mir | 116 +
.../CodeGen/PISA/GlobalISel/legalizer-i2f.mir | 35 +
.../PISA/GlobalISel/legalizer-icmp.mir | 443 +++
.../GlobalISel/legalizer-implicit-def.mir | 367 ++
.../GlobalISel/legalizer-insert-subvector.mir | 16 +
...egalizer-load-non-power-of-2-overfetch.mir | 388 ++
.../legalizer-load-store-i1-vec.mir | 98 +
.../legalizer-load-store-non-standard-vec.mir | 54 +
...legalizer-load-store-vec-of-large-ints.mir | 84 +
.../PISA/GlobalISel/legalizer-mulh-i64.mir | 139 +
.../PISA/GlobalISel/legalizer-mulh.mir | 88 +
.../PISA/GlobalISel/legalizer-phi-s5.mir | 66 +
.../PISA/GlobalISel/legalizer-phi-s96.mir | 64 +
.../GlobalISel/legalizer-phi-scalarize.mir | 33 +
.../PISA/GlobalISel/legalizer-phi-widen.mir | 33 +
.../GlobalISel/legalizer-select-widen.mir | 38 +
.../legalizer-store-long-alignment.mir | 24 +
.../GlobalISel/legalizer-udiv128-sequence.mir | 515 +++
.../GlobalISel/legalizer-unmerge-vectors.mir | 53 +
.../PISA/GlobalISel/legalizer-zext-big.mir | 30 +
.../PISA/GlobalISel/mir-print-bfloat.mir | 21 +
.../postlegalizer-compare-select.mir | 67 +
...ostlegalizer-extract-subvector-partial.mir | 38 +
.../postlegalizer-no-commute-shift-loop.mir | 52 +
.../PISA/GlobalISel/postrapseudo-i1.mir | 29 +
.../prelegalizer-reduce-predicates.mir | 181 +
.../PISA/GlobalISel/prelegalizer-zext-and.mir | 49 +
.../PISA/GlobalISel/retain-extractvec.mir | 37 +
79 files changed, 14629 insertions(+), 8 deletions(-)
create mode 100644 llvm/include/llvm/TargetParser/PISATargetParser.def
create mode 100644 llvm/include/llvm/TargetParser/PISATargetParser.h
create mode 100644 llvm/lib/Target/PISA/PISACallLowering.cpp
create mode 100644 llvm/lib/Target/PISA/PISACallLowering.h
create mode 100644 llvm/lib/Target/PISA/PISAISelLowering.cpp
create mode 100644 llvm/lib/Target/PISA/PISAISelLowering.h
create mode 100644 llvm/lib/Target/PISA/PISALegalizerInfo.cpp
create mode 100644 llvm/lib/Target/PISA/PISALegalizerInfo.h
create mode 100644 llvm/lib/Target/PISA/PISAMachineFunctionInfo.cpp
create mode 100644 llvm/lib/Target/PISA/PISAMachineFunctionInfo.h
create mode 100644 llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
create mode 100644 llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
create mode 100644 llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
create mode 100644 llvm/lib/Target/PISA/PISARegisterBankInfo.h
create mode 100644 llvm/lib/Target/PISA/PISATargetObjectFile.h
create mode 100644 llvm/lib/TargetParser/PISATargetParser.cpp
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalize-threeway-cmp.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
create mode 100644 llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index d2766a05ce9ba..3562be546aa57 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -159,6 +159,7 @@ enum Kind {
kw_ptx_device,
kw_spir_kernel,
kw_spir_func,
+ kw_pisa_kernel,
kw_x86_64_sysvcc,
kw_win64cc,
kw_anyregcc,
diff --git a/llvm/include/llvm/CodeGen/TargetFrameLowering.h b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
index 77a1b709d9e17..151671e42dd02 100644
--- a/llvm/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetFrameLowering.h
@@ -34,6 +34,7 @@ enum Value {
ScalableVector = 2,
WasmLocal = 3,
ScalablePredicateVector = 4,
+ PISAShared = 5,
NoAlloc = 255
};
}
diff --git a/llvm/include/llvm/TargetParser/CMakeLists.txt b/llvm/include/llvm/TargetParser/CMakeLists.txt
index af26db48f2132..ccd7bdfdc209c 100644
--- a/llvm/include/llvm/TargetParser/CMakeLists.txt
+++ b/llvm/include/llvm/TargetParser/CMakeLists.txt
@@ -16,5 +16,8 @@ tablegen(LLVM R600TargetParserDef.inc -gen-amdgpu-target-def EXTRA_INCLUDES ${PR
set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/AMDGPU/AMDGPU.td)
tablegen(LLVM AMDGPUTargetParserDef.inc -gen-amdgpu-target-def EXTRA_INCLUDES ${PROJECT_SOURCE_DIR}/lib/Target/AMDGPU)
+set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/PISA/PISA.td)
+tablegen(LLVM PISAGenTargetFeatures.inc -gen-target-features EXTRA_INCLUDES ${PROJECT_SOURCE_DIR}/lib/Target/PISA)
+
# This covers all of the tablegen calls above.
add_public_tablegen_target(target_parser_gen)
diff --git a/llvm/include/llvm/TargetParser/PISATargetParser.def b/llvm/include/llvm/TargetParser/PISATargetParser.def
new file mode 100644
index 0000000000000..fafae0ebd3e8a
--- /dev/null
+++ b/llvm/include/llvm/TargetParser/PISATargetParser.def
@@ -0,0 +1,18 @@
+//===- PISATargetParser.def - PISA target parsing defines -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides defines to build up the PISA target parser's logic.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+#ifndef PISA_TARGET
+#define PISA_TARGET(NAME, GEN, VARIANT, FWDCOMPAT)
+#endif
+
+PISA_TARGET("100", 100, VariantNone, true)
diff --git a/llvm/include/llvm/TargetParser/PISATargetParser.h b/llvm/include/llvm/TargetParser/PISATargetParser.h
new file mode 100644
index 0000000000000..e9d5cd5bb1d4a
--- /dev/null
+++ b/llvm/include/llvm/TargetParser/PISATargetParser.h
@@ -0,0 +1,116 @@
+//===-- PISATargetParser.h - PISA target parsing defines ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGETPARSER_PISATARGETPARSER_H
+#define LLVM_TARGETPARSER_PISATARGETPARSER_H
+
+#include "TargetParser.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+namespace PISA {
+
+// last known PISA version
+constexpr unsigned LatestPISAVersion = 100; // PISA 1.0
+
+enum class PISATargetVariant : unsigned {
+ VariantNone = 0,
+};
+
+struct PISATargetInfo {
+ StringRef Name;
+ unsigned Gen;
+ PISATargetVariant Variant;
+ bool FwdCompat;
+};
+
+inline PISATargetInfo getPISATargetInfo(StringRef Name) {
+ static constexpr PISATargetInfo DefaultInfo = {
+ "", 0, PISATargetVariant::VariantNone, false};
+ static constexpr PISATargetInfo Info[] = {
+#define PISA_TARGET(NAME, GEN, VARIANT, FWDCOMPAT) \
+ {NAME, GEN, PISATargetVariant::VARIANT, FWDCOMPAT},
+#include "PISATargetParser.def"
+#undef PISA_TARGET
+ };
+ auto *It = llvm::find_if(Info, [&Name](const PISATargetInfo &Entry) {
+ return Entry.Name == Name;
+ });
+ return It == std::end(Info) ? DefaultInfo : *It;
+}
+
+// Defined as a macro (rather than only a StringRef) so the bare target names
+// from PISATargetParser.def can be concatenated into full CPU names as
+// compile-time string literals in fillValidCPUList(); PISACPUPrefix is derived
+// from it so the "igca_" literal lives in exactly one place.
+#define PISA_CPU_PREFIX "igca_"
+inline constexpr StringRef PISACPUPrefix = PISA_CPU_PREFIX;
+
+inline StringRef stripCPUPrefix(StringRef Name) {
+ Name.consume_front(PISACPUPrefix);
+ return Name;
+}
+
+// Full CPU name (with the "igca_" prefix) used as the default device when no
+// -mcpu/-march is specified on the command-line. This is the single source of
+// truth for the default PISA target; the backend subtarget uses the bare name
+// via stripCPUPrefix(), while the Clang driver passes the prefixed name to
+// cc1's -target-cpu.
+inline StringRef getDefaultCPUName() { return PISA_CPU_PREFIX "100"; }
+
+inline bool isValidCPU(StringRef Name) {
+ if (!Name.consume_front(PISACPUPrefix))
+ return false;
+ return !getPISATargetInfo(Name).Name.empty();
+}
+
+inline void fillValidCPUList(SmallVectorImpl<StringRef> &Values) {
+ Values.append({
+#define PISA_TARGET(NAME, GEN, VARIANT, FWDCOMPAT) PISA_CPU_PREFIX NAME,
+#include "PISATargetParser.def"
+#undef PISA_TARGET
+ });
+}
+#undef PISA_CPU_PREFIX
+
+// Fills Features with the full, transitively-implied default feature set for
+// CPU, as declared by the Proc<>/SubtargetFeature Implies lists in
+// PISAFeatures.td/PISA.td (see PISATargetParser.cpp) -- this keeps the
+// CPU->feature expansion clang sees in sync with the real subtarget's
+// TableGen-generated expansion, with the .td file as the single source of
+// truth for both.
+LLVM_ABI void fillFeatureMap(StringRef CPU, StringMap<bool> &Features);
+
+// Check for compatible PISATargetInfo
+// - TInfo - PISA target specified on command-line via -mcpu=
+// - IInfo - instruction PISA target encoded in .td files
+inline bool isCompatiblePISATargetInfo(const PISATargetInfo &TInfo,
+ const PISATargetInfo &IInfo) {
+ if (TInfo.Gen == 0)
+ return false; // no -mcpu specified
+ if (IInfo.Gen == 0)
+ return false; // no instruction target
+ if (IInfo.Gen > TInfo.Gen)
+ return false; // future instruction
+ if ((IInfo.Variant != TInfo.Variant) &&
+ (IInfo.Variant != PISATargetVariant::VariantNone))
+ return false; // variant mismatch
+ if (!IInfo.FwdCompat) // exact match required
+ return (IInfo.Gen == TInfo.Gen) && (IInfo.Variant == TInfo.Variant) &&
+ !TInfo.FwdCompat;
+ return true;
+}
+
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_TARGETPARSER_PISATARGETPARSER_H
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 069a180056488..d40cd83971fb5 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -670,6 +670,7 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(ptx_device);
KEYWORD(spir_kernel);
KEYWORD(spir_func);
+ KEYWORD(pisa_kernel);
KEYWORD(intel_ocl_bicc);
KEYWORD(x86_64_sysvcc);
KEYWORD(win64cc);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 93a79a7035e6f..3aebe6a9461e5 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -2345,6 +2345,7 @@ bool LLParser::parseOptionalCallingConv(unsigned &CC) {
case lltok::kw_ptx_device: CC = CallingConv::PTX_Device; break;
case lltok::kw_spir_kernel: CC = CallingConv::SPIR_KERNEL; break;
case lltok::kw_spir_func: CC = CallingConv::SPIR_FUNC; break;
+ case lltok::kw_pisa_kernel: CC = CallingConv::PISA_KERNEL; break;
case lltok::kw_intel_ocl_bicc: CC = CallingConv::Intel_OCL_BI; break;
case lltok::kw_x86_64_sysvcc: CC = CallingConv::X86_64_SysV; break;
case lltok::kw_win64cc: CC = CallingConv::Win64; break;
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index c3202eea12c28..c0c8959b07c02 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -391,6 +391,7 @@ static void printCallingConv(unsigned cc, raw_ostream &Out) {
case CallingConv::Win64: Out << "win64cc"; break;
case CallingConv::SPIR_FUNC: Out << "spir_func"; break;
case CallingConv::SPIR_KERNEL: Out << "spir_kernel"; break;
+ case CallingConv::PISA_KERNEL: Out << "pisa_kernel"; break;
case CallingConv::Swift: Out << "swiftcc"; break;
case CallingConv::SwiftTail: Out << "swifttailcc"; break;
case CallingConv::X86_INTR: Out << "x86_intrcc"; break;
diff --git a/llvm/lib/Target/PISA/CMakeLists.txt b/llvm/lib/Target/PISA/CMakeLists.txt
index 17dd046b4645f..788ffb53ef90a 100644
--- a/llvm/lib/Target/PISA/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/CMakeLists.txt
@@ -2,19 +2,33 @@ add_llvm_component_group(PISA)
set(LLVM_TARGET_DEFINITIONS PISA.td)
tablegen(LLVM PISAGenAsmWriter.inc -gen-asm-writer)
+tablegen(LLVM PISAGenGlobalISel.inc -gen-global-isel -warn-on-skipped-patterns -gisel-extended-llt)
tablegen(LLVM PISAGenInstrInfo.inc -gen-instr-info)
tablegen(LLVM PISAGenMCCodeEmitter.inc -gen-emitter)
+tablegen(LLVM PISAGenRegisterBank.inc -gen-register-bank)
tablegen(LLVM PISAGenRegisterInfo.inc -gen-register-info)
tablegen(LLVM PISAGenSubtargetInfo.inc -gen-subtarget)
tablegen(LLVM PISAGenSearchableTables.inc -gen-searchable-tables)
+tablegen(LLVM PISAGenPreLegalizeGICombiner.inc -gen-global-isel-combiner
+ -combiners="PISAPreLegalizerCombiner")
+tablegen(LLVM PISAGenPostLegalizeGICombiner.inc -gen-global-isel-combiner
+ -combiners="PISAPostLegalizerCombiner")
+
add_public_tablegen_target(PISACommonTableGen)
add_llvm_target(PISACodeGen
PISACacheCtrlMMRA.cpp
+ PISACallLowering.cpp
+ PISAISelLowering.cpp
PISAInstrInfo.cpp
+ PISALegalizerInfo.cpp
PISAMCInstLower.cpp
+ PISAMachineFunctionInfo.cpp
+ PISAPostLegalizerCombiner.cpp
+ PISAPreLegalizerCombiner.cpp
PISARegManager.cpp
+ PISARegisterBankInfo.cpp
PISARegisterInfo.cpp
PISASubtarget.cpp
PISATargetMachine.cpp
diff --git a/llvm/lib/Target/PISA/PISA.h b/llvm/lib/Target/PISA/PISA.h
index 21e5c367ce48e..375208bf28350 100644
--- a/llvm/lib/Target/PISA/PISA.h
+++ b/llvm/lib/Target/PISA/PISA.h
@@ -10,10 +10,71 @@
#define LLVM_LIB_TARGET_PISA_PISA_H
#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "PISADefines.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Target/TargetMachine.h"
namespace llvm {
+class ImmutablePass;
+class InstructionSelector;
+class RegisterBankInfo;
+class PISASubtarget;
class PISATargetMachine;
-class PassRegistry;
+
+ModulePass *createPISALegalizeCallsPass();
+ModulePass *createPISAVerifierPass();
+ModulePass *createPISAKernelByValArgsLoweringLegacyPass();
+ModulePass *createPISAPropagateNullPointersPass();
+
+FunctionPass *createPISAExpandIntrinsicsPass();
+FunctionPass *createPISAEmitIntrinsicsPass();
+FunctionPass *createPISALegalizeSubregAccess();
+FunctionPass *createPISAOptimizeSubregAccess();
+FunctionPass *createPISAOptimizeRedundantCopies();
+FunctionPass *createPISAInsertLifetimeStart();
+FunctionPass *createPISAMarkConvergentNoMerge();
+FunctionPass *createPISAPreLegalizerCombiner();
+FunctionPass *createPISAPostLegalizerCombiner();
+FunctionPass *createPISAReplaceIntrinsicsPass();
+MachineFunctionPass *createPISALegalizePredicatesPass();
+MachineFunctionPass *createCacheHintSelectorPass();
+MachineFunctionPass *createPISAScopeSelectorPass();
+
+InstructionSelector *
+createPISAInstructionSelector(const PISATargetMachine &TM,
+ const PISASubtarget &Subtarget,
+ const RegisterBankInfo &RBI);
+
+MachineFunctionPass *
+createPISAMachineFunctionPrinterPass(const std::string &Banner,
+ unsigned Counter);
+FunctionPass *createPISALayoutPass();
+MachineFunctionPass *createPISAVerifyTypesPass();
+
+void initializeCacheHintSelectorPass(PassRegistry &);
+void initializePISAEmitIntrinsicsPass(PassRegistry &);
+void initializePISAExpandIntrinsicsPass(PassRegistry &);
+void initializePISAInsertLifetimeStartPass(PassRegistry &);
+void initializePISAKernelByValArgsLoweringLegacyPass(PassRegistry &);
+void initializePISALegalizeCallsPass(PassRegistry &);
+void initializePISALegalizeSubregAccessPass(PassRegistry &);
+void initializePISAMachineFunctionPrinterPass(PassRegistry &);
+void initializePISAMarkConvergentNoMergePass(PassRegistry &);
+void initializePISAOptimizeRedundantCopiesPass(PassRegistry &);
+void initializePISAOptimizeSubregAccessPass(PassRegistry &);
+void initializePISALegalizePredicatesPass(PassRegistry &);
+void initializePISAPostLegalizerCombinerPass(PassRegistry &);
+void initializePISAPreLegalizerCombinerPass(PassRegistry &);
+void initializePISAPropagateNullPointersPass(PassRegistry &);
+void initializePISAReplaceIntrinsicsPass(PassRegistry &);
+void initializePISAScopeSelectorPass(PassRegistry &);
+void initializePISAVerifierPass(PassRegistry &);
+void initializePISALayoutPass(PassRegistry &);
+void initializePISAVerifyTypesPass(PassRegistry &);
+
+namespace PISA {
+LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx);
+} // namespace PISA
} // namespace llvm
#endif // LLVM_LIB_TARGET_PISA_PISA_H
diff --git a/llvm/lib/Target/PISA/PISACallLowering.cpp b/llvm/lib/Target/PISA/PISACallLowering.cpp
new file mode 100644
index 0000000000000..6928ed63d3aab
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISACallLowering.cpp
@@ -0,0 +1,758 @@
+//===-- PISACallLowering.cpp - Call lowering ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISACallLowering.h"
+#include "MCTargetDesc/PISABaseInfo.h"
+#include "PISA.h"
+#include "PISAISelLowering.h"
+#include "PISAMachineFunctionInfo.h"
+#include "PISARegisterInfo.h"
+#include "PISASubtarget.h"
+#include "PISAUtils.h"
+#include "llvm/CodeGen/FunctionLoweringInfo.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/Support/ModRef.h"
+
+using namespace llvm;
+
+PISACallLowering::PISACallLowering(const PISATargetLowering &TLI)
+ : CallLowering(&TLI) {}
+
+bool PISACallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
+ const Value *Val, ArrayRef<Register> VRegs,
+ FunctionLoweringInfo &FLI,
+ Register SwiftErrorVReg) const {
+ // FIXME: Currently the return support is only for registers.
+ // Pending:
+ // - return immediates: fold immediates to return operand
+ if (VRegs.size() > 1)
+ return false;
+ if (Val) {
+ auto &DL = MIRBuilder.getDataLayout();
+ const auto &STI = MIRBuilder.getMF().getSubtarget();
+ unsigned Op = 0;
+ auto *Ty = Val->getType();
+ auto VReg = VRegs[0];
+ if (Ty->isVectorTy()) {
+ auto *VTy = cast<FixedVectorType>(Ty);
+ unsigned NumElts = VTy->getNumElements();
+ unsigned EltSize = DL.getTypeSizeInBits(Ty->getScalarType());
+ switch (EltSize) {
+ case 8:
+ switch (NumElts) {
+ case 2:
+ Op = PISA::retValue_v2i8_r;
+ break;
+ case 3:
+ Op = PISA::retValue_v3i8_r;
+ break;
+ case 4:
+ Op = PISA::retValue_v4i8_r;
+ break;
+ default:
+ llvm_unreachable("Unknown return vector size!");
+ break;
+ }
+ break;
+ case 16:
+ switch (NumElts) {
+ case 2:
+ Op = PISA::retValue_v2i16_r;
+ break;
+ case 3:
+ Op = PISA::retValue_v3i16_r;
+ break;
+ case 4:
+ Op = PISA::retValue_v4i16_r;
+ break;
+ default:
+ llvm_unreachable("Unknown return vector size!");
+ break;
+ }
+ break;
+ case 32:
+ switch (NumElts) {
+ case 2:
+ Op = PISA::retValue_v2i32_r;
+ break;
+ case 3:
+ Op = PISA::retValue_v3i32_r;
+ break;
+ case 4:
+ Op = PISA::retValue_v4i32_r;
+ break;
+ case 5:
+ Op = PISA::retValue_v5i32_r;
+ break;
+ case 6:
+ Op = PISA::retValue_v6i32_r;
+ break;
+ case 7:
+ Op = PISA::retValue_v7i32_r;
+ break;
+ case 8:
+ Op = PISA::retValue_v8i32_r;
+ break;
+ case 16:
+ Op = PISA::retValue_v16i32_r;
+ break;
+ case 32:
+ Op = PISA::retValue_v32i32_r;
+ break;
+ case 64:
+ Op = PISA::retValue_v64i32_r;
+ break;
+ default:
+ llvm_unreachable("Unknown return vector size!");
+ break;
+ }
+ break;
+ case 64:
+ switch (NumElts) {
+ case 2:
+ Op = PISA::retValue_v2i64_r;
+ break;
+ case 3:
+ Op = PISA::retValue_v3i64_r;
+ break;
+ case 4:
+ Op = PISA::retValue_v4i64_r;
+ break;
+ default:
+ llvm_unreachable("Unknown return vector size!");
+ break;
+ }
+ break;
+ default:
+ llvm_unreachable("Unknown return size!");
+ break;
+ }
+ } else {
+ unsigned BitSize = DL.getTypeSizeInBits(Ty);
+ switch (BitSize) {
+ case 1: // change i1 to i16 (see lowerCall())
+ {
+ const LLT I16 = LLT::integer(16);
+ auto Dst = MIRBuilder.getMRI()->createGenericVirtualRegister(I16);
+ auto &MF = MIRBuilder.getMF();
+ auto &F = MF.getFunction();
+ const DataLayout &DL = MF.getDataLayout();
+ ArgInfo RetInfo(VReg, *Val, 0);
+ setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);
+ auto Sext = llvm::any_of(
+ RetInfo.Flags, [](const auto &Flag) { return Flag.isSExt(); });
+ auto Zext = llvm::any_of(
+ RetInfo.Flags, [](const auto &Flag) { return Flag.isZExt(); });
+ if (Sext) {
+ MIRBuilder.buildSExt(Dst, VReg);
+ } else if (Zext) {
+ MIRBuilder.buildZExt(Dst, VReg);
+ } else {
+ MIRBuilder.buildAnyExt(Dst, VReg);
+ }
+ VReg = Dst;
+ }
+ Op = PISA::retValue_i16_r;
+ break;
+ case 8:
+ Op = PISA::retValue_i8_r;
+ break;
+ case 16:
+ Op = PISA::retValue_i16_r;
+ break;
+ case 32:
+ Op = PISA::retValue_i32_r;
+ break;
+ case 64:
+ Op = PISA::retValue_i64_r;
+ break;
+ default:
+ llvm_unreachable("Unknown return size!");
+ break;
+ }
+ }
+ // Backend-defined opcodes, e.g. retValue* must have a register
+ // class assigned to their 'source' register. During instruction
+ // combine, a preceeding instructions may be combined, with a new
+ // 'dest' register being assigned. Attempt to replace 'source'
+ // with 'dest' will trigger an assertion in canReplaceReg(), since
+ // there is an expectation of no register class being assigned.
+ // Having an extra copy here eliminates the problem; copy itself
+ // will be removed during instruction selection.
+ auto *MRI = MIRBuilder.getMRI();
+ auto Tmp = MRI->createGenericVirtualRegister(MRI->getType(VReg));
+ MIRBuilder.buildCopy(Tmp, VReg);
+ MIRBuilder.buildInstr(Op).addUse(Tmp).constrainAllUses(
+ MIRBuilder.getTII(), *STI.getRegisterInfo(), *STI.getRegBankInfo());
+ return true;
+ }
+ MIRBuilder.buildInstr(PISA::ret);
+ return true;
+}
+
+bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
+ const Function &F,
+ ArrayRef<ArrayRef<Register>> VRegs,
+ FunctionLoweringInfo &FLI) const {
+ auto *MRI = MIRBuilder.getMRI();
+ auto &MF = MIRBuilder.getMF();
+ auto &Ctx = F.getContext();
+ auto *MFInfo = MF.getInfo<PISAMachineFunctionInfo>();
+ auto &DL = F.getParent()->getDataLayout();
+ bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
+ for (const auto [i, Arg] : llvm::enumerate(F.args())) {
+ assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs");
+
+ ArgInfo OrigArg{VRegs[i], Arg, static_cast<unsigned>(i)};
+ setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
+ auto *ArgType = OrigArg.OrigValue->getType();
+ const bool IsByRef = ArgType->isPointerTy() && OrigArg.Flags[0].isByRef();
+ const unsigned ArgSize = IsByRef
+ ? OrigArg.Flags[0].getByRefSize()
+ : MRI->getType(VRegs[i][0]).getSizeInBytes();
+ MFInfo->setArgInfo(i, ArgSize, IsByRef);
+
+ if (IsKernel && Arg.use_empty())
+ continue;
+
+ unsigned Op = 0;
+ if (IsByRef) {
+ assert(IsKernel && "'byref' is only used in kernel!");
+ Op = PISA::G_PISA_PARAM_SLOT;
+ loadParamWithOpcode(MIRBuilder, F, VRegs[i][0], ArgType, Op,
+ Arg.getArgNo(), 0);
+ } else if (IsKernel && ArgType->isVectorTy()) {
+ auto *VectorTy = cast<FixedVectorType>(ArgType);
+ auto NumElts = VectorTy->getNumElements();
+ auto *EltTy = VectorTy->getElementType();
+ auto EltSize = DL.getTypeSizeInBits(EltTy);
+ auto Split = (NumElts > 4) || ((NumElts == 3) && (EltSize != 32)) ||
+ ((NumElts == 4) && (EltSize == 64)) || (NumElts == 1);
+ if (Split) {
+ // handle odd-sized and large kernel args, e.g.
+ // <3 x i8> <16 x i16>
+ // loadParam_8b @[arg+0] loadParam_v4_32b @[arg+0]
+ // loadParam_8b @[arg+1] loadParam_v4_32b @[arg+16]
+ // loadParam_8b @[arg+2] buildVector(<8 x i32>)
+ // buildVector(<3 x i8>) bitcast(<16 x i16)
+ auto TargetReg = VRegs[i][0];
+
+ const auto *EltRegClass =
+ (EltSize == 8
+ ? &PISA::Reg8bRegClass
+ : (EltSize == 16 ? &PISA::Reg16bRegClass
+ : (EltSize == 32 ? &PISA::Reg32bRegClass
+ : &PISA::Reg64bRegClass)));
+ auto TotalSize = NumElts * EltSize;
+ auto EltLLT = LLT::integer(EltSize);
+ auto I32 = LLT::integer(32);
+ if (NumElts <= 4) {
+ // do not group
+ } else if (TotalSize % 128 == 0) { // 4 x i32
+ EltTy = FixedVectorType::get(Type::getInt32Ty(Ctx), 4);
+ EltRegClass = &PISA::RegV4_32bRegClass;
+ EltLLT = LLT::vector(ElementCount::getFixed(4), I32);
+ TargetReg = MRI->createGenericVirtualRegister(
+ LLT::vector(ElementCount::getFixed(TotalSize / 32), I32));
+ NumElts = TotalSize / 128;
+ } else if (TotalSize % 64 == 0) { // 2 x i32
+ EltTy = FixedVectorType::get(Type::getInt32Ty(Ctx), 2);
+ EltRegClass = &PISA::RegV2_32bRegClass;
+ EltLLT = LLT::vector(ElementCount::getFixed(2), I32);
+ TargetReg = MRI->createGenericVirtualRegister(
+ LLT::vector(ElementCount::getFixed(TotalSize / 32), I32));
+ NumElts = TotalSize / 64;
+ } else if (TotalSize % 32 == 0) { // 1 x i32
+ EltTy = Type::getInt32Ty(Ctx);
+ EltRegClass = &PISA::Reg32bRegClass;
+ EltLLT = LLT::integer(32);
+ TargetReg = MRI->createGenericVirtualRegister(
+ LLT::vector(ElementCount::getFixed(TotalSize / 32), I32));
+ NumElts = TotalSize / 32;
+ }
+
+ SmallVector<Register, 4> Regs;
+ for (unsigned I = 0; I < NumElts; I++) {
+ auto Reg = MRI->createGenericVirtualRegister(EltLLT);
+ MRI->setRegClass(Reg, EltRegClass);
+ Op = getLoadParamOpcode(MIRBuilder, F, Reg, EltTy);
+ loadParamWithOpcode(MIRBuilder, F, Reg, EltTy, Op, Arg.getArgNo(),
+ I * EltLLT.getSizeInBytes());
+ if (EltTy->isPointerTy()) {
+ // ld.param loads a scalar value, so convert to ptr here
+ auto AS = cast<PointerType>(EltTy)->getAddressSpace();
+ auto PtrLLT = LLT::pointer(AS, EltSize);
+ auto CastReg = MRI->createGenericVirtualRegister(PtrLLT);
+ MRI->setRegClass(CastReg, EltRegClass);
+ MIRBuilder.buildIntToPtr(CastReg, Reg);
+ Regs.push_back(CastReg);
+ } else if (EltTy->isFloatingPointTy()) {
+ // ld.param loads an integer value; bitcast to float so that
+ // G_BUILD_VECTOR element types match the result vector element
+ // type.
+ auto FloatLLT = EltTy->isBFloatTy() ? LLT::bfloat16()
+ : EltSize == 16 ? LLT::float16()
+ : EltSize == 32 ? LLT::float32()
+ : LLT::float64();
+ auto CastReg = MRI->createGenericVirtualRegister(FloatLLT);
+ MRI->setRegClass(CastReg, EltRegClass);
+ MIRBuilder.buildBitcast(CastReg, Reg);
+ Regs.push_back(CastReg);
+ } else {
+ Regs.push_back(Reg);
+ }
+ }
+ if (NumElts == 1)
+ MIRBuilder.buildCopy(TargetReg, Regs[0]);
+ else if (EltLLT.isVector())
+ MIRBuilder.buildConcatVectors(TargetReg, Regs);
+ else
+ MIRBuilder.buildBuildVector(TargetReg, Regs);
+ if (TargetReg != VRegs[i][0]) {
+ if (MRI->getType(TargetReg) != MRI->getType(VRegs[i][0]))
+ MIRBuilder.buildBitcast(VRegs[i][0], TargetReg);
+ else
+ MIRBuilder.buildCopy(VRegs[i][0], TargetReg);
+ }
+ } else {
+ Op = getLoadParamOpcode(MIRBuilder, F, VRegs[i][0], Arg.getType());
+ loadParamWithOpcode(MIRBuilder, F, VRegs[i][0], ArgType, Op,
+ Arg.getArgNo(), 0);
+ }
+ } else {
+ Op = getLoadParamOpcode(MIRBuilder, F, VRegs[i][0], Arg.getType());
+ loadParamWithOpcode(MIRBuilder, F, VRegs[i][0], ArgType, Op,
+ Arg.getArgNo(), 0);
+ }
+ }
+ return true;
+}
+
+void PISACallLowering::loadParamWithOpcode(MachineIRBuilder &MIRBuilder,
+ const Function &F,
+ const Register &VReg, Type *ArgType,
+ unsigned Opcode, unsigned ArgNo,
+ unsigned Offset) const {
+ auto *MRI = MIRBuilder.getMRI();
+ auto &DL = F.getParent()->getDataLayout();
+ bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
+ const auto BitSize = DL.getTypeSizeInBits(ArgType->getScalarType());
+
+ auto VReg16 = VReg;
+ if (BitSize == 1) { // load arg into i16
+ VReg16 = MRI->createGenericVirtualRegister(LLT::integer(16));
+ MRI->setRegClass(VReg16, &PISA::Reg16bRegClass);
+ }
+
+ auto MIB = MIRBuilder.buildInstr(Opcode).addDef(VReg16).addImm(ArgNo);
+ if (IsKernel)
+ MIB.addImm(Offset);
+
+ // Attach the kernel argument name from !kernel_arg_name metadata as an
+ // extra symbol operand on the loadParam instruction. This allows
+ // PISAInstPrinter to print the actual argument name (e.g., [%input])
+ // instead of the generic [%argN].
+ if (IsKernel)
+ if (MDNode *MD = F.getMetadata("kernel_arg_name"))
+ if (ArgNo < MD->getNumOperands())
+ if (auto *S = dyn_cast<MDString>(MD->getOperand(ArgNo)))
+ if (!S->getString().empty())
+ MIB.addExternalSymbol(
+ MIRBuilder.getMF().createExternalSymbolName(S->getString()));
+
+ if (BitSize == 1) { // convert i16 into i1
+ MIRBuilder.buildTrunc(VReg, VReg16);
+ }
+}
+
+unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
+ const Function &F,
+ const Register &VReg,
+ Type *ArgType) const {
+ auto *MRI = MIRBuilder.getMRI();
+ auto &MF = MIRBuilder.getMF();
+ const auto *TRI = static_cast<const PISARegisterInfo *>(
+ MF.getSubtarget().getRegisterInfo());
+
+ bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
+ unsigned Op = 0;
+ auto &DL = F.getParent()->getDataLayout();
+
+ const unsigned ParamScalar[2][4] = {
+ // [isKernel][8/16/32/64]
+ {PISA::functionParameter_i8, PISA::functionParameter_i16,
+ PISA::functionParameter_i32, PISA::functionParameter_i64},
+ {PISA::loadParam_i8, PISA::loadParam_i16, PISA::loadParam_i32,
+ PISA::loadParam_i64}};
+ const unsigned ParamVector[2][4][3] = {
+ // [isKernel][8/16/32/64][v2/v3/v4]
+ {
+ {PISA::functionParameter_v2i8, PISA::functionParameter_v3i8,
+ PISA::functionParameter_v4i8},
+ {PISA::functionParameter_v2i16, PISA::functionParameter_v3i16,
+ PISA::functionParameter_v4i16},
+ {PISA::functionParameter_v2i32, PISA::functionParameter_v3i32,
+ PISA::functionParameter_v4i32},
+ {PISA::functionParameter_v2i64, PISA::functionParameter_v3i64,
+ PISA::functionParameter_v4i64},
+ },
+ {{PISA::loadParam_v2i8, 0, PISA::loadParam_v4i8},
+ {PISA::loadParam_v2i16, 0, PISA::loadParam_v4i16},
+ {PISA::loadParam_v2i32, PISA::loadParam_v3i32, PISA::loadParam_v4i32},
+ {PISA::loadParam_v2i64, PISA::loadParam_v3i64, 0}}};
+
+ const auto BitSize = DL.getTypeSizeInBits(ArgType->getScalarType());
+ // Calculate the argument size in bytes.
+ if (ArgType->isIntegerTy()) {
+ switch (BitSize) {
+ case 1: // i1 args are loaded via i16 register
+ case 16:
+ MRI->setRegClass(VReg, &PISA::Reg16bRegClass);
+ Op = ParamScalar[IsKernel][1];
+ break;
+ case 8:
+ MRI->setRegClass(VReg, &PISA::Reg8bRegClass);
+ Op = ParamScalar[IsKernel][0];
+ break;
+ case 32:
+ MRI->setRegClass(VReg, &PISA::Reg32bRegClass);
+ Op = ParamScalar[IsKernel][2];
+ break;
+ case 64:
+ MRI->setRegClass(VReg, &PISA::Reg64bRegClass);
+ Op = ParamScalar[IsKernel][3];
+ break;
+ default:
+ assert(false && "Bit size for call arg not supported");
+ }
+ } else if (ArgType->isPointerTy()) {
+ if (BitSize == 64) {
+ MRI->setRegClass(VReg, &PISA::Reg64bRegClass);
+ Op = ParamScalar[IsKernel][3]; // 64bit
+ } else if (BitSize == 32) {
+ MRI->setRegClass(VReg, &PISA::Reg32bRegClass);
+ Op = ParamScalar[IsKernel][2]; // 32bit
+ } else {
+ llvm_unreachable("unsupported pointer size");
+ }
+ } else if (ArgType->isHalfTy()) {
+ MRI->setRegClass(VReg, &PISA::Reg16bRegClass);
+ Op = ParamScalar[IsKernel][1];
+ } else if (ArgType->isBFloatTy()) {
+ MRI->setRegClass(VReg, &PISA::Reg16bRegClass);
+ Op = ParamScalar[IsKernel][1];
+ } else if (ArgType->isFloatTy()) {
+ MRI->setRegClass(VReg, &PISA::Reg32bRegClass);
+ Op = ParamScalar[IsKernel][2];
+ } else if (ArgType->isDoubleTy()) {
+ MRI->setRegClass(VReg, &PISA::Reg64bRegClass);
+ Op = ParamScalar[IsKernel][3];
+ } else if (ArgType->isVectorTy()) {
+ auto *VectorTy = cast<FixedVectorType>(ArgType);
+ auto NumElts = VectorTy->getNumElements();
+ assert(((((BitSize == 8) || (BitSize == 16) || (BitSize == 64)) &&
+ ((NumElts >= 2) && (NumElts <= 4))) ||
+ ((BitSize == 32) &&
+ (((NumElts >= 2) && (NumElts <= 8)) || (NumElts == 16) ||
+ (NumElts == 32) || (NumElts == 64)))) &&
+ "unsupported vector size");
+ MRI->setRegClass(VReg, TRI->getVectorRegClass(NumElts, BitSize));
+ switch (BitSize) {
+ case 8:
+ Op = ParamVector[IsKernel][0][NumElts - 2];
+ break;
+ case 16:
+ Op = ParamVector[IsKernel][1][NumElts - 2];
+ break;
+ case 32: {
+ if (NumElts > 4) {
+ assert(!IsKernel && "large vector arg in kernel is not supported");
+ switch (NumElts) {
+ default:
+ llvm_unreachable("unsupported number of elements in large vector");
+ break;
+ case 5:
+ Op = PISA::functionParameter_v5i32;
+ break;
+ case 6:
+ Op = PISA::functionParameter_v6i32;
+ break;
+ case 7:
+ Op = PISA::functionParameter_v7i32;
+ break;
+ case 8:
+ Op = PISA::functionParameter_v8i32;
+ break;
+ case 16:
+ Op = PISA::functionParameter_v16i32;
+ break;
+ case 32:
+ Op = PISA::functionParameter_v32i32;
+ break;
+ case 64:
+ Op = PISA::functionParameter_v64i32;
+ break;
+ }
+ } else {
+ Op = ParamVector[IsKernel][2][NumElts - 2];
+ }
+ } break;
+ case 64:
+ Op = ParamVector[IsKernel][3][NumElts - 2];
+ break;
+ default:
+ assert(false && "Bit size for call arg not supported");
+ }
+ assert(Op && "argument type is not supported");
+ } else {
+ report_fatal_error("Argument type not supported");
+ }
+ return Op;
+}
+
+bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
+ CallLoweringInfo &Info) const {
+ // Currently call returns should have single vregs.
+ // TODO: handle the case of multiple registers.
+ if (Info.OrigRet.Regs.size() > 1)
+ return false;
+
+ bool IsIndirectCall = Info.Callee.isReg();
+ if (!IsIndirectCall) {
+ assert(Info.Callee.isGlobal());
+ const Function *CF =
+ dyn_cast_or_null<const Function>(Info.Callee.getGlobal());
+ if (CF == nullptr)
+ return false;
+ }
+
+ MachineInstrBuilder MIB;
+ const auto *TRI = static_cast<const PISARegisterInfo *>(
+ MIRBuilder.getMF().getSubtarget().getRegisterInfo());
+
+ if (IsIndirectCall) {
+ Register CalleeReg = Info.Callee.getReg();
+ auto *MRI = MIRBuilder.getMRI();
+ LLT CalleeTy = MRI->getType(CalleeReg);
+ Register CalleeI64Reg = MRI->createGenericVirtualRegister(LLT::integer(64));
+
+ if (CalleeTy.isPointer())
+ MIRBuilder.buildPtrToInt(CalleeI64Reg, CalleeReg);
+ else
+ llvm_unreachable("Unexpected indirect callee register type");
+
+ MRI->setRegClass(CalleeI64Reg, TRI->getRegClassFromLLT(LLT::integer(64)));
+ Info.Callee = MachineOperand::CreateReg(CalleeI64Reg, false);
+ }
+
+ // promote i1 args to i16
+ SmallVector<Register, 8> ArgRegs;
+ for (const auto &Arg : Info.OrigArgs) {
+ // Currently call args should have single vregs.
+ if (Arg.Regs.size() > 1)
+ return false;
+ if (Arg.Ty->getScalarSizeInBits() == 1) {
+ const LLT I16 = LLT::integer(16);
+ auto Reg = MIRBuilder.getMRI()->createGenericVirtualRegister(I16);
+ MIRBuilder.getMRI()->setRegClass(Reg, TRI->getRegClassFromLLT(I16));
+ auto Sext = llvm::any_of(Arg.Flags,
+ [](const auto &Flag) { return Flag.isSExt(); });
+ auto Zext = llvm::any_of(Arg.Flags,
+ [](const auto &Flag) { return Flag.isZExt(); });
+ if (Sext) {
+ MIRBuilder.buildSExt(Reg, Arg.Regs[0]);
+ } else if (Zext) {
+ MIRBuilder.buildZExt(Reg, Arg.Regs[0]);
+ } else {
+ MIRBuilder.buildAnyExt(Reg, Arg.Regs[0]);
+ }
+ ArgRegs.push_back(Reg);
+ } else {
+ ArgRegs.push_back(Arg.Regs[0]);
+ }
+ }
+
+ auto MutateI1 = false;
+ if (!Info.OrigRet.Ty->isVoidTy()) {
+ // select the call op according to return type and build MI
+ auto RetLLT =
+ llvm::getLLTForType(*Info.OrigRet.Ty, MIRBuilder.getDataLayout());
+ unsigned CallOp = 0;
+ if (RetLLT.isScalar() || RetLLT.isPointer()) {
+ switch (RetLLT.getSizeInBits()) {
+ case 1:
+ MutateI1 = true;
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_i16_r_i64_r
+ : PISA::functionCall_i16_r;
+ break;
+ case 8:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_i8_r_i64_r
+ : PISA::functionCall_i8_r;
+ break;
+ case 16:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_i16_r_i64_r
+ : PISA::functionCall_i16_r;
+ break;
+ case 32:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_i32_r_i64_r
+ : PISA::functionCall_i32_r;
+ break;
+ case 64:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_i64_r_i64_r
+ : PISA::functionCall_i64_r;
+ break;
+ default:
+ llvm_unreachable("Unsupported function return type");
+ break;
+ }
+ } else if (RetLLT.isVector()) {
+ auto TypeBitSize = RetLLT.getElementType().getSizeInBits();
+ auto NumElts = RetLLT.getNumElements();
+ switch (TypeBitSize) {
+ case 8:
+ switch (NumElts) {
+ case 2:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v2i8_r_i64_r
+ : PISA::functionCall_v2i8_r;
+ break;
+ case 3:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v3i8_r_i64_r
+ : PISA::functionCall_v3i8_r;
+ break;
+ case 4:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v4i8_r_i64_r
+ : PISA::functionCall_v4i8_r;
+ break;
+ default:
+ llvm_unreachable("Vector size not supported");
+ }
+ break;
+ case 16:
+ switch (NumElts) {
+ case 2:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v2i16_r_i64_r
+ : PISA::functionCall_v2i16_r;
+ break;
+ case 3:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v3i16_r_i64_r
+ : PISA::functionCall_v3i16_r;
+ break;
+ case 4:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v4i16_r_i64_r
+ : PISA::functionCall_v4i16_r;
+ break;
+ default:
+ llvm_unreachable("Vector size not supported");
+ }
+ break;
+ case 32:
+ switch (NumElts) {
+ case 2:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v2i32_r_i64_r
+ : PISA::functionCall_v2i32_r;
+ break;
+ case 3:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v3i32_r_i64_r
+ : PISA::functionCall_v3i32_r;
+ break;
+ case 4:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v4i32_r_i64_r
+ : PISA::functionCall_v4i32_r;
+ break;
+ case 5:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v5i32_r_i64_r
+ : PISA::functionCall_v5i32_r;
+ break;
+ case 6:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v6i32_r_i64_r
+ : PISA::functionCall_v6i32_r;
+ break;
+ case 7:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v7i32_r_i64_r
+ : PISA::functionCall_v7i32_r;
+ break;
+ case 8:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v8i32_r_i64_r
+ : PISA::functionCall_v8i32_r;
+ break;
+ case 16:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v16i32_r_i64_r
+ : PISA::functionCall_v16i32_r;
+ break;
+ case 32:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v32i32_r_i64_r
+ : PISA::functionCall_v32i32_r;
+ break;
+ case 64:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v64i32_r_i64_r
+ : PISA::functionCall_v64i32_r;
+ break;
+ default:
+ llvm_unreachable("Vector size not supported");
+ }
+ break;
+ case 64:
+ switch (NumElts) {
+ case 2:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v2i64_r_i64_r
+ : PISA::functionCall_v2i64_r;
+ break;
+ case 3:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v3i64_r_i64_r
+ : PISA::functionCall_v3i64_r;
+ break;
+ case 4:
+ CallOp = IsIndirectCall ? PISA::indirectFunctionCall_v4i64_r_i64_r
+ : PISA::functionCall_v4i64_r;
+ break;
+ default:
+ llvm_unreachable("Vector size not supported");
+ }
+ break;
+ default:
+ llvm_unreachable("Unsupported function return type");
+ break;
+ }
+ } else {
+ llvm_unreachable("Unsupported call return type");
+ }
+
+ // set Ret register class
+ assert(!Info.OrigRet.Regs.empty());
+ Register ResVReg = Info.OrigRet.Regs[0];
+ auto OrigRetLLT = RetLLT;
+ if (MutateI1) { // will return i16 (see lowerReturn())
+ RetLLT = LLT::integer(16);
+ ResVReg = MIRBuilder.getMRI()->createGenericVirtualRegister(RetLLT);
+ }
+
+ MIRBuilder.getMRI()->setRegClass(ResVReg, TRI->getRegClassFromLLT(RetLLT));
+ MIB = MIRBuilder.buildInstr(CallOp).addDef(ResVReg).add(Info.Callee);
+
+ if (MutateI1) { // change i16 back to i1
+ Register VReg = Info.OrigRet.Regs[0];
+ MIRBuilder.getMRI()->setRegClass(VReg,
+ TRI->getRegClassFromLLT(OrigRetLLT));
+ MIRBuilder.buildTrunc(VReg, ResVReg);
+ }
+ } else {
+ // void return
+ MIB = MIRBuilder
+ .buildInstr(IsIndirectCall ? PISA::indirectFunctionCall_void_r
+ : PISA::functionCall_void)
+ .add(Info.Callee);
+ }
+
+ // add function args into MI if any
+ for (auto Reg : ArgRegs) {
+ MIB.addUse(Reg);
+ }
+
+ return true;
+}
diff --git a/llvm/lib/Target/PISA/PISACallLowering.h b/llvm/lib/Target/PISA/PISACallLowering.h
new file mode 100644
index 0000000000000..b3cd194bfd242
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISACallLowering.h
@@ -0,0 +1,44 @@
+//===-- PISACallLowering.h - Call lowering --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISACALLLOWERING_H
+#define LLVM_LIB_TARGET_PISA_PISACALLLOWERING_H
+
+#include "llvm/CodeGen/GlobalISel/CallLowering.h"
+
+namespace llvm {
+
+class PISATargetLowering;
+
+class PISACallLowering : public CallLowering {
+
+public:
+ PISACallLowering(const PISATargetLowering &TLI);
+
+ bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val,
+ ArrayRef<Register> VRegs, FunctionLoweringInfo &FLI,
+ Register SwiftErrorVReg) const override;
+
+ bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F,
+ ArrayRef<ArrayRef<Register>> VRegs,
+ FunctionLoweringInfo &FLI) const override;
+
+ // Build OpCall, or replace with a builtin function.
+ bool lowerCall(MachineIRBuilder &MIRBuilder,
+ CallLoweringInfo &Info) const override;
+
+private:
+ unsigned getLoadParamOpcode(MachineIRBuilder &MIRBuilder, const Function &F,
+ const Register &VReg, Type *ArgType) const;
+ void loadParamWithOpcode(MachineIRBuilder &MIRBuilder, const Function &F,
+ const Register &VReg, Type *ArgType, unsigned Opcode,
+ unsigned ArgNo, unsigned Offset) const;
+};
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISACALLLOWERING_H
diff --git a/llvm/lib/Target/PISA/PISAFrameLowering.h b/llvm/lib/Target/PISA/PISAFrameLowering.h
index c3a3b88031240..04767e0aaf690 100644
--- a/llvm/lib/Target/PISA/PISAFrameLowering.h
+++ b/llvm/lib/Target/PISA/PISAFrameLowering.h
@@ -26,6 +26,16 @@ class PISAFrameLowering : public TargetFrameLowering {
MachineBasicBlock &MBB) const override {}
bool hasFPImpl(const MachineFunction &MF) const override { return false; }
+
+ bool isSupportedStackID(TargetStackID::Value ID) const override {
+ switch (ID) {
+ default:
+ return false;
+ case TargetStackID::Default:
+ case TargetStackID::PISAShared:
+ return true;
+ }
+ }
};
} // namespace llvm
#endif // LLVM_LIB_TARGET_PISA_PISAFRAMELOWERING_H
diff --git a/llvm/lib/Target/PISA/PISAISelLowering.cpp b/llvm/lib/Target/PISA/PISAISelLowering.cpp
new file mode 100644
index 0000000000000..377303af97321
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAISelLowering.cpp
@@ -0,0 +1,712 @@
+//===-- PISAISelLowering.cpp - PISA DAG Lowering Impl ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAISelLowering.h"
+#include "PISA.h"
+#include "PISACacheCtrlMMRA.h"
+#include "PISASubtarget.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/KnownBits.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+#include <type_traits>
+
+#define DEBUG_TYPE "pisa-lower"
+
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+// Atomic legalization
+//
+// Atomics are legalized by LLVM's AtomicExpandPass, driven entirely through the
+// TargetLowering hooks below. These tables and helpers describe which native
+// atomic operations PISA supports and which operations need IR expansion.
+//===----------------------------------------------------------------------===//
+namespace {
+using namespace llvm::PISAAS;
+
+constexpr unsigned PrivateAS = static_cast<unsigned>(AddressSpace::PRIVATE);
+constexpr unsigned GenericAS = static_cast<unsigned>(AddressSpace::GENERIC);
+constexpr unsigned SharedAS = static_cast<unsigned>(AddressSpace::SHARED);
+constexpr unsigned GlobalAS = static_cast<unsigned>(AddressSpace::GLOBAL);
+
+// clang-format off
+const bool SupportedGlobal[AtomicRMWInst::LAST_BINOP + 1][4] = {
+ // 16b 32b 64b 128b
+ { true, true, true, true}, // Xchg
+ { true, true, true, false}, // Add
+ { true, true, true, false}, // Sub
+ { true, true, true, false}, // And
+ { true, true, true, false}, // Nand
+ { true, true, true, false}, // Or
+ { true, true, true, false}, // Xor
+ { true, true, true, false}, // Max
+ { true, true, true, false}, // Min
+ { true, true, true, false}, // Umax
+ { true, true, true, false}, // Umin
+ { true, true, true, false}, // FAdd
+ { true, true, true, false}, // FSub
+ { true, true, false, false}, // FMax
+ { true, true, false, false}, // FMin
+ {false, false, false, false}, // FMaximum
+ {false, false, false, false}, // FMinimum
+ {false, false, false, false}, // FMaximumNum
+ {false, false, false, false}, // FMinimumNum
+ {false, true, true, false}, // Uinc_wrap
+ {false, true, true, false}, // Udec_wrap
+ {false, false, false, false}, // Usub_cond
+ {false, false, false, false}, // Usub_sat
+};
+// clang-format on
+
+constexpr uint8_t SupportedGlobalCols =
+ std::extent<decltype(SupportedGlobal), 1>::value;
+constexpr uint8_t SupportedGlobalRows =
+ std::extent<decltype(SupportedGlobal), 0>::value;
+
+// clang-format off
+const bool SupportedShared[AtomicRMWInst::LAST_BINOP + 1][4] = {
+ // 16b 32b 64b 128b
+ { true, true, false, true}, // Xchg
+ { true, true, false, false}, // Add
+ { true, true, false, false}, // Sub
+ { true, true, false, false}, // And
+ { true, true, false, false}, // Nand
+ { true, true, false, false}, // Or
+ { true, true, false, false}, // Xor
+ { true, true, false, false}, // Max
+ { true, true, false, false}, // Min
+ { true, true, false, false}, // Umax
+ { true, true, false, false}, // Umin
+ { true, true, false, false}, // FAdd
+ { true, true, false, false}, // FSub
+ { true, true, false, false}, // FMax
+ { true, true, false, false}, // FMin
+ {false, false, false, false}, // FMaximum
+ {false, false, false, false}, // FMinimum
+ {false, false, false, false}, // FMaximumNum
+ {false, false, false, false}, // FMinimumNum
+ {false, true, false, false}, // Uinc_wrap
+ {false, true, false, false}, // Udec_wrap
+ {false, false, false, false}, // Usub_cond
+ {false, false, false, false}, // Usub_sat
+};
+// clang-format on
+
+constexpr uint8_t SupportedSharedCols =
+ std::extent<decltype(SupportedShared), 1>::value;
+constexpr uint8_t SupportedSharedRows =
+ std::extent<decltype(SupportedShared), 0>::value;
+
+const bool SupportedLoadStoreGlobal[4] = {
+ true, // 16b
+ true, // 32b
+ true, // 64b
+ true, // 128b
+};
+
+const bool SupportedLoadStoreShared[4] = {
+ true, // 16b
+ true, // 32b
+ false, // 64b
+ true, // 128b
+};
+
+bool isLegalLoadStore(unsigned BitWidth, unsigned AddrSpace) {
+ // Don't legalize loads/stores of unsupported bitwidths
+ if (BitWidth < 16 || BitWidth > 128)
+ return true;
+
+ unsigned TableCol = Log2_32(BitWidth / 16);
+ switch (AddrSpace) {
+ case GlobalAS:
+ return SupportedLoadStoreGlobal[TableCol];
+ case SharedAS:
+ return SupportedLoadStoreShared[TableCol];
+ case GenericAS:
+ return SupportedLoadStoreGlobal[TableCol] &&
+ SupportedLoadStoreShared[TableCol];
+ }
+
+ // Don't legalize loads/stores in other address spaces
+ return false;
+}
+
+bool isAtomicRMWLegal(const AtomicRMWInst *A) {
+ Type *Ty = A->getType();
+ const DataLayout &DL = A->getDataLayout();
+ unsigned BitWidth = DL.getTypeSizeInBits(Ty);
+ if (BitWidth < 16 || BitWidth > 128)
+ return false;
+ unsigned TableCol = Log2_32(BitWidth / 16);
+ unsigned Op = A->getOperation();
+
+ unsigned AS = A->getPointerAddressSpace();
+ if (((AS != SharedAS) &&
+ (TableCol >= SupportedGlobalCols || Op >= SupportedGlobalRows)) ||
+ ((AS != GlobalAS) &&
+ (TableCol >= SupportedSharedCols || Op >= SupportedSharedRows)))
+ return false;
+ switch (AS) {
+ default:
+ return false;
+ case GlobalAS:
+ return SupportedGlobal[Op][TableCol];
+ case SharedAS:
+ return SupportedShared[Op][TableCol];
+ case GenericAS:
+ return SupportedGlobal[Op][TableCol] && SupportedShared[Op][TableCol];
+ }
+}
+
+TargetLowering::AtomicExpansionKind
+computeRMWExpansion(const AtomicRMWInst *A) {
+ using Kind = TargetLowering::AtomicExpansionKind;
+ switch (A->getPointerAddressSpace()) {
+ case PrivateAS:
+ return Kind::NotAtomic; // load-op-store, single work-item
+ case GenericAS:
+ // Legal in BOTH global and shared -> native generic; else runtime dispatch.
+ return isAtomicRMWLegal(A) ? Kind::None : Kind::CustomExpand;
+ case GlobalAS:
+ case SharedAS:
+ return isAtomicRMWLegal(A) ? Kind::None : Kind::CmpXChg;
+ default:
+ return Kind::None;
+ }
+}
+
+SyncScope::ID hoistedFenceScope(Instruction *Inst) {
+ // An AtomicRMWInst (the CAS-loop op, incl. the xchg from an expanded store)
+ // is what reaches here today, but stay robust to other atomic instructions.
+ SyncScope::ID SSID = getAtomicSyncScopeID(Inst).value_or(SyncScope::System);
+ // Shared memory has no meaningful system scope, so narrow the scope used for
+ // hoisted fences to the widest valid shared-memory scope.
+ if (const auto *RMW = dyn_cast<AtomicRMWInst>(Inst))
+ if (RMW->getPointerAddressSpace() == SharedAS && SSID == SyncScope::System)
+ return Inst->getContext().getOrInsertSyncScopeID("gpu-shared");
+ return SSID;
+}
+} // namespace
+
+PISATargetLowering::PISATargetLowering(const TargetMachine &TM,
+ const PISASubtarget &STI)
+ : TargetLowering(TM, STI) {
+ // Route atomics through AtomicExpandPass. PISA supports up to
+ // 128-bit atomics; without this every atomic wider than the default falls
+ // back to an unsupported __atomic_* libcall.
+ setMaxAtomicSizeInBitsSupported(128);
+
+ // these numbers need to be large enough to cover cases that are not
+ // expanded by PISAExpandIntrinsics into a ld-st loop.
+ MaxStoresPerMemcpy = 64;
+ MaxStoresPerMemmove = 64;
+ MaxStoresPerMemset = 64;
+
+ // map int types to registers classes
+ addRegisterClass(MVT::i8, &PISA::Reg8bRegClass);
+ addRegisterClass(MVT::i16, &PISA::Reg16bRegClass);
+ addRegisterClass(MVT::i32, &PISA::Reg32bRegClass);
+ addRegisterClass(MVT::i64, &PISA::Reg64bRegClass);
+ addRegisterClass(MVT::bf16, &PISA::Reg16bRegClass);
+ addRegisterClass(MVT::f16, &PISA::Reg16bRegClass);
+ addRegisterClass(MVT::f32, &PISA::Reg32bRegClass);
+ addRegisterClass(MVT::f64, &PISA::Reg64bRegClass);
+ addRegisterClass(MVT::v2i8, &PISA::RegV2_8bRegClass);
+ addRegisterClass(MVT::v3i8, &PISA::RegV3_8bRegClass);
+ addRegisterClass(MVT::v4i8, &PISA::RegV4_8bRegClass);
+ addRegisterClass(MVT::v2i16, &PISA::RegV2_16bRegClass);
+ addRegisterClass(MVT::v3i16, &PISA::RegV3_16bRegClass);
+ addRegisterClass(MVT::v4i16, &PISA::RegV4_16bRegClass);
+ addRegisterClass(MVT::v2i32, &PISA::RegV2_32bRegClass);
+ addRegisterClass(MVT::v3i32, &PISA::RegV3_32bRegClass);
+ addRegisterClass(MVT::v4i32, &PISA::RegV4_32bRegClass);
+ addRegisterClass(MVT::v5i32, &PISA::RegV5_32bRegClass);
+ addRegisterClass(MVT::v6i32, &PISA::RegV6_32bRegClass);
+ addRegisterClass(MVT::v7i32, &PISA::RegV7_32bRegClass);
+ addRegisterClass(MVT::v8i32, &PISA::RegV8_32bRegClass);
+ addRegisterClass(MVT::v16i32, &PISA::RegV16_32bRegClass);
+ addRegisterClass(MVT::v32i32, &PISA::RegV32_32bRegClass);
+ addRegisterClass(MVT::v64i32, &PISA::RegV64_32bRegClass);
+ addRegisterClass(MVT::v2i64, &PISA::RegV2_64bRegClass);
+ addRegisterClass(MVT::v3i64, &PISA::RegV3_64bRegClass);
+ addRegisterClass(MVT::v4i64, &PISA::RegV4_64bRegClass);
+ // map floating-point types to registers classes
+ addRegisterClass(MVT::v2f16, &PISA::RegV2_16bRegClass);
+ addRegisterClass(MVT::v3f16, &PISA::RegV3_16bRegClass);
+ addRegisterClass(MVT::v4f16, &PISA::RegV4_16bRegClass);
+ addRegisterClass(MVT::v2bf16, &PISA::RegV2_16bRegClass);
+ addRegisterClass(MVT::v3bf16, &PISA::RegV3_16bRegClass);
+ addRegisterClass(MVT::v4bf16, &PISA::RegV4_16bRegClass);
+ addRegisterClass(MVT::v2f32, &PISA::RegV2_32bRegClass);
+ addRegisterClass(MVT::v3f32, &PISA::RegV3_32bRegClass);
+ addRegisterClass(MVT::v4f32, &PISA::RegV4_32bRegClass);
+ addRegisterClass(MVT::v5f32, &PISA::RegV5_32bRegClass);
+ addRegisterClass(MVT::v6f32, &PISA::RegV6_32bRegClass);
+ addRegisterClass(MVT::v7f32, &PISA::RegV7_32bRegClass);
+ addRegisterClass(MVT::v8f32, &PISA::RegV8_32bRegClass);
+ addRegisterClass(MVT::v2f64, &PISA::RegV2_64bRegClass);
+ addRegisterClass(MVT::v3f64, &PISA::RegV3_64bRegClass);
+ addRegisterClass(MVT::v4f64, &PISA::RegV4_64bRegClass);
+ // must be done after all classes are added
+ computeRegisterProperties(STI.getRegisterInfo());
+
+ // Jump is Expensive. Don't create extra control flow for 'and', 'or'
+ // condition branches.
+ setJumpIsExpensive(true);
+ setMaxDivRemBitWidthSupported(64);
+}
+
+unsigned PISATargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC,
+ EVT VT) const {
+ // This code avoids CallLowering fail inside getVectorTypeBreakdown
+ // on v3i1 arguments. Maybe we need to return 1 for all types.
+ // TODO: remove it once this case is supported by the default implementation.
+ if (VT.isVector() && VT.getVectorNumElements() == 3 &&
+ (VT.getVectorElementType() == MVT::i1 ||
+ VT.getVectorElementType() == MVT::i8))
+ return 1;
+ return getNumRegisters(Context, VT);
+}
+
+bool PISATargetLowering::isCheapToSpeculateCttz(Type *Ty) const { return true; }
+
+bool PISATargetLowering::isCheapToSpeculateCtlz(Type *Ty) const { return true; }
+
+bool PISATargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
+ Register N0, Register N1) const {
+ auto GetOneDefInst = [&MRI](Register N) -> MachineInstr * {
+ auto *Def = MRI.getOneDef(N);
+ if (Def)
+ return Def->getParent();
+ return nullptr;
+ };
+
+ auto *I0 = GetOneDefInst(N0);
+ auto *I1 = GetOneDefInst(N1);
+ if (I0 && I1) {
+ // Prevent reassociating the following pattern
+ // (add (mul a, b), (add (mul c, d), e))
+ // into
+ // (add (add (mul a, b), (mul c, d)), e)
+ // so that more 'mad's could be selected.
+ if (I0->getOpcode() != TargetOpcode::G_MUL)
+ std::swap(I0, I1);
+ if (I0->getOpcode() == TargetOpcode::G_MUL &&
+ I1->getOpcode() == TargetOpcode::G_ADD) {
+ auto *NI0 = GetOneDefInst(I1->getOperand(1).getReg());
+ auto *NI1 = GetOneDefInst(I1->getOperand(2).getReg());
+ if (NI0 && NI1 &&
+ (NI0->getOpcode() == TargetOpcode::G_MUL ||
+ NI1->getOpcode() == TargetOpcode::G_MUL)) {
+ // Don't reassociate to break the selection of MAD.
+ return false;
+ }
+ }
+ }
+
+ return TargetLowering::isReassocProfitable(MRI, N0, N1);
+}
+
+MVT PISATargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC,
+ EVT VT) const {
+ // This code avoids CallLowering fail inside getVectorTypeBreakdown
+ // on v3i1 arguments. Maybe we need to return i32 for all types.
+ // TODO: remove it once this case is supported by the default implementation.
+ if (VT.isVector() && VT.getVectorNumElements() == 3) {
+ if (VT.getVectorElementType() == MVT::i1)
+ return MVT::v4i1;
+ if (VT.getVectorElementType() == MVT::i8)
+ return MVT::v4i8;
+ }
+ return getRegisterType(Context, VT);
+}
+
+void PISATargetLowering::getTgtMemIntrinsic(
+ SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
+ MachineFunction &MF, unsigned Intrinsic) const {
+ IntrinsicInfo Info;
+ Info.flags = MachineMemOperand::MONone;
+ switch (Intrinsic) {
+ case Intrinsic::pisa_cas_fatom:
+ Info.memVT = MVT::getVT(I.getType());
+ Info.ptrVal = I.getArgOperand(0);
+ Info.align.reset();
+ Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+ Info.flags |= getTargetMMOFlags(I);
+ // syncscope("<target-scope>") support for atomics
+ if (auto *ConstInt = dyn_cast<ConstantInt>(I.getArgOperand(3)))
+ Info.order = static_cast<llvm::AtomicOrdering>(ConstInt->getZExtValue());
+ Infos.push_back(Info);
+ return;
+ default:
+ break;
+ }
+ return;
+}
+
+LLT PISATargetLowering::getOptimalMemOpLLT(
+ const MemOp &Op, const AttributeList &FuncAttributes) const {
+ auto I8 = LLT::integer(8);
+ auto I16 = LLT::integer(16);
+ auto I32 = LLT::integer(32);
+ auto I64 = LLT::integer(64);
+
+ if (Op.size() >= 16 && Op.isAligned(Align(8)))
+ return LLT::fixed_vector(2, I64);
+ if (Op.size() >= 16 && Op.isAligned(Align(4)))
+ return LLT::fixed_vector(4, I32);
+ if (Op.size() >= 12 && Op.isAligned(Align(4)))
+ return LLT::fixed_vector(3, I32);
+ if (Op.size() >= 8 && Op.isAligned(Align(4)))
+ return LLT::fixed_vector(2, I32);
+ if (Op.size() >= 8 && Op.isAligned(Align(2)))
+ return LLT::fixed_vector(4, I16);
+ if (Op.size() >= 4 && Op.isAligned(Align(4)))
+ return I32;
+ if (Op.size() >= 4 && Op.isAligned(Align(2)))
+ return LLT::fixed_vector(2, I16);
+ if (Op.size() >= 4 && Op.isAligned(Align(1)))
+ return LLT::fixed_vector(4, I8);
+ if (Op.size() >= 2 && Op.isAligned(Align(2)))
+ return I16;
+ if (Op.size() >= 2 && Op.isAligned(Align(1)))
+ return LLT::fixed_vector(2, I8);
+ if (Op.size() >= 1 && Op.isAligned(Align(1)))
+ return I8;
+ return LLT();
+}
+
+bool PISATargetLowering::useFTZ(const MachineFunction &MF) const {
+ return MF.getDenormalMode(APFloat::IEEEsingle()).Output ==
+ DenormalMode::PreserveSign;
+}
+
+TargetLowering::ConstraintType
+PISATargetLowering::getConstraintType(StringRef Constraint) const {
+ if (Constraint.size() == 1) {
+ switch (Constraint[0]) {
+ default:
+ break;
+ case 'P': // Predicate register.
+ return C_RegisterClass;
+ }
+ }
+
+ return TargetLowering::getConstraintType(Constraint);
+}
+
+std::pair<unsigned, const TargetRegisterClass *>
+PISATargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
+ StringRef Constraint,
+ MVT VT) const {
+ using namespace PISA;
+ static const TargetRegisterClass *Vector8BitClass[] = {
+ &Reg8bRegClass, &RegV2_8bRegClass, &RegV3_8bRegClass, &RegV4_8bRegClass};
+ static const TargetRegisterClass *Vector16BitClass[] = {
+ &Reg16bRegClass, &RegV2_16bRegClass, &RegV3_16bRegClass,
+ &RegV4_16bRegClass};
+ static const TargetRegisterClass *Vector32BitClass[] = {
+ &Reg32bRegClass, &RegV2_32bRegClass, &RegV3_32bRegClass,
+ &RegV4_32bRegClass, &RegV5_32bRegClass, &RegV6_32bRegClass,
+ &RegV7_32bRegClass, &RegV8_32bRegClass,
+ };
+ static const TargetRegisterClass *Vector64BitClass[] = {
+ &Reg64bRegClass, &RegV2_64bRegClass, &RegV3_64bRegClass,
+ &RegV4_64bRegClass};
+
+ if (Constraint.size() == 1) {
+ const TargetRegisterClass *RC = nullptr;
+ switch (Constraint[0]) {
+ default:
+ break;
+ case 'P':
+ RC = &PredRegClass;
+ break;
+ case 'r': {
+ // FIXME: we already have a method to get the register class from LLT
+ auto VectorSize = VT.isVector() ? VT.getVectorNumElements() : 1;
+ auto ElementSize = VT.getScalarSizeInBits();
+ assert(VectorSize >= 1 &&
+ (ElementSize == 32 ? VectorSize <= 8 || VectorSize == 16 ||
+ VectorSize == 32 || VectorSize == 64
+ : VectorSize <= 4));
+
+ switch (ElementSize) {
+ case 8:
+ RC = Vector8BitClass[VectorSize - 1];
+ break;
+ case 16:
+ RC = Vector16BitClass[VectorSize - 1];
+ break;
+ case 32:
+ if (VectorSize == 64)
+ RC = &RegV64_32bRegClass;
+ else if (VectorSize == 32)
+ RC = &RegV32_32bRegClass;
+ else if (VectorSize == 16)
+ RC = &RegV16_32bRegClass;
+ else
+ RC = Vector32BitClass[VectorSize - 1];
+ break;
+ case 64:
+ RC = Vector64BitClass[VectorSize - 1];
+ break;
+ }
+ } break;
+ }
+
+ if (RC)
+ return std::make_pair(0u, RC);
+ }
+
+ return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
+}
+
+MachineMemOperand::Flags
+PISATargetLowering::getTargetMMOFlags(const Instruction &I) const {
+ MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
+ if (auto Hint = PISA::getCacheCtrlFromMMRA(I)) {
+ auto HintValue = *Hint & 0xF;
+ Flags |= static_cast<MachineMemOperand::Flags>(HintValue << 6);
+ }
+ return Flags;
+}
+
+void PISATargetLowering::computeKnownBitsForTargetInstr(
+ GISelValueTracking &Analysis, Register R, KnownBits &Known,
+ const APInt &DemandedElts, const MachineRegisterInfo &MRI,
+ unsigned Depth) const {
+ MachineInstr *MI = MRI.getVRegDef(R);
+
+ // As we go we can add more cases here for now only enable for
+ // G_INTRINSIC for using for folding range attributes
+ if (!MI || MI->getOpcode() != TargetOpcode::G_INTRINSIC)
+ return;
+
+ Intrinsic::ID IID = cast<GIntrinsic>(*MI).getIntrinsicID();
+
+ if (Intrinsic::isOverloaded(IID))
+ return;
+
+ auto *Ctx = &MI->getMF()->getFunction().getContext();
+ FunctionType *FT = Intrinsic::getType(*Ctx, IID);
+ AttributeList Attrs = Intrinsic::getAttributes(*Ctx, IID, FT);
+
+ if (Attrs.hasRetAttr(Attribute::Range)) {
+ const ConstantRange &CR =
+ Attrs.getRetAttr(Attribute::Range).getValueAsConstantRange();
+ Known = CR.toKnownBits();
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Atomic legalization hooks
+//===----------------------------------------------------------------------===//
+
+TargetLowering::AtomicExpansionKind
+PISATargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
+ return computeRMWExpansion(RMW);
+}
+
+TargetLowering::AtomicExpansionKind
+PISATargetLowering::shouldExpandAtomicCmpXchgInIR(
+ const AtomicCmpXchgInst *CI) const {
+ // Private memory is single work-item, so cmpxchg can become a plain
+ // load/compare/conditional-store. Other address spaces keep cmpxchg semantics
+ // and are left for native backend selection.
+ return CI->getPointerAddressSpace() == PrivateAS
+ ? AtomicExpansionKind::NotAtomic
+ : AtomicExpansionKind::None;
+}
+
+TargetLowering::AtomicExpansionKind
+PISATargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
+ unsigned AS = LI->getPointerAddressSpace();
+ if (AS == PrivateAS)
+ return AtomicExpansionKind::NotAtomic; // plain load
+ unsigned BW = LI->getDataLayout().getTypeSizeInBits(LI->getType());
+ if (BW < 16 || BW > 128)
+ return AtomicExpansionKind::None; // inverted vs rmw: leave native
+ if (AS == GlobalAS || AS == SharedAS || AS == GenericAS) {
+ if (isLegalLoadStore(BW, AS))
+ return AtomicExpansionKind::None;
+ // Illegal-width atomic load: emulate with an integer cmpxchg in
+ // emitExpandAtomicLoad. AtomicExpand's generic CmpXChg path
+ // (expandAtomicLoadToCmpXchg) is unsuitable for PISA:
+ // - Pointers: it casts only FP/vector to integer, so a pointer load
+ // stays a pointer cmpxchg, which asserts (isScalar) in IRTranslator.
+ // emitExpandAtomicLoad instead emits a same-width integer cmpxchg that
+ // preserves the syncscope, then casts the loaded bits back.
+ return AtomicExpansionKind::CustomExpand;
+ }
+ return AtomicExpansionKind::None;
+}
+
+TargetLowering::AtomicExpansionKind
+PISATargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
+ unsigned AS = SI->getPointerAddressSpace();
+ if (AS == PrivateAS)
+ return AtomicExpansionKind::NotAtomic; // plain store
+ Type *ValTy = SI->getValueOperand()->getType();
+ unsigned BW = SI->getDataLayout().getTypeSizeInBits(ValTy);
+ if (BW < 16 || BW > 128)
+ return AtomicExpansionKind::None; // leave native
+ if (AS == GlobalAS || AS == SharedAS || AS == GenericAS) {
+ if (isLegalLoadStore(BW, AS))
+ return AtomicExpansionKind::None;
+ // Illegal-width atomic store: emulate with an integer xchg in
+ // emitExpandAtomicStore. AtomicExpand's generic Expand path
+ // (store -> xchg, expandAtomicStore) is unsuitable for PISA:
+ // - Fences: it re-expands via a direct tryExpandAtomicRMW that skips
+ // the driver's fence-hoisting path, so no leading fence is hoisted.
+ // - Pointers: it feeds a pointer value into the CAS loop, where
+ // createCmpXchgInstFun asserts on the pointer operand.
+ // emitExpandAtomicStore casts the value to a same-width integer and
+ // splits the block so the driver re-walks the xchg and applies fence
+ // hoisting + gpu-shared narrowing, with the scope preserved.
+ return AtomicExpansionKind::CustomExpand;
+ }
+ return AtomicExpansionKind::None;
+}
+
+bool PISATargetLowering::shouldInsertFencesForAtomic(
+ const Instruction *I) const {
+ // Use AtomicExpand fence splitting only for RMWs that become CAS loops.
+ // Native atomics keep their ordering.
+ if (const auto *RMW = dyn_cast<AtomicRMWInst>(I))
+ return computeRMWExpansion(RMW) == AtomicExpansionKind::CmpXChg;
+ return false;
+}
+
+Instruction *PISATargetLowering::emitLeadingFence(IRBuilderBase &Builder,
+ Instruction *Inst,
+ AtomicOrdering Ord) const {
+ if (!isReleaseOrStronger(Ord))
+ return nullptr;
+ AtomicOrdering FenceOrd = (Ord == AtomicOrdering::SequentiallyConsistent)
+ ? Ord
+ : AtomicOrdering::Release;
+ return Builder.CreateFence(FenceOrd, hoistedFenceScope(Inst));
+}
+
+Instruction *PISATargetLowering::emitTrailingFence(IRBuilderBase &Builder,
+ Instruction *Inst,
+ AtomicOrdering Ord) const {
+ if (!isAcquireOrStronger(Ord))
+ return nullptr;
+ AtomicOrdering FenceOrd = (Ord == AtomicOrdering::SequentiallyConsistent)
+ ? Ord
+ : AtomicOrdering::Acquire;
+ return Builder.CreateFence(FenceOrd, hoistedFenceScope(Inst));
+}
+
+void PISATargetLowering::emitExpandAtomicRMW(AtomicRMWInst *A) const {
+ // Generic pointer in atomicrmw points to GLOBAL or SHARED (PRIVATE is UB per
+ // OpenCL). Probe the real address space at runtime with pisa_isaddr_shared
+ // and branch; AtomicExpandPass then re-walks and expands each arm
+ // independently (global -> native, shared -> CAS loop + hoisted fences).
+ LLVMContext &Ctx = A->getContext();
+ Function *F = A->getFunction();
+ IRBuilder<> IR(A);
+
+ BasicBlock *OrigBB = A->getParent();
+ BasicBlock *JoinBB = OrigBB->splitBasicBlock(A->getNextNode());
+
+ Type *I32 = IR.getInt32Ty();
+ Value *IsShared = IR.CreateIntrinsic(I32, Intrinsic::pisa_isaddr_shared,
+ {A->getPointerOperand()});
+ Value *SharedCmp = IR.CreateICmpNE(IsShared, ConstantInt::get(I32, 0));
+
+ BasicBlock *GlobalBB = BasicBlock::Create(Ctx, "", F, JoinBB);
+ BasicBlock *SharedBB = BasicBlock::Create(Ctx, "", F, JoinBB);
+
+ IR.CreateCondBr(SharedCmp, SharedBB, GlobalBB);
+ OrigBB->getTerminator()->eraseFromParent();
+
+ IR.SetInsertPoint(JoinBB->begin());
+ PHINode *Res = IR.CreatePHI(A->getType(), 2);
+
+ auto EmitArm = [&](BasicBlock *BB, unsigned AS) {
+ IR.SetInsertPoint(BB);
+ Value *Cast = IR.CreateAddrSpaceCast(A->getPointerOperand(),
+ PointerType::get(Ctx, AS));
+ AtomicRMWInst *NewA = IR.CreateAtomicRMW(
+ A->getOperation(), Cast, A->getValOperand(), A->getAlign(),
+ A->getOrdering(), A->getSyncScopeID());
+ Res->addIncoming(NewA, IR.GetInsertBlock());
+ IR.CreateBr(JoinBB);
+ };
+ EmitArm(GlobalBB, GlobalAS);
+ EmitArm(SharedBB, SharedAS);
+
+ A->replaceAllUsesWith(Res);
+ A->eraseFromParent();
+}
+
+void PISATargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
+ // Reached for every illegal-width atomic store (integer, FP, or pointer).
+ // PISA's xchg/CAS expansion operates on integers, so cast the value to an
+ // integer of the same width and rewrite the store as an integer atomicrmw
+ // xchg while preserving the sync scope. Place it in a fresh block so the
+ // AtomicExpand driver loop revisits and CAS-expands it with hoisted,
+ // scope-narrowed fences (it does not re-walk ops created in the block
+ // currently being processed).
+ BasicBlock *BB = SI->getParent();
+ BB->splitBasicBlock(SI->getIterator());
+
+ IRBuilder<> IR(SI);
+ Value *Val = SI->getValueOperand();
+ Type *IntTy =
+ IR.getIntNTy(SI->getDataLayout().getTypeSizeInBits(Val->getType()));
+ Value *IntVal =
+ Val->getType() == IntTy ? Val : IR.CreateBitOrPointerCast(Val, IntTy);
+ AtomicOrdering Ord = SI->getOrdering() == AtomicOrdering::Unordered
+ ? AtomicOrdering::Monotonic
+ : SI->getOrdering();
+ IR.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(), IntVal,
+ SI->getAlign(), Ord, SI->getSyncScopeID());
+ SI->eraseFromParent();
+}
+
+void PISATargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
+ // Emulate an illegal-width atomic load with an integer cmpxchg (PISA's CAS
+ // needs scalar operands) using zero for both compare and new value, then cast
+ // the loaded bits back to the original type. The integer representation keeps
+ // FP/pointer loads compatible with cmpxchg while preserving the sync scope.
+ //
+ // Note the deliberate load/store asymmetry: the load's emulating cmpxchg
+ // keeps the original ordering and scope (it is not an AtomicRMWInst, so the
+ // fence gate does not hoist/narrow it), whereas an illegal store becomes a
+ // relaxed CAS loop bracketed by hoisted fences in emitExpandAtomicStore.
+ IRBuilder<> IR(LI);
+ Type *Ty = LI->getType();
+ unsigned BW = LI->getDataLayout().getTypeSizeInBits(Ty);
+ Type *IntTy = IR.getIntNTy(BW);
+ AtomicOrdering Ord = LI->getOrdering() == AtomicOrdering::Unordered
+ ? AtomicOrdering::Monotonic
+ : LI->getOrdering();
+ Constant *Zero = ConstantInt::get(IntTy, 0);
+ Value *Pair = IR.CreateAtomicCmpXchg(
+ LI->getPointerOperand(), Zero, Zero, LI->getAlign(), Ord,
+ AtomicCmpXchgInst::getStrongestFailureOrdering(Ord),
+ LI->getSyncScopeID());
+ Value *Loaded = IR.CreateExtractValue(Pair, 0);
+ if (Loaded->getType() != Ty)
+ Loaded = IR.CreateBitOrPointerCast(Loaded, Ty);
+ LI->replaceAllUsesWith(Loaded);
+ LI->eraseFromParent();
+}
diff --git a/llvm/lib/Target/PISA/PISAISelLowering.h b/llvm/lib/Target/PISA/PISAISelLowering.h
new file mode 100644
index 0000000000000..08096962b55cb
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAISelLowering.h
@@ -0,0 +1,112 @@
+//===-- PISAISelLowering.h - PISA DAG Lowering Interface ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAISELLOWERING_H
+#define LLVM_LIB_TARGET_PISA_PISAISELLOWERING_H
+
+#include "llvm/CodeGen/TargetLowering.h"
+
+namespace llvm {
+class PISASubtarget;
+
+class PISATargetLowering : public TargetLowering {
+public:
+ explicit PISATargetLowering(const TargetMachine &TM,
+ const PISASubtarget &STI);
+
+ bool isCheapToSpeculateCttz(Type *Ty) const override;
+
+ bool isCheapToSpeculateCtlz(Type *Ty) const override;
+
+ bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0,
+ Register N1) const override;
+
+ // Stop IRTranslator breaking up FMA instrs to preserve types information.
+ bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ EVT) const override {
+ return true;
+ }
+ bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+ LLT) const override {
+ return true;
+ }
+ bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *) const override {
+ return true;
+ }
+
+ // This is to prevent sexts of non-i64 vector indices which are generated
+ // within general IRTranslator hence type generation for it is omitted.
+ unsigned getVectorIdxWidth(const DataLayout &DL) const override { return 32; }
+
+ unsigned getNumRegistersForCallingConv(LLVMContext &Context,
+ CallingConv::ID CC,
+ EVT VT) const override;
+ MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
+ EVT VT) const override;
+ void getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
+ const CallBase &I, MachineFunction &MF,
+ unsigned Intrinsic) const override;
+ LLT getOptimalMemOpLLT(const MemOp &Op,
+ const AttributeList &FuncAttributes) const override;
+
+ bool useFTZ(const MachineFunction &MF) const;
+
+ bool areJTsAllowed(const Function *Fn) const override { return false; }
+ bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override {
+ return false;
+ }
+ ConstraintType getConstraintType(StringRef Constraint) const override;
+
+ std::pair<unsigned, const TargetRegisterClass *>
+ getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
+ StringRef Constraint, MVT VT) const override;
+
+ MachineMemOperand::Flags
+ getTargetMMOFlags(const Instruction &I) const override;
+
+ bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override {
+ return false;
+ }
+ bool isFMADLegal(const MachineInstr &MI, const LLT Ty) const override {
+ return false;
+ }
+
+ void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R,
+ KnownBits &Known,
+ const APInt &DemandedElts,
+ const MachineRegisterInfo &MRI,
+ unsigned Depth = 0) const override;
+
+ // --- Atomic legalization: driven by AtomicExpandPass ---
+ AtomicExpansionKind
+ shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override;
+ AtomicExpansionKind
+ shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *CI) const override;
+ AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
+ AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
+
+ bool shouldInsertFencesForAtomic(const Instruction *I) const override;
+
+ // Seed the CAS emulation loop with a plain load. Private/shared/global paths
+ // add the synchronization they need outside the loop, so the initial load
+ // does not need to be a redundant relaxed atomic load.
+ bool shouldIssueAtomicLoadForAtomicEmulationLoop() const override {
+ return false;
+ }
+
+ Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
+ AtomicOrdering Ord) const override;
+ Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
+ AtomicOrdering Ord) const override;
+ void emitExpandAtomicRMW(AtomicRMWInst *AI) const override;
+ void emitExpandAtomicStore(StoreInst *SI) const override;
+ void emitExpandAtomicLoad(LoadInst *LI) const override;
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISAISELLOWERING_H
diff --git a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
new file mode 100644
index 0000000000000..0597d75b67edf
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
@@ -0,0 +1,3513 @@
+//===-- PISALegalizerInfo.cpp --- PISA Legalization Rules -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISALegalizerInfo.h"
+#include "PISA.h"
+#include "PISASubtarget.h"
+#include "PISATargetMachine.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/IR/PISAIntrinsicUtils.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+using namespace llvm;
+using namespace llvm::LegalizeActions;
+using namespace llvm::LegalizeMutations;
+using namespace llvm::LegalityPredicates;
+
+namespace {
+constexpr ElementCount EC0 = ElementCount::getFixed(0);
+constexpr ElementCount EC2 = ElementCount::getFixed(2);
+constexpr ElementCount EC4 = ElementCount::getFixed(4);
+
+// integer types
+constexpr LLT I1 = LLT(LLT::Kind::INTEGER, EC0, 1);
+constexpr LLT I8 = LLT(LLT::Kind::INTEGER, EC0, 8);
+constexpr LLT I16 = LLT(LLT::Kind::INTEGER, EC0, 16);
+constexpr LLT I32 = LLT(LLT::Kind::INTEGER, EC0, 32);
+constexpr LLT I64 = LLT(LLT::Kind::INTEGER, EC0, 64);
+constexpr LLT I128 = LLT(LLT::Kind::INTEGER, EC0, 128);
+
+constexpr LLT V2I8 = LLT::fixed_vector(2, I8);
+constexpr LLT V2I16 = LLT::fixed_vector(2, I16);
+constexpr LLT V2I32 = LLT::fixed_vector(2, I32);
+
+constexpr LLT V4I8 = LLT::fixed_vector(4, I8);
+
+// floating-point types
+constexpr LLT BF16 = LLT::bfloat16();
+constexpr LLT F16 = LLT::float16();
+constexpr LLT F32 = LLT::float32();
+constexpr LLT F64 = LLT::float64();
+
+// return true if natively supported type
+static bool isLegalType(LLT Ty, bool Vector = true) {
+ auto EltSize = Ty.getScalarSizeInBits();
+ if (Ty.isVector() && !Vector)
+ return false;
+ if (Ty.isVector()) {
+ auto NumElts = Ty.getNumElements();
+ if (EltSize == 32)
+ return NumElts <= 8 || NumElts == 16 || NumElts == 32 || NumElts == 64;
+ if (!llvm::isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 64)
+ return false;
+ return NumElts <= 4;
+ }
+ if (!llvm::isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 128)
+ return false;
+ return true;
+}
+
+LegalityPredicate is3x8BitVector(unsigned TypeIdx) {
+ return [=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[TypeIdx];
+ return Ty.isVector() && Ty.getScalarSizeInBits() == 8 &&
+ Ty.getNumElements() == 3;
+ };
+}
+
+LegalityPredicate isWiderThan2x16BitVector(unsigned TypeIdx) {
+ return [=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[TypeIdx];
+ return Ty.isVector() && Ty.getScalarSizeInBits() == 16 &&
+ Ty.getNumElements() > 2;
+ };
+}
+
+LegalityPredicate isFloatingPointType(unsigned TypeIdx) {
+ return [=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[TypeIdx];
+ return Ty.getScalarType().isFloat();
+ };
+}
+LegalizeMutation changeElementTypeToInteger(unsigned TypeIdx) {
+ return [=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[TypeIdx];
+ auto NewEltTy = LLT::integer(Ty.getScalarSizeInBits());
+ auto NewTy = Ty.isVector()
+ ? LLT::fixed_vector(Ty.getNumElements(), NewEltTy)
+ : NewEltTy;
+ return std::pair(TypeIdx, NewTy);
+ };
+}
+} // namespace
+
+/// Returns true if the given G_LOAD instruction operates on a vector of 5-7
+/// elements each of 32 bits and should be widened for better hardware
+/// utilization. Potentially can be used for other memory types.
+static bool shouldWidenLoad(unsigned int Opcode, const LLT Ty,
+ unsigned AddressSpace, uint64_t Alignbits) {
+ if (AddressSpace == static_cast<unsigned>(PISAAS::AddressSpace::SHARED)) {
+ return (Opcode == TargetOpcode::G_LOAD && Ty.isVector() &&
+ Ty.getScalarSizeInBits() == 32 && Ty.getNumElements() >= 5 &&
+ Ty.getNumElements() <= 7);
+ }
+ if (AddressSpace == static_cast<unsigned>(PISAAS::AddressSpace::GLOBAL) ||
+ AddressSpace == static_cast<unsigned>(PISAAS::AddressSpace::CONSTANT)) {
+ return (Opcode == TargetOpcode::G_LOAD && Ty.isVector() &&
+ Ty.getScalarSizeInBits() == 32 && Ty.getNumElements() >= 5 &&
+ Ty.getNumElements() <= 7 && Alignbits >= 64);
+ }
+ return false;
+}
+
+PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
+ using namespace TargetOpcode;
+
+ auto &TM = ST.getTargetLowering()->getTargetMachine();
+ auto GetPointerLlt = [&](PISAAS::AddressSpace Addrspace) {
+ uint32_t NumBits =
+ TM.getPointerSizeInBits(static_cast<unsigned>(Addrspace));
+ return LLT::pointer(static_cast<unsigned>(Addrspace), NumBits);
+ };
+
+ const LLT PrivatePtr = GetPointerLlt(PISAAS::AddressSpace::PRIVATE);
+ const LLT GlobalPtr = GetPointerLlt(PISAAS::AddressSpace::GLOBAL);
+ const LLT ConstantPtr = GetPointerLlt(PISAAS::AddressSpace::CONSTANT);
+ const LLT SharedPtr = GetPointerLlt(PISAAS::AddressSpace::SHARED);
+ const LLT GenericPtr = GetPointerLlt(PISAAS::AddressSpace::GENERIC);
+
+ const std::initializer_list<LLT> AddrSpaces64 = {GlobalPtr, ConstantPtr,
+ GenericPtr};
+ const std::initializer_list<LLT> AddrSpaces32 = {PrivatePtr, SharedPtr};
+
+ auto AllIntegers = {I8, I16, I32, I64};
+ auto AllFloats = {BF16, F16, F32, F64};
+ auto AllPtrs = {PrivatePtr, GlobalPtr, ConstantPtr, SharedPtr, GenericPtr};
+
+ getActionDefinitionsBuilder(
+ {G_FADD, G_FCONSTANT, G_FSUB, G_FMUL, G_FMINNUM, G_FMAXNUM, G_FMINIMUM,
+ G_FMAXIMUM, G_FNEG, G_FMA, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
+ G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN, G_FSQRT, G_INTRINSIC_TRUNC})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor(AllFloats)
+ .scalarize(0);
+
+ // G_FABS is lowered to bitwise AND to clear the sign bit (strict IEEE
+ // semantics). For nnan cases, llvm.pisa.fabs is used instead which maps
+ // directly to the PISA fabs instruction.
+ getActionDefinitionsBuilder(G_FABS)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, ElementCount::getFixed(4)))
+ .fewerElementsIf(isWiderThan2x16BitVector(0),
+ changeElementCountTo(0, EC2))
+ .customFor(AllFloats)
+ .customIf([](const LegalityQuery &Q) {
+ LLT Ty = Q.Types[0];
+ return Ty.isVector() && Ty.getNumElements() == 2 &&
+ Ty.getScalarSizeInBits() == 16;
+ })
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(
+ {G_ADD, G_SUB, G_MUL, G_SDIV, G_UDIV, G_SREM, G_UREM})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({I16, I32, I64})
+ .clampScalar(0, I16, I64)
+ .widenScalarToNextPow2(0)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_UMULO, G_SMULO})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .minScalar(0, I16)
+ .lower();
+
+ // leave these as scalar type for now, as they are used for legalization
+ // of e.g. shufflevector, which operates on both floating and integer types
+ getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .fewerElementsIf(is3x8BitVector(0), changeTo(0, V2I8))
+ .fewerElementsIf(isWiderThan2x16BitVector(0), changeTo(0, V2I16))
+ .bitcastIf(LegalityPredicate(([=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[0];
+ if (!Ty.isVector())
+ return false;
+ auto VecBitSize = Ty.getSizeInBits();
+ return VecBitSize == 32 || VecBitSize == 16;
+ })),
+ LegalizeMutation(([=](const LegalityQuery &Query) {
+ return std::pair(
+ 0, LLT::integer(Query.Types[0].getSizeInBits()));
+ })))
+ .legalFor({I16, I32, I64})
+ .widenScalarIf(
+ [=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[0];
+ return Ty.getSizeInBits() == 1;
+ },
+ [=](const LegalityQuery &Query) { return std::pair(0, I32); })
+ .widenScalarToNextPow2(0, 16)
+ .clampScalar(0, I16, I64)
+ .scalarize(0);
+
+ // prelegalizer rules (div_rem_to_divrem) that generate these are disabled
+ getActionDefinitionsBuilder({G_UDIVREM, G_SDIVREM}).unsupported();
+
+ getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I16, I32}, {I32, I32}, {I64, I32}})
+ .clampScalar(1, I32, I32)
+ .widenScalarToNextPow2(0, 16)
+ .clampScalar(0, I16, I64)
+ .scalarize(0)
+ .lower();
+
+ getActionDefinitionsBuilder(G_TRUNC)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .legalFor(
+ {{I8, I16}, {I16, I32}, {I8, I32}, {I32, I64}, {I16, I64}, {I8, I64}})
+ .customIf([=](const LegalityQuery &Query) {
+ return Query.Types[0].getScalarSizeInBits() == 1 ||
+ (Query.Types[1].getScalarSizeInBits() == 128 &&
+ !(Query.Types[0].getScalarSizeInBits() > 64));
+ })
+ .alwaysLegal();
+
+ getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .legalFor(
+ {{I16, I8}, {I32, I8}, {I64, I8}, {I32, I16}, {I64, I16}, {I64, I32}})
+ .customIf([=](const LegalityQuery &Query) {
+ auto DstSize = Query.Types[0].getScalarSizeInBits();
+ auto SrcSize = Query.Types[1].getScalarSizeInBits();
+ auto UseSelect = SrcSize == 1;
+ auto UseShuffle = (SrcSize % 8 == 0 && !isPowerOf2_32(SrcSize)) ||
+ (DstSize % 8 == 0 && !isPowerOf2_32(DstSize));
+ return UseSelect || UseShuffle;
+ })
+ .clampScalar(0, I16, I64)
+ .clampScalar(1, I16, I32);
+
+ getActionDefinitionsBuilder(G_SEXT_INREG).lower();
+
+ getActionDefinitionsBuilder({G_FPTRUNC, G_INTRINSIC_FPTRUNC_ROUND})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{BF16, F32}, {F16, F32}, {BF16, F64}, {F16, F64}, {F32, F64}})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_FPEXT)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{F32, BF16}, {F32, F16}, {F64, BF16}, {F64, F16}, {F64, F32}})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_CTPOP)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I16, I16}, {I32, I32}})
+ .clampScalar(0, I16, I32)
+ .clampScalar(1, I16, I32)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_CTTZ, G_CTLZ})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I16, I16}, {I32, I32}})
+ .clampScalar(0, I16, I32)
+ .clampScalar(1, I16, I32)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_CTTZ_ZERO_POISON, G_CTLZ_ZERO_POISON})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I16, I16}, {I32, I32}})
+ .clampScalar(0, I16, I32)
+ .clampScalar(1, I16, I64)
+ .maxScalar(1, I32)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_BITREVERSE)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({I32})
+ .clampScalar(0, I32, I32)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_FDIV)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .customFor({BF16, F16})
+ .legalFor({F32, F64})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_FREM)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .customFor({BF16, F16, F32, F64})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_CONSTANT)
+ .legalFor({I1, I8, I16, I32, I64})
+ .legalIf(isPointer(0))
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, I16, I64)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_PTR_ADD)
+ .legalIf(all(isPointer(0), sameSize(0, 1)))
+ .scalarize(0)
+ .scalarSameSizeAs(1, 0);
+
+ getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .customFor({{BF16, I32}, {F16, I32}, {F32, I32}, {F64, I32}})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_FSHR, G_FSHL})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I32, I32}})
+ .scalarize(0)
+ .lower();
+
+ getActionDefinitionsBuilder({G_ROTL, G_ROTR})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ // Scalarize vectors with i32 elements to enable lowering to scalar
+ // fshl/fshr.
+ .scalarizeIf(
+ [](const LegalityQuery &Query) {
+ return Query.Types[0].isVector() &&
+ Query.Types[0].getScalarSizeInBits() == 32;
+ },
+ 0)
+ .lower();
+
+ getActionDefinitionsBuilder(G_IS_FPCLASS).scalarize(0).custom();
+
+ /////////////////////////////////////////////////////////////////////////
+
+ getActionDefinitionsBuilder(G_GLOBAL_VALUE).alwaysLegal();
+
+ getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS,
+ G_INTRINSIC_CONVERGENT,
+ G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
+ .alwaysLegal();
+
+ getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
+ .customIf([](const LegalityQuery &Query) {
+ auto Ty = Query.Types[0];
+ return Ty.isVector() && (Ty.getScalarSizeInBits() == 32) &&
+ isPowerOf2_32(Ty.getNumElements());
+ })
+ .lower();
+
+ getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET})
+ .lower();
+
+ getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
+ .scalarize(0)
+ .customIf([=](const LegalityQuery &Query) -> bool {
+ unsigned DstAS = Query.Types[0].getAddressSpace();
+ unsigned SrcAS = Query.Types[1].getAddressSpace();
+ return (DstAS != (unsigned)PISAAS::AddressSpace::GENERIC) &&
+ (SrcAS != (unsigned)PISAAS::AddressSpace::GENERIC);
+ })
+ .legalForCartesianProduct(AllPtrs, AllPtrs);
+
+ getActionDefinitionsBuilder({G_LOAD, G_STORE})
+ .bitcastIf(isFloatingPointType(0), changeElementTypeToInteger(0))
+ // Handle sub-byte types: vectors with sub-byte elements (>1 bit) are
+ // bitcast to scalar, then widened to multiple of 8 bits. Sub-byte
+ // scalars are widened directly.
+ //.widenScalar does not update MI.memoperands()[0].getType(), hence
+ .customIf([=](const LegalityQuery &Query) -> bool {
+ auto Ty = Query.Types[0];
+ if (Ty.isVector() && (Ty.getScalarSizeInBits() > 1) &&
+ (Ty.getScalarSizeInBits() < 8))
+ return true;
+ if (!Ty.isVector() && ((Ty.getSizeInBits() % 8) != 0))
+ return true;
+ return false;
+ })
+ .fewerElementsIf(
+ [=](const LegalityQuery &Query) -> bool {
+ auto EltTy = Query.Types[0];
+ auto BitSize = EltTy.getScalarSizeInBits();
+ auto NumElts = EltTy.isVector() ? EltTy.getNumElements() : 1;
+ auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ // small (bitsize<32) vectors with non-power-of-2 elements
+ // can be broken into power-of-2 vectors that can be later
+ // upconverted to vectors of i32 for better codegen
+ return EltTy.isVector() && !isPowerOf2_32(NumElts) &&
+ BitSize != 1 && (BitSize < 32) && (BitSize < AlignInBits);
+ },
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ auto EltTy = Query.Types[0];
+ auto NumElts = EltTy.getNumElements();
+ auto NewNumElts = PowerOf2Ceil(NumElts) / 2;
+ return std::make_pair(
+ 0, LLT::fixed_vector(NewNumElts, EltTy.getScalarType()));
+ })
+ // split up vectors of non-standard size elements
+ .fewerElementsIf(
+ [=](const LegalityQuery &Query) -> bool {
+ auto EltTy = Query.Types[0];
+ return EltTy.isVector() &&
+ !isPowerOf2_32(EltTy.getScalarSizeInBits());
+ },
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ auto EltTy = Query.Types[0];
+ return std::make_pair(0, EltTy.getScalarType());
+ })
+ // cast non-^2 scalars to vectors of i8
+ .bitcastIf(
+ [=](const LegalityQuery &Query) -> bool {
+ const LLT EltTy = Query.Types[0];
+ auto NumBits = EltTy.getSizeInBits();
+ return !EltTy.isVector() && !isPowerOf2_32(NumBits);
+ },
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ auto Size = Query.Types[0].getSizeInBits();
+ return std::pair(0, LLT::fixed_vector(Size / 8, I8));
+ })
+ // cast scalar/vector with large bitsize into <? x i32>
+ .bitcastIf(
+ [=](const LegalityQuery &Query) -> bool {
+ const LLT EltTy = Query.Types[0];
+ auto NumBits = EltTy.getScalarSizeInBits();
+ auto IsAtomic128 =
+ EltTy.isScalar() && (NumBits == 128) &&
+ isStrongerThanMonotonic(Query.MMODescrs[0].Ordering);
+ return !IsAtomic128 && (NumBits % 32 == 0) && (NumBits > 64);
+ },
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ const LLT EltTy = Query.Types[0];
+ auto NumBits = EltTy.getSizeInBits();
+ return std::pair(0, LLT::fixed_vector(NumBits / 32, I32));
+ })
+ .bitcastIf(([=](const LegalityQuery &Query) -> bool {
+ auto EltTy = Query.Types[0];
+ auto BitSize = EltTy.getScalarSizeInBits();
+ auto AccSize = EltTy.getSizeInBits();
+ auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ auto SmallVectorWithManyElements =
+ (BitSize < 32) && EltTy.isVector() &&
+ (EltTy.getNumElements() > 4);
+
+ if ((AlignInBits >= AccSize) && !SmallVectorWithManyElements)
+ return false; // all good already
+ if (AlignInBits == BitSize)
+ return false; // handled by scalarizeIf code below
+ if ((AlignInBits < AccSize) && (AccSize % AlignInBits))
+ return false; // weird size/alignment
+ if ((BitSize < 32) && (AccSize % 32 == 0) &&
+ (AlignInBits % 32 == 0))
+ return true; // will bitcast to <? x i32>
+
+ return (AlignInBits < BitSize) ||
+ ((BitSize < 32) && (AlignInBits < AccSize));
+ }),
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ auto EltTy = Query.Types[0];
+ auto BitSize = EltTy.getScalarSizeInBits();
+ auto AccSize = EltTy.getSizeInBits();
+ auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+
+ if ((BitSize < 32) && (AccSize % 32 == 0) &&
+ (AlignInBits % 32 == 0))
+ AlignInBits = 32;
+
+ auto NewEltTy = LLT::integer(AlignInBits);
+ auto NewNumElts = AccSize / AlignInBits;
+ auto NewTy = NewNumElts == 1
+ ? NewEltTy
+ : LLT::fixed_vector(NewNumElts, NewEltTy);
+ return std::pair(0, NewTy);
+ })
+ // bitcast <6 x i32> to <3 x i64> if alignment is sufficient
+ .bitcastIf(([=](const LegalityQuery &Query) -> bool {
+ auto EltTy = Query.Types[0];
+ return EltTy.isVector() &&
+ (EltTy.getScalarSizeInBits() == 32) &&
+ (EltTy.getNumElements() == 6) &&
+ (Query.MMODescrs[0].AlignInBits >= 64);
+ }),
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ auto NewTy = LLT::fixed_vector(3, LLT::integer(64));
+ return std::pair(0, NewTy);
+ })
+ // Increase the number of elements to corresponding vector of i8
+ .customIf([=](const LegalityQuery &Query) {
+ auto EltTy = Query.Types[0];
+ return EltTy.getScalarSizeInBits() == 1;
+ })
+ // expand s32 vectors with 4 < elts < 8 to have 8 elements
+ // Enabled only for shared memory where loads of OOB accesses are
+ // guaranteed to return 0
+ .customIf([=](const LegalityQuery &Query) {
+ auto EltTy = Query.Types[0];
+ auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ return shouldWidenLoad(Query.Opcode, EltTy,
+ Query.Types[1].getAddressSpace(), AlignInBits);
+ })
+ // <4 x i8> align 1 .. needs to be broken down into 4 loads
+ .scalarizeIf(([=](const LegalityQuery &Query) -> bool {
+ auto EltTy = Query.Types[0];
+ auto BitSize = EltTy.getScalarSizeInBits();
+ auto AccSize = EltTy.getSizeInBits();
+ auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ return ((BitSize < 32) && (AlignInBits < AccSize));
+ }),
+ 0)
+ // maximum number of each type that we can load/store
+ .clampMaxNumElements(0, PrivatePtr, 8)
+ .clampMaxNumElements(0, GlobalPtr, 4)
+ .clampMaxNumElements(0, ConstantPtr, 4)
+ .clampMaxNumElements(0, SharedPtr, 8)
+ .clampMaxNumElements(0, GenericPtr, 4)
+ .clampMaxNumElements(0, I8, 4)
+ .clampMaxNumElements(0, I16, 4)
+ .clampMaxNumElements(0, I32, 8)
+ .clampMaxNumElements(0, I64, 4)
+ .clampMaxNumElements(0, I1, 64)
+ // support odd-element vectors, e.g. <7 x i32>
+ // others, e.g. <5 x i16> have been clamped above
+ .fewerElementsIf(
+ [=](const LegalityQuery &Query) -> bool {
+ const LLT EltTy = Query.Types[0];
+ if (!EltTy.isVector())
+ return false;
+ auto NumElements = EltTy.getNumElements();
+ if (!isPowerOf2_32(NumElements))
+ return !((NumElements == 3) &&
+ ((EltTy.getScalarSizeInBits() == 32) ||
+ (EltTy.getScalarSizeInBits() == 64)));
+ return false;
+ },
+ [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
+ const LLT EltTy = Query.Types[0];
+ auto NewNumElts = PowerOf2Ceil(EltTy.getNumElements()) / 2;
+ return std::pair(
+ 0, LLT::fixed_vector(NewNumElts, EltTy.getScalarType()));
+ })
+ // load/store of ptr requires inttoptr/ptrtoint
+ // - has to come after clamping of max elements
+ .customIf([=](const LegalityQuery &Query) {
+ return Query.Types[0].getScalarType().isPointer();
+ })
+ // default
+ .legalIf(typeInSet(1, AllPtrs));
+
+ // lower to a narrow G_LOAD + // G_SEXT/G_ZEXT.
+ getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}).custom();
+
+ getActionDefinitionsBuilder(G_FCANONICALIZE).legalFor(AllFloats);
+
+ getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI, G_FPTOSI_SAT, G_FPTOUI_SAT})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalForCartesianProduct(AllIntegers, AllFloats)
+ .scalarize(0)
+ .minScalar(0, I8);
+
+ getActionDefinitionsBuilder(G_LROUND)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I32, F32}, {I64, F32}, {I32, F64}, {I64, F64}})
+ .clampScalar(0, I32, I64)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_LLROUND)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I64, F32}, {I64, F64}})
+ .clampScalar(0, I64, I64)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ // GlobalIsel built-in lowering doesn't fully support fp16 yet,
+ // so we have to custom lower it for I1 source type
+ .customIf(all(typeIs(1, I1), typeIs(0, BF16)))
+ .customIf(all(typeIs(1, I1), typeIs(0, F16)))
+ .legalForCartesianProduct(AllFloats, AllIntegers)
+ // other types should prefer built-in lowering
+ .lowerIf(typeIs(1, I1))
+ .widenScalarToNextPow2(1)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({I16, I32, I64})
+ .minScalar(0, I16)
+ .scalarize(0)
+ .lower();
+
+ // G_PHI is legal for vector types. However, since most of
+ // the PISA operations are scalar, there will be a need for
+ // (vector) extract op. The assumption here is that extraction
+ // in the loop header will allow for better loop body codegen.
+ getActionDefinitionsBuilder(G_PHI)
+ .legalFor(AllPtrs)
+ .legalFor(AllIntegers)
+ .legalFor(AllFloats)
+ .legalFor({I1})
+ .widenScalarToNextPow2(0, 16)
+ .clampScalar(0, I16, I64)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_BITCAST)
+ // allow bitcasts between pointers and non-pointers (ptr2int/int2ptr)
+ .customIf([=](const LegalityQuery &Query) {
+ auto DstTy = Query.Types[0];
+ auto SrcTy = Query.Types[1];
+ return (DstTy.isPointer() != SrcTy.isPointer());
+ })
+ // In cases where both source and destination operands are vectors,
+ // the standard bitcast lowering expects the number of elements to be
+ // divisible by each other, e.g. <4 x i32> to <8 x i16>; use custom
+ // legalization to handle other cases, e.g. <5 x i32> to <2 x i80>
+ .customIf([=](const LegalityQuery &Query) {
+ auto DstTy = Query.Types[0];
+ auto SrcTy = Query.Types[1];
+ if (!SrcTy.isVector() || !DstTy.isVector())
+ return false;
+ unsigned SrcNumElts = SrcTy.getNumElements();
+ unsigned DstNumElts = DstTy.getNumElements();
+ return (SrcNumElts % DstNumElts != 0) && (DstNumElts % SrcNumElts != 0);
+ })
+ // Handle bitcasts between vectors with the same element count and scalar
+ // size but more than 4 elements whose total bit width is not a power of
+ // 2 (e.g. <5 x f16> to <5 x i16>, 80 bits). Decompose element-wise to
+ // avoid creating illegal G_UNMERGE_VALUES on odd-sized vectors
+ // downstream.
+ .customIf([=](const LegalityQuery &Query) {
+ auto DstTy = Query.Types[0];
+ auto SrcTy = Query.Types[1];
+ if (!SrcTy.isVector() || !DstTy.isVector())
+ return false;
+ auto DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
+ auto SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
+ auto DstEltSize = DstTy.getScalarSizeInBits();
+ auto SrcEltSize = SrcTy.getScalarSizeInBits();
+ if (DstNumElts != SrcNumElts)
+ return false;
+ if (DstEltSize != SrcEltSize)
+ return false;
+ return DstNumElts > 4 && DstEltSize != 32 &&
+ !isPowerOf2_32(DstTy.getSizeInBits());
+ })
+ .legalIf([=](const LegalityQuery &Query) {
+ auto DstTy = Query.Types[0];
+ auto SrcTy = Query.Types[1];
+ auto DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
+ auto SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
+ auto DstEltSize = DstTy.getScalarSizeInBits();
+ auto SrcEltSize = SrcTy.getScalarSizeInBits();
+ auto CastSize = DstTy.getSizeInBits();
+ if (DstEltSize == 32 && SrcEltSize == 32)
+ // vectors of 32bit integer <=> floats
+ return DstNumElts <= 8 || DstNumElts == 16 || DstNumElts == 32 ||
+ DstNumElts == 64;
+ if (DstNumElts > 4 || SrcNumElts > 4)
+ // can not use swizzle for copy
+ return false;
+ if (DstEltSize < 8 || SrcEltSize < 8)
+ // sub-byte types are not natively supported
+ return false;
+ if ((DstTy.isVector() && DstEltSize > 64) ||
+ (SrcTy.isVector() && SrcEltSize > 64))
+ // No vector register class exists for elements wider than 64 bits
+ // (e.g. <2 x i128>); such casts must be lowered element-wise rather
+ // than marked legal, otherwise instruction selection has no vector
+ // register class to constrain to and asserts.
+ return false;
+ if (CastSize > 128)
+ // no registers of such size
+ return DstNumElts != 1 && SrcNumElts != 1;
+ if (!llvm::isPowerOf2_32(CastSize))
+ // non-power-of-2 bitcasts can only be between 3-element vectors
+ return DstNumElts == 3 && SrcNumElts == 3;
+ return true;
+ })
+ .lower();
+
+ for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
+ unsigned SrcTyIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
+
+ getActionDefinitionsBuilder(Op)
+ // extend vectors of i1 to have power of two elements
+ .moreElementsIf(
+ ([=](const LegalityQuery &Query) {
+ auto DstTy = Query.Types[SrcTyIdx];
+ auto BitSize = DstTy.getSizeInBits();
+ return DstTy.isVector() && (DstTy.getScalarSizeInBits() == 1) &&
+ ((BitSize < 8) || !isPowerOf2_32(BitSize));
+ }),
+ [=](const LegalityQuery &Query) {
+ auto DstTy = Query.Types[SrcTyIdx];
+ unsigned NumElts = PowerOf2Ceil(DstTy.getNumElements());
+ NumElts = std::max(8u, NumElts);
+ return std::pair(
+ SrcTyIdx, LLT::fixed_vector(NumElts, DstTy.getScalarType()));
+ })
+ // <? x i1>
+ .customIf([=](const LegalityQuery &Query) {
+ auto EltSize = Query.Types[SrcTyIdx].getScalarSizeInBits();
+ return (EltSize == 1);
+ })
+ // increase to a multiple of elements, e.g. <5 x i16> => <8 x i16>
+ .moreElementsIf(
+ [=](const LegalityQuery &Query) {
+ auto SrcTy = Query.Types[SrcTyIdx];
+ unsigned NumElts = SrcTy.getNumElements();
+ if (SrcTy.getScalarSizeInBits() != 32)
+ return (NumElts > 4) && (NumElts % 4);
+ return (NumElts > 8) && (NumElts != 16) && (NumElts % 32);
+ },
+ [=](const LegalityQuery &Query) {
+ auto SrcTy = Query.Types[SrcTyIdx];
+ auto ScalarTy = SrcTy.getScalarType();
+ auto NumElts = PowerOf2Ceil(SrcTy.getNumElements());
+ return std::pair(SrcTyIdx, LLT::fixed_vector(NumElts, ScalarTy));
+ })
+ // cast non-s32 elements to s32 vector, e.g.
+ // <N x s8> => <N/4 x s32>, iff the index is non-constant
+ .customIf([=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[SrcTyIdx];
+ const auto EltSize = Ty.getScalarSizeInBits();
+ const auto NumElts = Ty.getNumElements();
+ // Only needed for non-s32 elements
+ if (EltSize != 8 && EltSize != 16 && EltSize != 64)
+ return false;
+ // The vector must fit into <64 x s32>, otherwise cannot bitcast
+ if (NumElts * EltSize / 32 > 64)
+ return false;
+ return true;
+ })
+ // reduce to a multiple of elements, e.g. <8 x i16> => <4 x i16>
+ // (x2)
+ // - each multiple of elements is supported natively
+ // - operation will use 'insert/extract' or swizzle
+ .fewerElementsIf(
+ [=](const LegalityQuery &Query) {
+ auto SrcTy = Query.Types[SrcTyIdx];
+ auto MaxElts = SrcTy.getScalarSizeInBits() == 32 ? 64 : 4;
+ return SrcTy.getNumElements() > MaxElts;
+ },
+ [=](const LegalityQuery &Query) {
+ auto SrcTy = Query.Types[SrcTyIdx];
+ auto ScalarTy = SrcTy.getScalarType();
+ return (SrcTy.getScalarSizeInBits() == 32)
+ ? std::pair(SrcTyIdx, LLT::fixed_vector(64, ScalarTy))
+ : std::pair(SrcTyIdx, LLT::fixed_vector(4, ScalarTy));
+ })
+ .lowerIf([=](const LegalityQuery &Query) {
+ return Query.Types[SrcTyIdx].getScalarSizeInBits() != 32;
+ })
+ .alwaysLegal();
+ }
+
+ getActionDefinitionsBuilder(G_INSERT_SUBVECTOR)
+ .customIf([=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[0];
+ return (Ty.getScalarSizeInBits() == 32 ||
+ Ty.getScalarSizeInBits() == 64);
+ })
+ .unsupported(); // no lower() implementation
+
+ getActionDefinitionsBuilder(G_EXTRACT_SUBVECTOR)
+ // A 2-element sub-vector extracted from a wider same-element vector is a
+ // nameable composite sub-register slice (.xy / .zw); ISel lowers it to a
+ // sub-register COPY. Mark it legal so the post-legalizer combiner may
+ // produce it (see build_vector_from_unmerge_lanes).
+ .legalIf([=](const LegalityQuery &Query) {
+ const LLT Dst = Query.Types[0];
+ const LLT Src = Query.Types[1];
+ return Dst.isVector() && Src.isVector() && Dst.getNumElements() == 2 &&
+ Src.getNumElements() > 2 && Src.getNumElements() <= 4 &&
+ Src.getElementType() == Dst.getElementType() &&
+ (Dst.getScalarSizeInBits() == 8 ||
+ Dst.getScalarSizeInBits() == 16);
+ })
+ .customIf([=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[0];
+ return (Ty.getScalarSizeInBits() == 32 ||
+ Ty.getScalarSizeInBits() == 64);
+ })
+ .unsupported(); // no lower() implementation
+
+ getActionDefinitionsBuilder({G_INSERT, G_EXTRACT}).lower();
+
+ getActionDefinitionsBuilder(G_CONCAT_VECTORS)
+ .legalIf([=](const LegalityQuery &Query) {
+ // vector(big) <=> vector(lit)
+ const LLT BigTy = Query.Types[0];
+ return BigTy.isVector() && (BigTy.getNumElements() <= 4) &&
+ (BigTy.getSizeInBits() <= 256); // v4s64
+ })
+ .customIf([=](const LegalityQuery &Query) {
+ // return true if we want to use 'insert', instead of swizzle
+ auto LitTy = Query.Types[1];
+ auto BigTy = Query.Types[0];
+ auto EltOk = BigTy.getScalarSizeInBits() == 32;
+ auto VecOk = LitTy.isVector() && BigTy.isVector();
+ return EltOk && VecOk &&
+ ((LitTy.getNumElements() > 4) || (BigTy.getNumElements() > 4));
+ })
+ .clampMaxNumElements(1, I8, 4)
+ .clampMaxNumElements(1, I16, 4)
+ // .clampMaxNumElements(1, I32, 32) handled by customIf.
+ .clampMaxNumElements(1, I64, 4);
+
+ getActionDefinitionsBuilder(
+ {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
+ G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_OR, G_VECREDUCE_AND,
+ G_VECREDUCE_XOR, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
+ G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
+ // fewerElementsVectorReductions does not handle (SrcElts % DstElts != 0)
+ .moreElementsIf(([=](const LegalityQuery &Query) {
+ auto NumElts = Query.Types[1].getNumElements();
+ return NumElts > 4 && NumElts % 4 != 0;
+ }),
+ [=](const LegalityQuery &Query) {
+ auto SrcTy = Query.Types[1];
+ auto NewNumElts =
+ llvm::PowerOf2Ceil(SrcTy.getNumElements());
+ auto NewSrcTy = LLT::fixed_vector(
+ NewNumElts, SrcTy.getScalarType());
+ return std::pair(1, NewSrcTy);
+ })
+ .fewerElementsIf(vectorElementCountIsGreaterThan(1, 4),
+ changeElementCountTo(1, EC4))
+ .scalarize(1)
+ .lower();
+
+ for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
+ unsigned BigTyIdx = Op == G_UNMERGE_VALUES ? 1 : 0;
+ unsigned LitTyIdx = Op == G_UNMERGE_VALUES ? 0 : 1;
+
+ auto &Builder = getActionDefinitionsBuilder(Op);
+ Builder.customIf([=](const LegalityQuery &Query) {
+ // return true if we want to use 'extract', instead of swizzle
+ auto BigTy = Query.Types[BigTyIdx];
+ return BigTy.isVector() && (BigTy.getScalarSizeInBits() == 32) &&
+ (BigTy.getNumElements() > 4);
+ });
+
+ Builder
+ .legalIf([=](const LegalityQuery &Query) {
+ // vector(big) <=> scalar/vector(lit)
+ auto BigTy = Query.Types[BigTyIdx];
+ auto LitTy = Query.Types[LitTyIdx];
+ auto LitTyValid = LitTy.isScalar() ? BigTy.getScalarType() == LitTy
+ : LitTy.getScalarSizeInBits() >= 8;
+ // No register class exists for vectors with elements wider than
+ // 64 bits, so <N x i128> and friends must not be marked legal here
+ // (they would otherwise crash instruction selection when looking up
+ // a vector register class). Cap the vector element size at 64.
+ return LitTyValid && BigTy.isVector() &&
+ (BigTy.getScalarSizeInBits() <= 64) &&
+ (BigTy.getNumElements() <= 4) &&
+ (BigTy.getSizeInBits() <= 256); // v4s64
+ })
+ .widenScalarIf(
+ [=](const LegalityQuery &Query) {
+ auto BigTy = Query.Types[BigTyIdx];
+ auto BigTySize = BigTy.getSizeInBits();
+ return BigTy.isScalar() && BigTySize > 64 &&
+ !isPowerOf2_32(BigTySize);
+ },
+ [=](const LegalityQuery &Query) {
+ auto BigTy = Query.Types[BigTyIdx];
+ unsigned NewSizeInBits =
+ 1 << Log2_32_Ceil(BigTy.getSizeInBits() + 1);
+ return std::pair(BigTyIdx, LLT::integer(NewSizeInBits));
+ })
+ .lowerIf([=](const LegalityQuery &Query) {
+ // lower to shift/mask if conversion would
+ // result in a vector with >4 elements
+ auto BigTy = Query.Types[BigTyIdx];
+ auto LitTy = Query.Types[LitTyIdx];
+ auto NumElts = BigTy.getSizeInBits() / LitTy.getScalarSizeInBits();
+ return BigTy.isScalar() && (NumElts > 4);
+ })
+ .lowerIf(all(vectorElementCountIsGreaterThan(LitTyIdx, 4),
+ vectorElementCountIsGreaterThan(BigTyIdx, 4)))
+ .fewerElementsIf(vectorElementCountIsGreaterThan(BigTyIdx, 4),
+ changeElementCountTo(BigTyIdx, EC4))
+ .minScalarOrEltIf(scalarNarrowerThan(LitTyIdx, 16), LitTyIdx, I16)
+ .legalIf([=](const LegalityQuery &Query) {
+ return (Query.Types[BigTyIdx].isScalar() ||
+ Query.Types[BigTyIdx].isPointer()) &&
+ (Query.Types[LitTyIdx].isScalar() ||
+ Query.Types[LitTyIdx].isPointer());
+ });
+ }
+
+ getActionDefinitionsBuilder(G_BUILD_VECTOR).alwaysLegal();
+
+ getActionDefinitionsBuilder(G_IMPLICIT_DEF)
+ .legalIf([=](const LegalityQuery &Query) {
+ return isLegalType(Query.Types[0]);
+ })
+ .legalFor({I1})
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, I16, I64)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_FREEZE)
+ .legalFor(AllIntegers)
+ .legalFor(AllFloats)
+ .legalFor(AllPtrs)
+ .widenScalarToNextPow2(0)
+ .clampScalar(0, I32, I64)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_INTTOPTR)
+ // List the common cases
+ .legalForCartesianProduct(AddrSpaces64, {I64})
+ .legalForCartesianProduct(AddrSpaces32, {I32})
+ .scalarize(0)
+ // Accept any address space as long as the size matches
+ .legalIf(sameSize(0, 1))
+ .widenScalarIf(smallerThan(1, 0),
+ [](const LegalityQuery &Query) {
+ return std::pair(
+ 1, LLT::integer(Query.Types[0].getSizeInBits()));
+ })
+ .narrowScalarIf(largerThan(1, 0), [](const LegalityQuery &Query) {
+ return std::pair(1, LLT::integer(Query.Types[0].getSizeInBits()));
+ });
+
+ getActionDefinitionsBuilder(G_PTRTOINT)
+ // List the common cases
+ .legalForCartesianProduct(AddrSpaces64, {I64})
+ .legalForCartesianProduct(AddrSpaces32, {I32})
+ .scalarize(0)
+ // Accept any address space as long as the size matches
+ .legalIf(sameSize(0, 1))
+ .widenScalarIf(smallerThan(0, 1),
+ [](const LegalityQuery &Query) {
+ return std::pair(
+ 0, LLT::integer(Query.Types[1].getSizeInBits()));
+ })
+ .narrowScalarIf(largerThan(0, 1), [](const LegalityQuery &Query) {
+ return std::pair(0, LLT::integer(Query.Types[1].getSizeInBits()));
+ });
+
+ getActionDefinitionsBuilder(G_ICMP)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalIf(all(
+ typeIs(0, I1),
+ LegalityPredicates::any(isPointer(1), typeInSet(1, {I16, I32, I64}))))
+ .widenScalarToNextPow2(1)
+ .scalarize(0)
+ .clampScalar(1, I16, I64);
+
+ getActionDefinitionsBuilder(G_FCMP)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .custom();
+
+ getActionDefinitionsBuilder({G_SCMP, G_UCMP})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .lower();
+
+ getActionDefinitionsBuilder(G_SELECT)
+ .legalIf(all(
+ LegalityPredicates::any(isPointer(0), typeInSet(0, {I16, I32, I64})),
+ typeIs(1, I1)))
+ .legalIf(
+ all(LegalityPredicates::any(isPointer(0), typeInSet(0, AllFloats)),
+ typeIs(1, I1)))
+ .scalarize(0)
+ .clampScalar(0, I16, I64)
+ .widenScalarToNextPow2(0);
+
+ getActionDefinitionsBuilder(
+ {G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND, G_ATOMICRMW_MAX,
+ G_ATOMICRMW_MIN, G_ATOMICRMW_SUB, G_ATOMICRMW_XOR, G_ATOMICRMW_UMAX,
+ G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
+ // PISA supports up to acq_rel, anything that is not included in that
+ // needs custom legalization.
+ // In other words if acq_rel provides the same or stronger guarantees
+ // than the requested ordering, then the operation is legal, otherwise
+ // it needs custom legalization.
+ // Note that this is not equivalent to isStrongerThan(Ordering, AcqRel)
+ // because memory orderings do not form a total order.
+ // For example, Acquire is neither stronger nor weaker than Release.
+ .customIf([=](const LegalityQuery &Query) {
+ return !isAtLeastOrStrongerThan(AtomicOrdering::AcquireRelease,
+ Query.MMODescrs[0].Ordering);
+ })
+ .legalForCartesianProduct({I16, I32, I64},
+ {GlobalPtr, SharedPtr, GenericPtr});
+
+ getActionDefinitionsBuilder(
+ {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
+ .customIf([=](const LegalityQuery &Query) {
+ return !isAtLeastOrStrongerThan(AtomicOrdering::AcquireRelease,
+ Query.MMODescrs[0].Ordering);
+ })
+ .legalForCartesianProduct(AllFloats, {GlobalPtr, SharedPtr, GenericPtr});
+
+ getActionDefinitionsBuilder(G_ATOMICRMW_XCHG)
+ .customIf([=](const LegalityQuery &Query) {
+ return !isAtLeastOrStrongerThan(AtomicOrdering::AcquireRelease,
+ Query.MMODescrs[0].Ordering);
+ })
+ .legalForCartesianProduct({I16, I32, I64, I128},
+ {GlobalPtr, SharedPtr, GenericPtr})
+ .customIf([=](const LegalityQuery &Query) {
+ return Query.Types[0].getScalarType().isPointer();
+ });
+
+ getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();
+ // For cmpxchg in case of failure the strongest ordering we can do
+ // directly is 'acquire', anything stronger needs custom legalization.
+ getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
+ .customIf([=](const LegalityQuery &Query) {
+ return !isAtLeastOrStrongerThan(AtomicOrdering::AcquireRelease,
+ Query.MMODescrs[0].Ordering) ||
+ !isAtLeastOrStrongerThan(AtomicOrdering::Acquire,
+ Query.MMODescrs[0].FailureOrdering);
+ })
+ .alwaysLegal();
+
+ getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .minScalar(0, I16)
+ .scalarize(0)
+ .lower();
+
+ getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .minScalar(0, I16)
+ .legalFor({I16, I32, I64})
+ .scalarize(0)
+ .lower();
+
+ getActionDefinitionsBuilder({G_UADDO, G_USUBO, G_UADDE, G_USUBE})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .clampScalar(0, I32, I32)
+ .legalFor({{I32, I1}});
+
+ getActionDefinitionsBuilder({G_SADDO, G_SSUBO, G_SADDE, G_SSUBE})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .lower();
+
+ // pointer-handling.
+ getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr, SharedPtr});
+
+ // control-flow. In some cases (e.g. constants) i1 may be promoted to i32.
+ getActionDefinitionsBuilder(G_BR).alwaysLegal();
+ getActionDefinitionsBuilder(G_BRCOND).legalFor({I1, I32});
+ getActionDefinitionsBuilder(G_FENCE).alwaysLegal();
+ getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
+
+ getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FTANH, G_FEXP2, G_FLOG2})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({BF16, F16, F32})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder({G_FEXP, G_FEXP10, G_FLOG, G_FLOG10, G_FPOW})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .customFor({BF16, F16, F32})
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_FPOWI).lower();
+
+ getActionDefinitionsBuilder(G_FCOPYSIGN)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .lower();
+
+ getActionDefinitionsBuilder({G_SMULH, G_UMULH})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .customFor({I64})
+ .lower();
+
+ getActionDefinitionsBuilder(G_BSWAP)
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .scalarize(0)
+ .customIf([=](const LegalityQuery &Query) {
+ const LLT Ty = Query.Types[0];
+ unsigned BitSize = Ty.getSizeInBits();
+ return Ty.isScalar() && (BitSize % 16 == 0);
+ })
+ .unsupported();
+
+ getActionDefinitionsBuilder(G_CONSTANT_FOLD_BARRIER)
+ .legalFor({I8, I16, I32, I64});
+
+ getActionDefinitionsBuilder({G_SBFX, G_UBFX})
+ .fewerElementsIf(vectorElementCountIsGreaterThan(0, 4),
+ changeElementCountTo(0, EC4))
+ .legalFor({{I32, I32}})
+ .clampScalar(1, I32, I32)
+ .clampScalar(0, I32, I32)
+ .scalarize(0);
+
+ getActionDefinitionsBuilder(G_DYN_STACKALLOC).legalFor({{PrivatePtr, I32}});
+
+ getActionDefinitionsBuilder({G_READSTEADYCOUNTER, G_READCYCLECOUNTER})
+ .legalFor({I64});
+
+ verify(*ST.getInstrInfo());
+}
+
+// scalarize an intrinsic instruction with vector arguments
+static SmallVector<MachineInstr *> scalarizeIntrinsic(MachineInstr &MI) {
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ MachineIRBuilder B(MI);
+ auto &MRI = *B.getMRI();
+
+ SmallVector<MachineInstr *> NewMIs;
+ auto DstTy = MRI.getType(MI.getOperand(0).getReg());
+ if (!DstTy.isVector()) {
+ NewMIs.push_back(&MI);
+ return NewMIs;
+ }
+
+ SmallVector<Register, 4> VecRegs;
+ for (unsigned I = 0; I < DstTy.getNumElements(); I++) {
+ SmallVector<MachineOperand, 4> Opnds;
+ for (unsigned J = 2; J < MI.getNumOperands(); J++) { // dst, iid
+ auto Opnd = MI.getOperand(J);
+ if (Opnd.isReg()) {
+ auto ArgTy = MRI.getType(Opnd.getReg());
+ if (ArgTy.isVector()) {
+ ArgTy = ArgTy.getScalarType();
+ auto ArgReg = MRI.createGenericVirtualRegister(ArgTy);
+ B.buildExtractVectorElementConstant(ArgReg, Opnd, I);
+ Opnds.push_back(MachineOperand::CreateReg(ArgReg, false));
+ } else { // use register operand as-is
+ Opnds.push_back(Opnd);
+ }
+ } else { // use immediate operand as-is (e.g. rounding mode)
+ Opnds.push_back(Opnd);
+ }
+ }
+ auto DstReg = MRI.createGenericVirtualRegister(DstTy.getScalarType());
+ auto Res = B.buildIntrinsic(IntrinsicID, DstReg);
+ NewMIs.push_back(Res);
+ Res.setMIFlags(MI.getFlags());
+ for (auto It = Opnds.begin(), Ite = Opnds.end(); It != Ite; ++It)
+ Res.add(*It);
+ VecRegs.push_back(DstReg);
+ }
+ B.buildBuildVector(MI.getOperand(0), VecRegs);
+ MI.eraseFromParent();
+ return NewMIs;
+}
+
+// flog(x) = flog2(x) * ln(2)
+static bool legalizeGFlog(MachineInstr &MI, MachineIRBuilder &B,
+ double Log2BaseInverted) {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ LLT Ty = B.getMRI()->getType(Dst);
+ unsigned Flags = MI.getFlags();
+
+ auto &Semantics = getFltSemanticForLLT(Ty.getScalarType());
+ APFloat APFLog2BaseInverted(Log2BaseInverted);
+ bool LosesInfo; // ignored
+ APFLog2BaseInverted.convert(Semantics, APFloat::rmNearestTiesToEven,
+ &LosesInfo);
+
+ auto Log2Operand = B.buildFLog2(Ty, Src, Flags);
+ auto Log2BaseInvertedOperand = B.buildFConstant(Ty, APFLog2BaseInverted);
+
+ B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
+ MI.eraseFromParent();
+ return true;
+}
+
+// fexp(x) = fexp2(x * log2(e))
+static bool legalizeGFexp(MachineInstr &MI, MachineIRBuilder &B,
+ double Multiplicand) {
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(1).getReg();
+ unsigned Flags = MI.getFlags();
+ LLT Ty = B.getMRI()->getType(Dst);
+
+ auto &Semantics = getFltSemanticForLLT(Ty.getScalarType());
+ APFloat APFMultiplicand(Multiplicand);
+ bool LosesInfo; // ignored
+ APFMultiplicand.convert(Semantics, APFloat::rmNearestTiesToEven, &LosesInfo);
+
+ auto K = B.buildFConstant(Ty, APFMultiplicand);
+ auto Mul = B.buildFMul(Ty, Src, K, Flags);
+ B.buildFExp2(Dst, Mul, Flags);
+ MI.eraseFromParent();
+ return true;
+}
+
+// GlobalISel doesn't currently have builtin support to legalize based on
+// condition code like the SelectionDAG path does. We can move to that approach
+// if and when it is available. For now, we custom legalize it based upon the
+// approach in TargetLowering::LegalizeSetCCCondCode().
+static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
+ auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
+ Register Dst = MI.getOperand(0).getReg();
+ Register Op0 = MI.getOperand(2).getReg();
+ Register Op1 = MI.getOperand(3).getReg();
+ unsigned Flags = MI.getFlags();
+ switch (Pred) {
+ case CmpInst::FCMP_UNE:
+ case CmpInst::FCMP_OEQ:
+ case CmpInst::FCMP_OGT:
+ case CmpInst::FCMP_OGE:
+ case CmpInst::FCMP_OLT:
+ case CmpInst::FCMP_OLE:
+ // already legal
+ break;
+ case CmpInst::FCMP_ONE:
+ case CmpInst::FCMP_UEQ: {
+ // Without the explicit G_SEXT added here, the legalizer will typically
+ // G_ANYEXT the G_FCMP compare result to i16. Given that a .reg destination
+ // for fcmp is only available for 32-bit, we explicitly extend it here
+ // so we can fold the resulting select into the fcmp.
+ auto LHS =
+ B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OGT, I1, Op0, Op1, Flags));
+ auto RHS =
+ B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OLT, I1, Op0, Op1, Flags));
+ auto Result = B.buildOr(I32, LHS, RHS);
+ if (Pred == CmpInst::FCMP_UEQ)
+ Result = B.buildNot(I32, Result);
+ B.buildICmp(CmpInst::ICMP_EQ, Dst, Result, B.buildConstant(I32, -1));
+ MI.eraseFromParent();
+ break;
+ }
+ case CmpInst::FCMP_ORD: {
+ auto LHS =
+ B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OEQ, I1, Op0, Op0, Flags));
+ auto RHS =
+ B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OEQ, I1, Op1, Op1, Flags));
+ auto Result = B.buildAnd(I32, LHS, RHS);
+ B.buildICmp(CmpInst::ICMP_EQ, Dst, Result, B.buildConstant(I32, -1));
+ MI.eraseFromParent();
+ break;
+ }
+ case CmpInst::FCMP_UNO: {
+ // When checking if an op is NaN in OpenCL, the builtin generates an
+ // fcmp.uno with a non-NaN constant (usually zero). In that case, we don't
+ // need to generate two fcmps because only the non-const parameter is
+ // relevant to this comparison
+ auto Op0Cst = getFConstantVRegValWithLookThrough(Op0, *B.getMRI());
+ bool Op0IsOrdConstant = Op0Cst && !Op0Cst.value().Value.isNaN();
+
+ auto Op1Cst = getFConstantVRegValWithLookThrough(Op1, *B.getMRI());
+ bool Op1IsOrdConstant = Op1Cst && !Op1Cst.value().Value.isNaN();
+
+ if (Op0IsOrdConstant || Op1IsOrdConstant) {
+ auto Reg = Op1IsOrdConstant ? Op0 : Op1;
+ B.buildFCmp(CmpInst::FCMP_UNE, Dst, Reg, Reg, Flags);
+ } else {
+ // If the operands are both non-constant, we need to split this into two
+ // fcmps to ensure it returns false if they are unequal
+ auto LHS =
+ B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_UNE, I1, Op0, Op0, Flags));
+ auto RHS =
+ B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_UNE, I1, Op1, Op1, Flags));
+ auto Result = B.buildOr(I32, LHS, RHS);
+ B.buildICmp(CmpInst::ICMP_EQ, Dst, Result, B.buildConstant(I32, -1));
+ }
+ MI.eraseFromParent();
+ break;
+ }
+ case CmpInst::FCMP_UGT:
+ case CmpInst::FCMP_UGE:
+ case CmpInst::FCMP_ULT:
+ case CmpInst::FCMP_ULE: {
+ auto Cmp = B.buildSExt(I32, B.buildFCmp(FCmpInst::getInversePredicate(Pred),
+ I1, Op0, Op1, Flags));
+ auto Not = B.buildNot(I32, Cmp);
+ B.buildICmp(CmpInst::ICMP_EQ, Dst, Not, B.buildConstant(I32, -1));
+ MI.eraseFromParent();
+ break;
+ }
+ default:
+ llvm_unreachable("unknown predicate?");
+ }
+ return true;
+}
+
+static bool legalizeGTrunc(MachineInstr &MI, MachineIRBuilder &B) {
+ [[maybe_unused]] auto &MRI = *B.getMRI();
+ auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ if (DstTy.getSizeInBits() == 1) {
+ // truncate ??? to i1
+ // Since PISA does not support truncs to i1 (i8 is the minimum), we must
+ // turn it into an i1 by using an icmp instruction.
+ auto Zero = B.buildConstant(SrcTy, 0);
+ auto One = B.buildConstant(SrcTy, 1);
+ auto And = B.buildAnd(SrcTy, Src, One);
+ B.buildICmp(CmpInst::ICMP_NE, Dst, And, Zero);
+ } else {
+ // truncate i128 to ???
+ assert(SrcTy.getSizeInBits() == 128);
+ auto Unmerge = B.buildUnmerge(I64, Src);
+ if (DstTy.getSizeInBits() == 64)
+ B.buildCopy(Dst, Unmerge.getReg(0));
+ else
+ B.buildTrunc(Dst, Unmerge.getReg(0));
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGExt(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+
+ if (MRI.getType(Src).getSizeInBits() == 1) {
+ // i8 = G_*EXT i1
+ auto Zero = B.buildConstant(DstTy, 0);
+ int64_t ExtendedVal = (MI.getOpcode() == TargetOpcode::G_SEXT) ||
+ (MI.getOpcode() == TargetOpcode::G_ANYEXT)
+ ? -1
+ : 1;
+ auto One = B.buildConstant(DstTy, ExtendedVal);
+ B.buildSelect(Dst, Src, One, Zero);
+ } else {
+ // any G_*EXT where source and destination are byte size
+ auto DstSize = DstTy.getScalarSizeInBits();
+ auto SrcSize = SrcTy.getScalarSizeInBits();
+ assert((DstSize % 8 == 0) && "destination size is not byte size");
+ assert((SrcSize % 8 == 0) && "source size is not byte size");
+ auto EltSize =
+ ((DstSize % 32 == 0) && (SrcSize % 32 == 0))
+ ? 32
+ : (((DstSize % 16 == 0) && (SrcSize % 16 == 0)) ? 16 : 8);
+ unsigned NumDstElts = DstSize / EltSize;
+ unsigned NumSrcElts = SrcSize / EltSize;
+ LLT EltTy = LLT::integer(EltSize);
+ LLT VecDstTy = LLT::fixed_vector(NumDstElts, EltTy);
+
+ auto VecZero = MRI.createGenericVirtualRegister(VecDstTy);
+ SmallVector<APInt> Zeros(NumDstElts, APInt(EltSize, 0));
+ B.buildBuildVectorConstant(VecZero, Zeros);
+
+ Register VecSrc;
+ if (NumSrcElts == 1) {
+ SmallVector<Register> Ops(NumDstElts, Src);
+ VecSrc = MRI.createGenericVirtualRegister(VecDstTy);
+ B.buildBuildVector(VecSrc, Ops); // Splat scalar into vector
+ } else {
+ LLT VecSrcTy = LLT::fixed_vector(NumSrcElts, EltTy);
+ VecSrc = MRI.createGenericVirtualRegister(VecSrcTy);
+ B.buildBitcast(VecSrc, Src);
+ }
+
+ SmallVector<int> Mask;
+ for (unsigned I = 0; I < NumDstElts; I++) {
+ Mask.push_back((I < NumSrcElts) ? I
+ : MRI.getType(VecSrc).getNumElements());
+ }
+
+ auto VecDst = MRI.createGenericVirtualRegister(VecDstTy);
+ B.buildShuffleVector(VecDst, VecSrc, VecZero, Mask);
+
+ if (MI.getOpcode() == TargetOpcode::G_SEXT) {
+ auto CastReg = MRI.createGenericVirtualRegister(DstTy);
+ auto ShiftReg = MRI.createGenericVirtualRegister(DstTy);
+ auto ShiftAmt = B.buildConstant(I32, DstSize - SrcSize);
+ B.buildBitcast(CastReg, VecDst);
+ B.buildShl(ShiftReg, CastReg, ShiftAmt);
+ B.buildAShr(Dst, ShiftReg, ShiftAmt);
+ } else {
+ B.buildBitcast(Dst, VecDst);
+ }
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGItofp(MachineInstr &MI, MachineIRBuilder &B) {
+ auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ assert(SrcTy.isScalar() && SrcTy.getSizeInBits() == 1 &&
+ "Unexpected source type");
+ assert(DstTy.isScalar() && DstTy.getSizeInBits() == 16 &&
+ "Unexpected destination type");
+
+ unsigned Opc = MI.getOpcode();
+ assert((Opc == TargetOpcode::G_SITOFP || Opc == TargetOpcode::G_UITOFP) &&
+ "Unexpected instruction opcode");
+
+ const fltSemantics &Semantics =
+ DstTy == LLT::bfloat16() ? APFloat::BFloat() : APFloat::IEEEhalf();
+
+ auto TrueVal =
+ APFloat::getOne(Semantics, /*Negative=*/Opc == TargetOpcode::G_SITOFP);
+ auto FalseVal = APFloat::getZero(Semantics);
+
+ auto True = B.buildFConstant(DstTy, TrueVal);
+ auto False = B.buildFConstant(DstTy, FalseVal);
+ B.buildSelect(Dst, Src, True, False);
+ MI.eraseFromParent();
+ return true;
+}
+
+static void updateRegInDebugValue(Register OriginalVal, Register NewVal,
+ MachineRegisterInfo &MRI) {
+ llvm::SmallVector<MachineOperand *, 5> Opnds;
+ for (auto &Instr : MRI.use_instructions(OriginalVal)) {
+ if (!Instr.isDebugValue())
+ continue;
+ for (auto &Opnd : Instr.operands()) {
+ if (Opnd.isReg() && Opnd.getReg() == OriginalVal)
+ Opnds.push_back(&Opnd);
+ }
+ }
+ for (auto *Opnd : Opnds)
+ Opnd->setReg(NewVal);
+ return;
+}
+
+static bool legalizeGExtload(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &LoadMI = cast<GExtLoad>(MI);
+ Register DstReg = LoadMI.getDstReg();
+ Register PtrReg = LoadMI.getPointerReg();
+ LLT MemTy = LoadMI.getMMO().getMemoryType();
+
+ // legalizer will create scalar type here, e.g. s16
+ LLT EltTy = LLT::integer(MemTy.getScalarSizeInBits());
+ MemTy = MemTy.isVector() ? LLT::fixed_vector(MemTy.getNumElements(), EltTy)
+ : EltTy;
+
+ // Narrow load + extension: G_{S,Z}EXTLOAD(DstTy, ptr) ->
+ // %narrow = G_LOAD MemTy, ptr
+ // DstReg = G_{S,Z}EXT DstTy, %narrow
+ auto NarrowLoad = B.buildLoad(MemTy, PtrReg, LoadMI.getMMO());
+ if (isa<GSExtLoad>(MI))
+ B.buildSExt(DstReg, NarrowLoad);
+ else
+ B.buildZExt(DstReg, NarrowLoad);
+ LoadMI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
+ LegalizerHelper &Helper) {
+ auto &MRI = *B.getMRI();
+ GISelChangeObserver &Observer = Helper.Observer;
+ auto &ValMO = MI.getOperand(0);
+ Register Val = ValMO.getReg();
+ MachineMemOperand &MMO = **MI.memoperands_begin();
+ unsigned AddressSpace = MMO.getAddrSpace();
+ LLT CurTy = MRI.getType(Val);
+ auto CurTySize = CurTy.getSizeInBits();
+
+ if (!CurTy.isVector() && ((CurTySize % 8) != 0)) {
+ // Widen sub-byte scalar load/store to multiple of 8 bits.
+ auto NewSize = (CurTySize + 7) & ~7;
+ auto NewTy = LLT::integer(NewSize);
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ // For loads: widen the load and truncate result.
+ Helper.widenScalar(MI, 0, NewTy);
+ MI.memoperands()[0]->setType(NewTy);
+ } else if (CurTySize > 1) {
+ // For stores of i2+: fold through G_TRUNC/G_BITCAST chains to find a
+ // byte-sized source, avoiding G_ANYEXT from sub-byte types (which can't
+ // be legalized for sources wider than i1).
+ Register SrcReg = Val;
+ MachineInstr *Def = MRI.getVRegDef(SrcReg);
+ while (Def &&
+ (Def->getOpcode() == TargetOpcode::G_TRUNC ||
+ Def->getOpcode() == TargetOpcode::G_BITCAST) &&
+ MRI.getType(Def->getOperand(1).getReg()).getSizeInBits() <
+ NewSize) {
+ SrcReg = Def->getOperand(1).getReg();
+ Def = MRI.getVRegDef(SrcReg);
+ }
+ // If the walk stops on a cast, it is always a G_TRUNC: a G_BITCAST
+ // preserves its operand's (sub-byte) width, so it can never be the def
+ // with a source >= NewSize that ends the walk. A non-cast terminating
+ // def (e.g. G_CONSTANT) is handled by the G_INSERT widening below.
+ if (Def && Def->getOpcode() == TargetOpcode::G_TRUNC) {
+ // Found wider source via the G_TRUNC/G_BITCAST chain
+ Register WiderReg = Def->getOperand(1).getReg();
+ LLT WiderTy = MRI.getType(WiderReg);
+ Register StoreReg;
+ if (WiderTy.getSizeInBits() == NewSize)
+ StoreReg = WiderReg;
+ else
+ StoreReg = B.buildTrunc(NewTy, WiderReg).getReg(0);
+ Observer.changingInstr(MI);
+ ValMO.setReg(StoreReg);
+ MMO.setType(NewTy);
+ Observer.changedInstr(MI);
+ } else {
+ // No wider source found via G_TRUNC/G_BITCAST chain.
+ // Use G_INSERT into undef to widen without G_ANYEXT (which can't be
+ // legalized for non-byte-aligned sub-byte sources > i1).
+ Register UndefReg = B.buildUndef(NewTy).getReg(0);
+ Register WideReg = B.buildInsert(NewTy, UndefReg, Val, 0).getReg(0);
+ Observer.changingInstr(MI);
+ ValMO.setReg(WideReg);
+ MMO.setType(NewTy);
+ Observer.changedInstr(MI);
+ }
+ } else {
+ // For stores of i1: G_ANYEXT from i1 is custom-legalized (produces
+ // proper masking), so use Helper.widenScalar directly.
+ Helper.widenScalar(MI, 0, NewTy);
+ MI.memoperands()[0]->setType(NewTy);
+ }
+ } else if (CurTy.isVector() && (CurTy.getScalarSizeInBits() > 1) &&
+ (CurTy.getScalarSizeInBits() < 8)) {
+ // Vectors with sub-byte elements: bitcast to scalar, then widen to
+ // multiple of 8 bits (minimum 8).
+ // e.g. <2 x i4> -> i8, <2 x i2> -> i4 -> i8, <65 x i2> -> i130 -> i136
+ unsigned ScalarSize = CurTySize;
+ unsigned NewSize = std::max(8u, ((ScalarSize + 7) & ~7u));
+ auto NewTy = LLT::integer(NewSize);
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ Register NewVal = MRI.createGenericVirtualRegister(NewTy);
+ Observer.changingInstr(MI);
+ ValMO.setReg(NewVal);
+ MMO.setType(NewTy);
+ Observer.changedInstr(MI);
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ if (ScalarSize == NewSize) {
+ B.buildBitcast(Val, NewVal);
+ } else {
+ auto Trunc = B.buildTrunc(LLT::integer(ScalarSize), NewVal);
+ B.buildBitcast(Val, Trunc);
+ }
+ } else {
+ // Store: bitcast vector to scalar integer. If already byte-aligned,
+ // update MI directly. Otherwise, let the legalizer re-process as
+ // scalar sub-byte store on the next iteration.
+ if (ScalarSize == NewSize) {
+ Register NewVal = MRI.createGenericVirtualRegister(NewTy);
+ B.buildBitcast(NewVal, Val);
+ Observer.changingInstr(MI);
+ ValMO.setReg(NewVal);
+ MMO.setType(NewTy);
+ Observer.changedInstr(MI);
+ } else {
+ // Vector total size is not byte-aligned. Bitcast to scalar and
+ // insert into a wider byte-aligned integer using G_INSERT.
+ // Upper bits are don't-care for stores.
+ Register CastReg =
+ B.buildBitcast(LLT::integer(ScalarSize), Val).getReg(0);
+ Register UndefReg = B.buildUndef(NewTy).getReg(0);
+ Register NewVal = B.buildInsert(NewTy, UndefReg, CastReg, 0).getReg(0);
+ Observer.changingInstr(MI);
+ ValMO.setReg(NewVal);
+ MMO.setType(NewTy);
+ Observer.changedInstr(MI);
+ }
+ }
+ } else if (shouldWidenLoad(MI.getOpcode(), CurTy, AddressSpace,
+ MMO.getAlign().value() * 8)) {
+ assert(CurTy.getScalarSizeInBits() == 32 &&
+ "ShouldWidenLoad: Only 32-bit elements reach here");
+ assert(
+ (AddressSpace == static_cast<unsigned>(PISAAS::AddressSpace::GLOBAL) ||
+ AddressSpace == static_cast<unsigned>(PISAAS::AddressSpace::SHARED) ||
+ AddressSpace ==
+ static_cast<unsigned>(PISAAS::AddressSpace::CONSTANT)) &&
+ "ShouldWidenLoad: Only global,shared,constant loads should reach here");
+ // Get alignment in bytes
+ Align AlignInBytes = MMO.getAlign();
+ Register OriginalVal = ValMO.getReg();
+ Register NewVal;
+ auto NumElts = CurTy.getNumElements();
+ // If alignment is at least 8 bytes and number of elements is 5 or 6,
+ // widen to 3 elements of i64. Otherwise, widen to 8 elements of i32.
+ bool CanWidenToI64 = (AlignInBytes.value() >= 8) && (NumElts <= 6);
+ auto NewTy = CanWidenToI64 ? LLT::fixed_vector(3, LLT::integer(64))
+ : LLT::fixed_vector(8, LLT::integer(32));
+ auto VecN32Ty = LLT::fixed_vector(NumElts, LLT::integer(32));
+
+ // Create the new widened load/store
+ Observer.changingInstr(MI);
+ NewVal = MRI.createGenericVirtualRegister(NewTy);
+ MMO.setType(NewTy);
+ ValMO.setReg(NewVal);
+ Observer.changedInstr(MI);
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ // DBG_VALUE should be associated with the original load
+ updateRegInDebugValue(OriginalVal, NewVal, MRI);
+
+ Register ResultReg;
+ if (CanWidenToI64) {
+ // Handle vectors with 5 or 6 elements of i32 with alignment >= 8 bytes
+ // Legalize to 3 elements of i64 for better hardware utilization
+ auto V6i32Ty = LLT::fixed_vector(6, LLT::integer(32));
+ auto ExtrVal = MRI.createGenericVirtualRegister(V6i32Ty);
+ // Bitcast to 6xi32 first, then extract the
+ // first 5 elements
+ B.buildBitcast(ExtrVal, NewVal);
+ ResultReg = B.buildExtractSubvector(VecN32Ty, ExtrVal, 0).getReg(0);
+ } else {
+ // Handle vectors with 5, 6, or 7 elements of i32 with alignment <= 8
+ // bytes or 7 elements. Expand them to 8 elements for better hardware
+ // utilization
+ assert(NumElts == 7 || AlignInBytes.value() < 8);
+ // After loading the 8-element vector, extract the needed elements
+ ResultReg = B.buildExtractSubvector(VecN32Ty, NewVal, 0).getReg(0);
+ }
+ if (CurTy.getScalarType().isPointer()) {
+ B.buildIntToPtr(Val, ResultReg);
+ } else {
+ B.buildCopy(Val, ResultReg);
+ }
+ } else if (CurTy.getScalarType().isPointer()) {
+ // load/store of ptr requires inttoptr/ptrtoint
+ auto EltSize = CurTy.getScalarSizeInBits();
+ LLT NewTy = CurTy.changeElementType(LLT::integer(EltSize));
+ Register NewVal = MRI.createGenericVirtualRegister(NewTy);
+ MMO.setType(NewTy);
+ ValMO.setReg(NewVal);
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ B.buildIntToPtr(Val, NewVal);
+ } else {
+ B.buildPtrToInt(NewVal, Val);
+ }
+ } else if (CurTy.getScalarSizeInBits() == 1) {
+ auto BitSize = CurTy.getSizeInBits();
+ auto NumEltsI8 = (BitSize + 7) / 8;
+ auto NewBitSize = NumEltsI8 * 8;
+ assert(CurTy.isVector() &&
+ "Expected only vector of i1 to reach here, scalar was extended to "
+ "i8 on widen scalars to be multiple of 8");
+ bool NoExtensionNeeded = (BitSize == NewBitSize);
+ LLT NewI8Ty = (NumEltsI8 > 1) ? LLT::fixed_vector(NumEltsI8, I8) : I8;
+ Register NewI8Val = MRI.createGenericVirtualRegister(NewI8Ty);
+ Register OriginalVal = ValMO.getReg();
+ Observer.changingInstr(MI);
+ MMO.setType(NewI8Ty);
+ ValMO.setReg(NewI8Val);
+ Observer.changedInstr(MI);
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ updateRegInDebugValue(OriginalVal, NewI8Val, MRI);
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ if (NoExtensionNeeded)
+ B.buildBitcast(Val, NewI8Val);
+ else {
+ LLT NewI1Ty = LLT::fixed_vector(NewBitSize, I1);
+ Register NewI1Val = MRI.createGenericVirtualRegister(NewI1Ty);
+ B.buildBitcast(NewI1Val, NewI8Val);
+ B.buildDeleteTrailingVectorElements(Val, NewI1Val);
+ }
+ } else {
+ if (NoExtensionNeeded)
+ B.buildBitcast(NewI8Val, Val);
+ else {
+ LLT NewI1Ty = LLT::fixed_vector(NewBitSize, I1);
+ Register NewI1Val = MRI.createGenericVirtualRegister(NewI1Ty);
+ B.buildPadVectorWithUndefElements(NewI1Val, Val);
+ B.buildBitcast(NewI8Val, NewI1Val);
+ }
+ }
+ } else {
+ llvm_unreachable("unhandled load/store case");
+ }
+ return true;
+}
+
+static bool legalizeGFrem(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ Register DstReg = MI.getOperand(0).getReg();
+ Register Src0Reg = MI.getOperand(1).getReg();
+ Register Src1Reg = MI.getOperand(2).getReg();
+ auto Flags = MI.getFlags();
+ auto FmAfn = Flags & MachineInstr::FmAfn;
+ LLT Ty = MRI.getType(DstReg);
+
+ auto DivFlags = Flags;
+ if (FmAfn) {
+ DivFlags &= ~MachineInstr::FmAfn;
+ DivFlags |= MachineInstr::FmArcp;
+ }
+ auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, DivFlags);
+ auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
+ auto Neg = B.buildFNeg(Ty, Trunc, Flags);
+ if (!FmAfn) {
+ auto FMA = B.buildFMA(Ty, Neg, Src1Reg, Src0Reg, Flags);
+
+ auto &Semantics = getFltSemanticForLLT(Ty.getScalarType());
+ auto InfC = B.buildFConstant(Ty, APFloat::getInf(Semantics));
+
+ auto XAbs = B.buildIntrinsic(Intrinsic::pisa_fabs, {Ty})
+ .addUse(Src0Reg)
+ .setMIFlags(Flags);
+ auto YAbs = B.buildIntrinsic(Intrinsic::pisa_fabs, {Ty})
+ .addUse(Src1Reg)
+ .setMIFlags(Flags);
+ // Using pisa_fabs is safe here: the result is only compared against Inf
+ // via OEQ, which is false for any NaN regardless of signaling/quiet.
+ auto XFCmp = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, XAbs, InfC, Flags);
+ auto YFCmp = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, YAbs, InfC, Flags);
+ auto Sel = B.buildSelect(Ty, YFCmp, Src0Reg, FMA);
+ B.buildSelect(DstReg, XFCmp, FMA, Sel);
+ } else {
+ B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+// IEEE 754 fabs: clear the sign bit via bitwise AND.
+// This is used when NaN inputs cannot be ruled out, ensuring the sign bit
+// is cleared without quieting the NaN (unlike the PISA fabs instruction).
+static bool legalizeFAbs(MachineInstr &MI, MachineIRBuilder &B) {
+ Register DstReg = MI.getOperand(0).getReg();
+ Register SrcReg = MI.getOperand(1).getReg();
+ MachineRegisterInfo &MRI = *B.getMRI();
+ LLT Ty = MRI.getType(DstReg);
+
+ // <2 x half> / <2 x bfloat>: pack into a single 32-bit AND.
+ if (Ty.isVector() && Ty.getNumElements() == 2 &&
+ Ty.getScalarSizeInBits() == 16) {
+ auto Src32 = B.buildBitcast(I32, SrcReg);
+ // 0x7FFF7FFF: clears the sign bit of each 16-bit element.
+ auto Mask = B.buildConstant(I32, 0x7FFF7FFF);
+ auto And = B.buildAnd(I32, Src32, Mask);
+ B.buildBitcast(DstReg, And);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ unsigned BitWidth = Ty.getSizeInBits();
+
+ // Sign bit mask: all ones except the MSB.
+ APInt Mask = APInt::getSignedMaxValue(BitWidth);
+ LLT IntTy = LLT::integer(BitWidth);
+
+ // G_AND is only legal on (any-)scalar integer LLTs, so for typed-float
+ // operands we bitcast through the integer LLT. LLT::operator== treats an
+ // any-scalar as equal to a typed float of the same width, so distinguish
+ // by isFloat() instead of by inequality.
+ bool IsTypedFloat = Ty.isFloat();
+
+ Register IntSrc = SrcReg;
+ if (IsTypedFloat)
+ IntSrc = B.buildBitcast(IntTy, SrcReg).getReg(0);
+
+ auto MaskCst = B.buildConstant(IntTy, Mask);
+ auto And = B.buildAnd(IntTy, IntSrc, MaskCst);
+
+ if (IsTypedFloat)
+ B.buildBitcast(DstReg, And);
+ else
+ B.buildCopy(DstReg, And);
+
+ MI.eraseFromParent();
+ return true;
+}
+
+// Support for bf/hf type is limited to fdiv.fast
+// If non-afn division is requested, we extend args to float,
+// perform the division and truncate the result back to hf/bf
+static bool legalizeGFdiv(MachineInstr &MI, MachineIRBuilder &B) {
+ // natively supported
+ if (MI.getFlag(MachineInstr::FmArcp))
+ return true;
+
+ // perform converts
+ auto &MRI = *B.getMRI();
+ auto [DstReg, Src0Reg, Src1Reg] = MI.getFirst3Regs();
+
+ auto Src0Tmp = MRI.createGenericVirtualRegister(F32);
+ auto Src1Tmp = MRI.createGenericVirtualRegister(F32);
+ auto DstTmp = MRI.createGenericVirtualRegister(F32);
+
+ auto FPExt0 = B.buildFPExt(Src0Tmp, Src0Reg);
+ auto FPExt1 = B.buildFPExt(Src1Tmp, Src1Reg);
+ auto FDiv = B.buildFDiv(DstTmp, FPExt0, FPExt1);
+ B.buildFPTrunc(DstReg, FDiv);
+
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
+ MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ Register SrcReg = MI.getOperand(1).getReg();
+ Register EltReg = MI.getOperand(2).getReg();
+ Register IndexReg = MI.getOperand(3).getReg();
+
+ LLT SrcTy = MRI.getType(SrcReg);
+ LLT EltTy = MRI.getType(EltReg);
+ LLT IndexTy = MRI.getType(IndexReg);
+
+ int EltSize = EltTy.getSizeInBits();
+ int NumElts = SrcTy.getNumElements();
+
+ if (EltSize == 1) {
+ // Handle insertion to <n x i1>
+ int Size = MRI.getType(SrcReg).getSizeInBits();
+ assert(isPowerOf2_32(Size) && "need to extend source to be power of 2");
+
+ LLT ScalarTy = LLT::integer(Size);
+
+ auto SScalarReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto DScalarReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto MaskReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto NotReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto AndReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto ShiftReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto EltZExtReg = MRI.createGenericVirtualRegister(ScalarTy);
+ auto ShiftAmountReg = IndexReg;
+
+ B.buildBitcast(SScalarReg, SrcReg);
+ auto Value = getIConstantVRegValWithLookThrough(IndexReg, MRI);
+
+ if (Value.has_value()) { // constant index
+ B.buildConstant(MaskReg, 1ull << Value->Value.getZExtValue());
+ ShiftAmountReg = MRI.createGenericVirtualRegister(ScalarTy);
+ B.buildConstant(ShiftAmountReg, Value->Value.getZExtValue());
+ } else { // non-constant index
+ auto ConstReg = MRI.createGenericVirtualRegister(ScalarTy);
+ B.buildConstant(ConstReg, 1ull);
+ B.buildShl(MaskReg, ConstReg, IndexReg);
+ }
+
+ B.buildNot(NotReg, MaskReg);
+ B.buildAnd(AndReg, SScalarReg, NotReg);
+ B.buildZExt(EltZExtReg, EltReg);
+ B.buildShl(ShiftReg, EltZExtReg, ShiftAmountReg);
+ B.buildOr(DScalarReg, AndReg, ShiftReg);
+ B.buildBitcast(MI.getOperand(0), DScalarReg);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ assert((EltSize == 8 || EltSize == 16 || EltSize == 64) &&
+ "unexpected element size");
+
+ // If the index is constant, narrow the vector down to 4 elements.
+ if (auto MaybeValue = getIConstantVRegValWithLookThrough(IndexReg, MRI)) {
+ if (NumElts <= 4) {
+ SmallVector<Register, 4> Elements;
+ for (int I = 0; I < NumElts; ++I)
+ Elements.push_back(MRI.createGenericVirtualRegister(EltTy));
+
+ B.buildUnmerge(Elements, SrcReg);
+ Elements[MaybeValue->Value.getZExtValue()] = EltReg;
+ B.buildBuildVector(MI.getOperand(0), Elements);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Narrow to 4 elements.
+ auto Res = Helper.fewerElementsVector(MI, 0, LLT::fixed_vector(4, EltTy));
+ return Res != LegalizerHelper::UnableToLegalize;
+ }
+
+ if (EltSize <= 16) {
+ // Handle insertion to <n x i8> and <n x i16>, where the vector size is not
+ // a multiple of 32 bits. The vector is extended to the next multiple of 32
+ // bits, and then bitcast to a vector of i32 for the insertion. The vector
+ // cannot be narrowed here because the index is not constant, and we don't
+ // know which elements will be inserted.
+ if (NumElts * EltSize % 32 != 0) {
+ int NewNumElts = alignTo(NumElts, 32 / EltSize);
+ LLT NewVecTy = LLT::fixed_vector(NewNumElts, EltTy);
+ auto Res = Helper.moreElementsVector(MI, 0, NewVecTy);
+ if (Res == LegalizerHelper::UnableToLegalize)
+ return false;
+ NumElts = NewNumElts;
+ B.setInsertPt(*MI.getParent(), MI);
+ }
+
+ // Now the vector size is a multiple of 32 bits, we can bitcast to a vector
+ // of s32 and insert the element.
+ // When the element type is float (e.g. f16), bitcastInsertVectorElt will
+ // emit G_ZEXT on the element, but G_ZEXT of a float type is invalid.
+ // Work around by converting to integer types, delegating, and bitcasting
+ // back.
+ Register OrigDst = MI.getOperand(0).getReg();
+ Register IntDst;
+ bool NeedFloatBitcast = EltTy.isFloat();
+ if (NeedFloatBitcast) {
+ LLT IntEltTy = LLT::integer(EltSize);
+ LLT IntVecTy = LLT::fixed_vector(NumElts, IntEltTy);
+
+ // Bitcast element from float to integer.
+ Register IntElt = MRI.createGenericVirtualRegister(IntEltTy);
+ B.buildBitcast(IntElt, MI.getOperand(2).getReg());
+ MI.getOperand(2).setReg(IntElt);
+
+ // Bitcast source vector to integer element type.
+ Register IntSrc = MRI.createGenericVirtualRegister(IntVecTy);
+ B.buildBitcast(IntSrc, MI.getOperand(1).getReg());
+ MI.getOperand(1).setReg(IntSrc);
+
+ // Replace destination with integer vector type.
+ IntDst = MRI.createGenericVirtualRegister(IntVecTy);
+ MI.getOperand(0).setReg(IntDst);
+ }
+ int NewNumElts = NumElts * EltSize / 32;
+ LLT NewVecTy = NewNumElts == 1 ? I32 : LLT::fixed_vector(NewNumElts, I32);
+ auto Res = Helper.bitcastInsertVectorElt(MI, 0, NewVecTy);
+ if (NeedFloatBitcast && Res != LegalizerHelper::UnableToLegalize) {
+ MachineInstr *DefMI = MRI.getVRegDef(IntDst);
+ B.setInsertPt(*DefMI->getParent(), std::next(DefMI->getIterator()));
+ B.buildBitcast(OrigDst, IntDst);
+ }
+ return Res != LegalizerHelper::UnableToLegalize;
+ }
+
+ assert(EltSize == 64 && "unexpected element size");
+ LLT NewVecTy = LLT::fixed_vector(NumElts * 2, I32);
+
+ // Compute the low and high indices as low = index * 2, high = low + 1
+ auto One = B.buildConstant(IndexTy, 1).getReg(0);
+ auto LowIndexReg = B.buildShl(IndexTy, IndexReg, One).getReg(0);
+ auto HighIndexReg = B.buildAdd(IndexTy, LowIndexReg, One).getReg(0);
+
+ // Split the 64-bit element into two 32-bit elements
+ auto EltLowReg = MRI.createGenericVirtualRegister(I32);
+ auto EltHighReg = MRI.createGenericVirtualRegister(I32);
+ B.buildUnmerge({EltLowReg, EltHighReg}, EltReg);
+
+ // Bitcast the source vector to s32 vector
+ auto BitcastSrcReg = B.buildBitcast(NewVecTy, SrcReg).getReg(0);
+
+ // Insert the low and high parts
+ auto InsertLowReg = B.buildInsertVectorElement(NewVecTy, BitcastSrcReg,
+ EltLowReg, LowIndexReg)
+ .getReg(0);
+ auto InsertHighReg = B.buildInsertVectorElement(NewVecTy, InsertLowReg,
+ EltHighReg, HighIndexReg)
+ .getReg(0);
+
+ // Bitcast back to the original vector type
+ B.buildBitcast(MI.getOperand(0), InsertHighReg);
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
+ MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ Register VecReg = MI.getOperand(1).getReg();
+ LLT VecTy = MRI.getType(VecReg);
+ LLT EltTy = VecTy.getScalarType();
+ int EltSize = VecTy.getScalarSizeInBits();
+ int NumElts = VecTy.getNumElements();
+
+ if (EltSize == 1) {
+ // Handle extraction of <n x i1>
+ auto Size = MRI.getType(VecReg).getSizeInBits();
+ auto CastReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
+ B.buildBitcast(CastReg, VecReg);
+ auto ShiftReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
+ B.buildLShr(ShiftReg, CastReg, MI.getOperand(2));
+ B.buildTrunc(MI.getOperand(0), ShiftReg);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ assert((EltSize == 8 || EltSize == 16 || EltSize == 64) &&
+ "unexpected element size");
+
+ // If the index is constant, narrow the vector down to 4 elements.
+ Register IndexReg = MI.getOperand(2).getReg();
+ if (auto MaybeValue = getIConstantVRegValWithLookThrough(IndexReg, MRI)) {
+ if (NumElts <= 4) {
+ Register UnmergeReg = B.buildUnmerge(EltTy, VecReg)
+ .getReg(MaybeValue->Value.getZExtValue());
+ B.buildCopy(MI.getOperand(0), UnmergeReg);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // Narrow to 4 elements.
+ auto Res = Helper.fewerElementsVector(MI, 1, LLT::fixed_vector(4, EltTy));
+ return Res != LegalizerHelper::UnableToLegalize;
+ }
+
+ // Handle extraction from <n x i8> and <n x i16>, where the vector size is not
+ // a multiple of 32 bits. The vector is extended to the next multiple of 32
+ // bits, and then bitcast to a vector of i32 for the extraction. The vector
+ // cannot be narrowed here because the index is not constant, and we don't
+ // know which elements will be extracted.
+ if (NumElts * EltSize % 32 != 0) {
+ int NewNumElts = alignTo(NumElts, 32 / EltSize);
+ LLT NewVecTy = LLT::fixed_vector(NewNumElts, EltTy);
+ auto Res = Helper.moreElementsVector(MI, 1, NewVecTy);
+ if (Res == LegalizerHelper::UnableToLegalize)
+ return false;
+ NumElts = NewNumElts;
+ }
+
+ // Now the vector size is a multiple of 32 bits, we can bitcast to a vector
+ // of s32 and extract the element.
+ // When the element type is float (e.g. f16), bitcastExtractVectorElt will
+ // emit G_TRUNC to the original element type, but G_TRUNC to a float type is
+ // invalid. Work around this by replacing the destination with an integer
+ // type, delegating to the helper, and then bitcasting back to float.
+ Register OrigDst = MI.getOperand(0).getReg();
+ Register IntDst;
+ bool NeedFloatBitcast = EltTy.isFloat();
+ if (NeedFloatBitcast) {
+ LLT IntEltTy = LLT::integer(EltSize);
+ IntDst = MRI.createGenericVirtualRegister(IntEltTy);
+ MI.getOperand(0).setReg(IntDst);
+ // Also patch the source vector to integer element type so bitcast is valid.
+ LLT IntVecTy = LLT::fixed_vector(NumElts, IntEltTy);
+ Register IntVec = MRI.createGenericVirtualRegister(IntVecTy);
+ B.buildBitcast(IntVec, MI.getOperand(1).getReg());
+ MI.getOperand(1).setReg(IntVec);
+ VecTy = IntVecTy;
+ }
+ int NewNumElts = NumElts * EltSize / 32;
+ LLT NewVecTy = NewNumElts == 1 ? I32 : LLT::fixed_vector(NewNumElts, I32);
+ auto Res = Helper.bitcastExtractVectorElt(MI, 1, NewVecTy);
+ if (NeedFloatBitcast && Res != LegalizerHelper::UnableToLegalize) {
+ // MI has been erased by the helper. IntDst now has an integer-typed def
+ // from the helper's lowered sequence. Bitcast it back to the original
+ // float type. Reset the insert point since MI was erased.
+ MachineInstr *DefMI = MRI.getVRegDef(IntDst);
+ B.setInsertPt(*DefMI->getParent(), std::next(DefMI->getIterator()));
+ B.buildBitcast(OrigDst, IntDst);
+ }
+ return Res != LegalizerHelper::UnableToLegalize;
+}
+
+static bool legalizeGBswap(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ auto [Dst, Src] = MI.getFirst2Regs();
+ const LLT Ty = MRI.getType(Src);
+ unsigned BitSize = Ty.getScalarSizeInBits();
+
+ assert(BitSize % 16 == 0 && "bswap only supported for multiples of 16 bits");
+
+ // Masks for byte swapping
+ static const std::array<int, 2> SwapMask16 = {1, 0};
+ static const std::array<int, 4> SwapMask32 = {3, 2, 1, 0};
+
+ // Helper lambda for byte-swapping: returns a tuple describing how to swap
+ // bytes within each chunk. The tuple contains:
+ // - ChunkSize: the size in bits of each chunk (either 16 or 32).
+ // - ChunkByteSwapMask: the shuffle mask used to reverse the byte order within
+ // a chunk.
+ // - ChunkShuffleVecTy: the vector type used for shuffling bytes within a
+ // chunk.
+ auto GetSwapProps =
+ [&](unsigned BitSize) -> std::tuple<unsigned, ArrayRef<int>, LLT> {
+ return (BitSize % 32 == 0)
+ ? std::make_tuple(32u, ArrayRef<int>(SwapMask32), V4I8)
+ : std::make_tuple(16u, ArrayRef<int>(SwapMask16), V2I8);
+ };
+
+ auto [ChunkSize, ChunkByteSwapMask, ChunkShuffleVecTy] =
+ GetSwapProps(BitSize);
+
+ // For Src types that are multiples of 32 bits, the value is divided into
+ // 32-bit chunks. Each chunk is byte-swapped, and the resulting chunks are
+ // built into a vector in reverse order. For Src types that are multiples of
+ // 16 bits (but not 32), the value is divided into 16-bit chunks. Each chunk
+ // is byte-swapped and reassembled in reverse order.
+ if (BitSize == 16 || BitSize == 32) {
+ assert(ChunkSize == BitSize &&
+ "Single chunk case: ChunkSize must equal BitSize");
+ auto VecReg = B.buildBitcast(ChunkShuffleVecTy, Src);
+ auto ShufReg = B.buildShuffleVector(ChunkShuffleVecTy, VecReg, VecReg,
+ ChunkByteSwapMask);
+ B.buildBitcast(Dst, ShufReg);
+ } else {
+ unsigned NumChunks = BitSize / ChunkSize;
+ LLT ChunkTy = LLT::integer(ChunkSize);
+ LLT VecTy = LLT::fixed_vector(NumChunks, ChunkTy);
+ auto VecReg = B.buildBitcast(VecTy, Src);
+
+ SmallVector<Register, 8> SwappedChunks;
+ for (int I = NumChunks - 1; I >= 0; --I) {
+ auto Index = B.buildConstant(I32, I);
+ auto ChunkReg = B.buildExtractVectorElement(ChunkTy, VecReg, Index);
+ auto ChunkVec = B.buildBitcast(ChunkShuffleVecTy, ChunkReg);
+ auto SwappedVec = B.buildShuffleVector(ChunkShuffleVecTy, ChunkVec,
+ ChunkVec, ChunkByteSwapMask);
+ auto SwappedChunk = B.buildBitcast(ChunkTy, SwappedVec);
+ SwappedChunks.push_back(SwappedChunk.getReg(0));
+ }
+
+ auto FinalVec = B.buildBuildVector(VecTy, SwappedChunks);
+ B.buildBitcast(Dst, FinalVec);
+ }
+
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGFpow(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+
+ auto [Dst, Src0, Src1] = MI.getFirst3Regs();
+ auto DstTy = MRI.getType(Dst);
+ assert(DstTy.isScalar() &&
+ (DstTy.getSizeInBits() == 32 || DstTy.getSizeInBits() == 16));
+
+ unsigned Flags = MI.getFlags();
+
+ // can only do approximation of pow()
+ auto AllowApprox = MI.getFlag(MachineInstr::FmAfn);
+ if (!AllowApprox)
+ llvm_unreachable("not implemented (fpow)");
+
+ auto LogReg = MRI.createGenericVirtualRegister(DstTy);
+ auto MulReg = MRI.createGenericVirtualRegister(DstTy);
+ auto FExp2Reg = Dst;
+
+ B.buildFLog2(LogReg, Src0, Flags);
+ B.buildFMul(MulReg, LogReg, Src1, Flags);
+ B.buildFExp2(FExp2Reg, MulReg, Flags);
+
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ auto [Dst, Src0, Src1] = MI.getFirst3Regs();
+ unsigned Flags = MI.getFlags();
+
+ LLT XTy = MRI.getType(Src0);
+ LLT NTy = MRI.getType(Src1);
+ LLT Src1Ty = MRI.getType(Src1);
+
+ auto AllowApprox =
+ XTy.getSizeInBits() <= 32 && MI.getFlag(MachineInstr::FmAfn);
+ auto IsBFloat16 = XTy == LLT::bfloat16();
+ if (AllowApprox) {
+ auto RegLLT = MRI.getType(Dst);
+ auto FpReg = MRI.createGenericVirtualRegister(RegLLT);
+ auto ExpReg = MRI.createGenericVirtualRegister(RegLLT);
+
+ B.buildSITOFP(FpReg, Src1);
+ B.buildFExp2(ExpReg, FpReg, Flags);
+ B.buildFMul(Dst, Src0, ExpReg, Flags);
+
+ MI.eraseFromParent();
+ return true;
+ }
+
+ int NClampRangeVal, NShiftVal, NDivBy3ShiftVal, NDivBy3MulVal;
+ if (XTy.getSizeInBits() == 16 && !IsBFloat16) {
+ NClampRangeVal = 14;
+ NShiftVal = 10;
+ NDivBy3ShiftVal = 8;
+ NDivBy3MulVal = 0x56;
+ } else if (XTy.getSizeInBits() == 32 || IsBFloat16) {
+ NClampRangeVal = 126;
+ NShiftVal = IsBFloat16 ? 7 : 23;
+ NDivBy3ShiftVal = 16;
+ NDivBy3MulVal = 0x5556;
+ } else {
+ // double precision
+ NClampRangeVal = 1022;
+ NShiftVal = 52;
+ NDivBy3ShiftVal = 16;
+ NDivBy3MulVal = 0x5556;
+ }
+
+ // Limit range of n (such that all inputs can be handled correctly)
+ // For FP32, |n|>128+126+23 will definitely lead to overflow/underflow
+ // |n|<=126*3 is a sufficiently wide range for n (and FP32 x)
+ // For FP64, |n|>1024+1022+52 will definitely lead to overflow/underflow
+ // |n|<=1022*3 is a sufficiently wide range for n (and FP64 x)
+
+ auto ClampMax = B.buildConstant(NTy, -NClampRangeVal * 3);
+ auto NClampedMax = B.buildSMax(NTy, Src1, ClampMax);
+ auto ClampMin = B.buildConstant(NTy, NClampRangeVal * 3);
+ auto NClamped = B.buildSMin(NTy, NClampedMax, ClampMin);
+
+ auto AddConst = B.buildConstant(NTy, (NClampRangeVal + 1) * 3);
+ auto N = B.buildAdd(NTy, NClamped, AddConst);
+ if (XTy.getSizeInBits() == 16 && !IsBFloat16) {
+ NTy = I16;
+ N = B.buildTrunc(NTy, N);
+ }
+
+ // for fp16, n/3 performed as a 8x8-bit->16-bit integer MUL and SHR by 8.
+ // for others, n/3, performed as a 16x16-bit->32-bit integer MUL and SHR by 16
+ // (both LSHR or ASHR work, n is positive at this point)
+ auto MulConst = B.buildConstant(NTy, NDivBy3MulVal);
+ auto NMul = B.buildMul(NTy, N, MulConst);
+ auto ShrConst = B.buildConstant(I32, NDivBy3ShiftVal);
+ auto K0 = B.buildLShr(NTy, NMul, ShrConst);
+
+ auto NMinusK0 = B.buildSub(NTy, N, K0);
+ auto K1 = B.buildSub(NTy, NMinusK0, K0);
+
+ if (XTy.getSizeInBits() == 64) {
+ NTy = I64;
+ K0 = B.buildZExt(NTy, K0);
+ K1 = B.buildZExt(NTy, K1);
+ } else if (IsBFloat16) {
+ NTy = I16;
+ K0 = B.buildTrunc(NTy, K0);
+ K1 = B.buildTrunc(NTy, K1);
+ }
+
+ auto ShlConst = B.buildConstant(I32, NShiftVal);
+ auto SK0I = B.buildShl(NTy, K0, ShlConst);
+ auto SK1I = B.buildShl(NTy, K1, ShlConst);
+ auto SK0 = B.buildBitcast(XTy, SK0I);
+ auto SK1 = B.buildBitcast(XTy, SK1I);
+
+ SrcOp SwapperX(Src0), SwapperSK1(SK1);
+ if (XTy.getSizeInBits() > 16) {
+ // Swap Src0 with SK1 if n is sufficiently small for SK1 * SK0 * SK0 not to
+ // overflow (inf). This prevents a potential underflow that can happen with
+ // Src0 * SK0 * SK0.
+ int SmallThresholdVal = NClampRangeVal / 3;
+ auto SmallThresholdConst = B.buildConstant(Src1Ty, SmallThresholdVal);
+ auto Src1Abs = B.buildAbs(Src1Ty, NClamped);
+ auto IsSrc1Small = B.buildICmp(CmpInst::Predicate::ICMP_SLT, I1, Src1Abs,
+ SmallThresholdConst);
+
+ SwapperX = B.buildSelect(XTy, IsSrc1Small, SK1, Src0);
+ SwapperSK1 = B.buildSelect(XTy, IsSrc1Small, Src0, SK1);
+ }
+
+ auto Res0 = B.buildFMul(XTy, SwapperX, SK0, Flags);
+ auto Res1 = B.buildFMul(XTy, Res0, SK0, Flags);
+ B.buildFMul(Dst, Res1, SwapperSK1, Flags);
+
+ MI.eraseFromParent();
+ return true;
+}
+
+// PISA specification provides no support for 16bit fsqrt with rounding mode
+// - extend to 32bit value
+// - perform square root with rounding mode
+// - truncate to 16bit value
+static bool legalizeIntrinsicFSqrt(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ auto &MRI = *B.getMRI();
+
+ SmallVector<MachineInstr *, 4> MIs;
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ if (MRI.getType(MI.getOperand(0).getReg()).isVector()) {
+ MIs = scalarizeIntrinsic(MI);
+ } else {
+ MIs.push_back(&MI);
+ }
+
+ for (auto It = MIs.begin(), Ite = MIs.end(); It != Ite; ++It) {
+ auto *MI = *It;
+ MachineIRBuilder MIB(*MI);
+
+ auto Dst = MI->getOperand(0).getReg();
+ auto Src = MI->getOperand(2).getReg();
+ auto Imm = MI->getOperand(3).getImm();
+
+ if (MRI.getType(Dst).getScalarSizeInBits() != 16)
+ continue; // already legal
+
+ auto Src32 = MRI.createGenericVirtualRegister(F32);
+ auto Dst32 = MRI.createGenericVirtualRegister(F32);
+ MIB.buildFPExt(Src32, Src);
+ MIB.buildIntrinsic(IntrinsicID, Dst32).addReg(Src32).addImm(Imm);
+ MIB.buildFPTrunc(Dst, Dst32);
+ MI->eraseFromParent();
+ }
+ return true;
+}
+
+static bool legalizeIntrinsicFDiv(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ auto &MRI = *B.getMRI();
+
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ SmallVector<MachineInstr *> MIs = scalarizeIntrinsic(MI);
+
+ // fdiv only supports 32/64 width
+ for (auto It = MIs.begin(), Ite = MIs.end(); It != Ite; ++It) {
+ auto *MI = *It;
+ MachineIRBuilder MIB(*MI);
+
+ auto Dst = MI->getOperand(0).getReg();
+ auto Src0 = MI->getOperand(2).getReg();
+ auto Src1 = MI->getOperand(3).getReg();
+
+ if (MRI.getType(Dst).getScalarSizeInBits() != 16)
+ continue;
+
+ // s16 A = FDIV s16 B, s16 C
+ // => s32 B' = FEXT s16 B
+ // => s32 C' = FEXT s16 C
+ // => s32 A' = FDIV s32 B', s32 C'
+ // => s16 A = FTRUNC s32 A'
+ auto Src032 = MRI.createGenericVirtualRegister(F32);
+ auto Src132 = MRI.createGenericVirtualRegister(F32);
+ auto Dst32 = MRI.createGenericVirtualRegister(F32);
+ MIB.buildFPExt(Src032, Src0);
+ MIB.buildFPExt(Src132, Src1);
+ MIB.buildIntrinsic(IntrinsicID, Dst32)
+ .addReg(Src032)
+ .addReg(Src132)
+ .add(MI->getOperand(4));
+ MIB.buildFPTrunc(Dst, Dst32);
+
+ MI->eraseFromParent();
+ }
+ return true;
+}
+
+static SmallVector<Register> splitVectorByGrain(MachineIRBuilder &B,
+ Register Src, unsigned Grain) {
+ auto &MRI = *B.getMRI();
+ auto SrcTy = MRI.getType(Src);
+ auto EltTy = SrcTy.getScalarType();
+
+ auto SliceTy = LLT::fixed_vector(Grain, EltTy);
+
+ if (SrcTy.isScalar()) {
+ auto SliceUndef = MRI.createGenericVirtualRegister(SliceTy);
+ auto Slice = MRI.createGenericVirtualRegister(SliceTy);
+ B.buildUndef(SliceUndef);
+ B.buildInsertVectorElement(Slice, SliceUndef, Src, B.buildConstant(I32, 0));
+ return {Slice};
+ }
+
+ const unsigned NumElts = SrcTy.getNumElements();
+
+ SmallVector<Register> Elts;
+ for (unsigned I = 0; I < NumElts; I += Grain) {
+ auto Slice = MRI.createGenericVirtualRegister(SliceTy);
+ B.buildUndef(Slice);
+
+ for (unsigned J = 0; J < std::min(Grain, NumElts - I); ++J) {
+ auto Idx = B.buildConstant(I32, J).getReg(0);
+ auto Elt =
+ B.buildExtractVectorElementConstant(EltTy, Src, I + J).getReg(0);
+ Slice = B.buildInsertVectorElement(SliceTy, Slice, Elt, Idx).getReg(0);
+ }
+ Elts.push_back(Slice);
+ }
+ return Elts;
+}
+
+static void joinVectorByGrain(MachineIRBuilder &B, Register Dst,
+ ArrayRef<Register> Srcs, unsigned Grain) {
+ auto &MRI = *B.getMRI();
+ auto DstTy = MRI.getType(Dst);
+ auto EltTy = DstTy.getScalarType();
+
+ if (DstTy.isScalar()) {
+ auto Src = Srcs[0];
+ B.buildExtractVectorElementConstant(Dst, Src, 0);
+ return;
+ }
+
+ const unsigned NumElts = DstTy.getNumElements();
+
+ auto TmpDst = B.buildUndef(DstTy).getReg(0);
+
+ for (unsigned I = 0; I < NumElts; I += Grain) {
+ auto &Src = Srcs[I / Grain];
+
+ for (unsigned J = 0; J < std::min(Grain, NumElts - I); ++J) {
+ auto Idx = B.buildConstant(I32, I + J).getReg(0);
+ auto Elt = B.buildExtractVectorElementConstant(EltTy, Src, J).getReg(0);
+
+ TmpDst = B.buildInsertVectorElement(DstTy, TmpDst, Elt, Idx).getReg(0);
+ }
+ }
+
+ B.buildCopy(Dst, TmpDst);
+}
+
+static bool legalizeIntrinsicBfn(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ auto &MRI = *B.getMRI();
+ auto IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ assert(IntrinsicID == Intrinsic::pisa_bfn);
+
+ const auto BfnOpcode = MI.getOperand(2);
+
+ const auto OrigDst = MI.getOperand(0).getReg();
+ const auto OrigSrc0 = MI.getOperand(3).getReg();
+ const auto OrigSrc1 = MI.getOperand(4).getReg();
+ const auto OrigSrc2 = MI.getOperand(5).getReg();
+
+ const auto Ty = MRI.getType(OrigDst);
+
+ const auto BitWidth = Ty.getScalarSizeInBits();
+ switch (BitWidth) {
+ default:
+ llvm_unreachable("unexpected bitwidth");
+ case 8:
+ case 16: {
+ const auto Grain = 32 / BitWidth;
+ MachineIRBuilder MIB(MI);
+
+ auto Srcs0 = splitVectorByGrain(MIB, OrigSrc0, Grain);
+ auto Srcs1 = splitVectorByGrain(MIB, OrigSrc1, Grain);
+ auto Srcs2 = splitVectorByGrain(MIB, OrigSrc2, Grain);
+
+ auto GrainTy = MRI.getType(Srcs0[0]);
+ SmallVector<Register> Dsts;
+
+ for (auto [Src0, Src1, Src2] : zip(Srcs0, Srcs1, Srcs2)) {
+ auto Dst = MRI.createGenericVirtualRegister(I32);
+ auto Src0Cast = MRI.createGenericVirtualRegister(I32);
+ auto Src1Cast = MRI.createGenericVirtualRegister(I32);
+ auto Src2Cast = MRI.createGenericVirtualRegister(I32);
+
+ auto DstCast = MRI.createGenericVirtualRegister(GrainTy);
+
+ MIB.buildBitcast(Src0Cast, Src0);
+ MIB.buildBitcast(Src1Cast, Src1);
+ MIB.buildBitcast(Src2Cast, Src2);
+
+ MIB.buildIntrinsic(IntrinsicID, Dst)
+ .add(BfnOpcode)
+ .addReg(Src0Cast)
+ .addReg(Src1Cast)
+ .addReg(Src2Cast);
+
+ MIB.buildBitcast(DstCast, Dst);
+ Dsts.push_back(DstCast);
+ }
+
+ joinVectorByGrain(MIB, OrigDst, Dsts, Grain);
+ MI.eraseFromParent();
+ } break;
+ case 32:
+ scalarizeIntrinsic(MI);
+ return true;
+ case 64: {
+ auto MIs = scalarizeIntrinsic(MI);
+
+ for (auto *MI : MIs) {
+ MachineIRBuilder MIB(*MI);
+ auto Dst = MI->getOperand(0).getReg();
+ auto Src0 = MI->getOperand(3).getReg();
+ auto Src1 = MI->getOperand(4).getReg();
+ auto Src2 = MI->getOperand(5).getReg();
+
+ auto DstV2I32 = MRI.createGenericVirtualRegister(V2I32);
+ auto Src0V2I32 = MRI.createGenericVirtualRegister(V2I32);
+ auto Src1V2I32 = MRI.createGenericVirtualRegister(V2I32);
+ auto Src2V2I32 = MRI.createGenericVirtualRegister(V2I32);
+
+ MIB.buildUndef(DstV2I32);
+
+ MIB.buildBitcast(Src0V2I32, Src0);
+ MIB.buildBitcast(Src1V2I32, Src1);
+ MIB.buildBitcast(Src2V2I32, Src2);
+
+ for (int I = 0; I < 2; I++) {
+ auto DstI32 = MRI.createGenericVirtualRegister(I32);
+ auto Src0I32 = MRI.createGenericVirtualRegister(I32);
+ auto Src1I32 = MRI.createGenericVirtualRegister(I32);
+ auto Src2I32 = MRI.createGenericVirtualRegister(I32);
+ auto Idx = MRI.createGenericVirtualRegister(I32);
+
+ MIB.buildExtractVectorElementConstant(Src0I32, Src0V2I32, I);
+ MIB.buildExtractVectorElementConstant(Src1I32, Src1V2I32, I);
+ MIB.buildExtractVectorElementConstant(Src2I32, Src2V2I32, I);
+
+ MIB.buildIntrinsic(IntrinsicID, DstI32)
+ .add(MI->getOperand(2))
+ .addReg(Src0I32)
+ .addReg(Src1I32)
+ .addReg(Src2I32);
+
+ MIB.buildConstant(Idx, I);
+
+ auto DstNext = MRI.createGenericVirtualRegister(V2I32);
+ MIB.buildInsertVectorElement(DstNext, DstV2I32, DstI32, Idx);
+ DstV2I32 = DstNext;
+ }
+
+ MIB.buildBitcast(Dst, DstV2I32);
+ MI->eraseFromParent();
+ }
+ } break;
+ }
+
+ return true;
+}
+
+static bool legalizeIntrinsicRE(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+
+ auto RndMode = MI.getOperand(MI.getNumOperands() - 1).getImm();
+ if (static_cast<RoundingMode>(RndMode) != RoundingMode::NearestTiesToEven)
+ return false; // only .re is supported
+
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ switch (IntrinsicID) {
+ default:
+ return false;
+ case Intrinsic::pisa_log_rnd:
+ B.buildFLog(MI.getOperand(0), MI.getOperand(2), MI.getFlags());
+ break;
+ case Intrinsic::pisa_log2_rnd:
+ B.buildFLog2(MI.getOperand(0), MI.getOperand(2), MI.getFlags());
+ break;
+ case Intrinsic::pisa_log10_rnd:
+ B.buildInstr(TargetOpcode::G_FLOG10, {MI.getOperand(0)}, {MI.getOperand(2)},
+ MI.getFlags());
+ break;
+ case Intrinsic::pisa_sin_rnd:
+ B.buildInstr(TargetOpcode::G_FSIN, {MI.getOperand(0)}, {MI.getOperand(2)},
+ MI.getFlags());
+ break;
+ case Intrinsic::pisa_cos_rnd:
+ B.buildInstr(TargetOpcode::G_FCOS, {MI.getOperand(0)}, {MI.getOperand(2)},
+ MI.getFlags());
+ break;
+ case Intrinsic::pisa_tanh_rnd:
+ B.buildInstr(TargetOpcode::G_FTANH, {MI.getOperand(0)}, {MI.getOperand(2)},
+ MI.getFlags());
+ break;
+ case Intrinsic::pisa_exp_rnd:
+ B.buildInstr(TargetOpcode::G_FEXP, {MI.getOperand(0)}, {MI.getOperand(2)},
+ MI.getFlags());
+ break;
+ case Intrinsic::pisa_exp2_rnd:
+ B.buildFExp2(MI.getOperand(0), MI.getOperand(2), MI.getFlags());
+ break;
+ case Intrinsic::pisa_pow_rnd:
+ B.buildFPow(MI.getOperand(0), MI.getOperand(2), MI.getOperand(3),
+ MI.getFlags());
+ break;
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeIntrinsicI2F(LegalizerHelper &Helper, MachineInstr &MI) {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ auto &MRI = *B.getMRI();
+
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ SmallVector<MachineInstr *> MIs = scalarizeIntrinsic(MI);
+
+ // @llvm.experimental.constrained.sitofp.f32.i1
+ for (auto It = MIs.begin(), Ite = MIs.end(); It != Ite; ++It) {
+ auto *MI = *It;
+ MachineIRBuilder MIB(*MI);
+
+ auto SrcReg = MI->getOperand(2).getReg();
+ auto SrcTy = MRI.getType(SrcReg);
+
+ if (SrcTy.getSizeInBits() >= 8)
+ continue;
+
+ auto ExtReg = MRI.createGenericVirtualRegister(I8);
+ if (IntrinsicID == Intrinsic::pisa_uitofp)
+ MIB.buildZExt(ExtReg, SrcReg);
+ else
+ MIB.buildSExt(ExtReg, SrcReg);
+ auto NewMI = MIB.buildIntrinsic(IntrinsicID, MI->getOperand(0).getReg())
+ .addReg(ExtReg)
+ .add(MI->getOperand(3))
+ .add(MI->getOperand(4));
+ NewMI.setMIFlags(MI->getFlags());
+ MI->eraseFromParent();
+ }
+ return true;
+}
+
+// Legalize dp4a_uu with saturation enabled:
+// dp4a_uu(acc, src1, src2, sat=true)
+// => tmp = dp4a_uu(0, src1, src2, sat=false)
+// dst = G_UADDSAT(tmp, acc)
+static bool legalizeIntrinsicDp4a(LegalizerHelper &Helper, MachineInstr &MI) {
+ unsigned Sat = MI.getOperand(5).getImm();
+ if (Sat == 0)
+ return true;
+
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ auto &MRI = *B.getMRI();
+ B.setInstrAndDebugLoc(MI);
+
+ Register Dst = MI.getOperand(0).getReg();
+ Register Acc = MI.getOperand(2).getReg();
+ Register Src1 = MI.getOperand(3).getReg();
+ Register Src2 = MI.getOperand(4).getReg();
+
+ Register Zero = B.buildConstant(I32, 0).getReg(0);
+ Register Tmp = MRI.createGenericVirtualRegister(I32);
+ B.buildIntrinsic(Intrinsic::pisa_dp4a_uu, ArrayRef<Register>{Tmp})
+ .addUse(Zero)
+ .addUse(Src1)
+ .addUse(Src2)
+ .addImm(0);
+ B.buildInstr(TargetOpcode::G_UADDSAT, {DstOp(Dst)}, {SrcOp(Tmp), SrcOp(Acc)});
+
+ MI.eraseFromParent();
+ return true;
+}
+
+bool PISALegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
+ MachineInstr &MI) const {
+ Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ switch (IntrinsicID) {
+ case Intrinsic::pisa_dp4a_uu:
+ return legalizeIntrinsicDp4a(Helper, MI);
+ case Intrinsic::pisa_fsqrt_rnd:
+ return legalizeIntrinsicFSqrt(Helper, MI);
+ case Intrinsic::pisa_fdiv_rnd:
+ return legalizeIntrinsicFDiv(Helper, MI);
+ case Intrinsic::pisa_bfn:
+ return legalizeIntrinsicBfn(Helper, MI);
+ case Intrinsic::pisa_log_rnd:
+ case Intrinsic::pisa_log2_rnd:
+ case Intrinsic::pisa_log10_rnd:
+ case Intrinsic::pisa_sin_rnd:
+ case Intrinsic::pisa_cos_rnd:
+ case Intrinsic::pisa_tanh_rnd:
+ case Intrinsic::pisa_exp_rnd:
+ case Intrinsic::pisa_exp2_rnd:
+ case Intrinsic::pisa_pow_rnd:
+ return legalizeIntrinsicRE(Helper, MI);
+ case Intrinsic::pisa_ired:
+ case Intrinsic::pisa_fred:
+ case Intrinsic::pisa_frcp:
+ case Intrinsic::pisa_frsqrt:
+ case Intrinsic::pisa_fabs:
+ case Intrinsic::pisa_smad:
+ case Intrinsic::pisa_fptosi_rnd:
+ case Intrinsic::pisa_fptoui_rnd:
+ case Intrinsic::pisa_fadd:
+ case Intrinsic::pisa_fsub:
+ case Intrinsic::pisa_fmul:
+ case Intrinsic::pisa_fma:
+ case Intrinsic::pisa_ftrunc:
+ case Intrinsic::pisa_frnd_rnd:
+ scalarizeIntrinsic(MI);
+ return true;
+ case Intrinsic::pisa_sitofp:
+ case Intrinsic::pisa_uitofp:
+ return legalizeIntrinsicI2F(Helper, MI);
+ default:
+ return true;
+ }
+}
+
+static bool legalizeGConcatVectors(MachineInstr &MI, MachineIRBuilder &B) {
+ auto *MRI = B.getMRI();
+
+ auto Dst = MI.getOperand(0).getReg();
+ auto DstTy = MRI->getType(Dst);
+ assert(DstTy.getScalarSizeInBits() == 32);
+ unsigned Idx = 0;
+
+ auto TDst = MRI->createGenericVirtualRegister(DstTy);
+ B.buildInstr(TargetOpcode::IMPLICIT_DEF).addDef(TDst);
+ for (unsigned I = 1; I < MI.getNumOperands(); I++) {
+ auto Src = MI.getOperand(I).getReg();
+ auto SrcTy = B.getMRI()->getType(Src);
+ auto NewDst = MRI->createGenericVirtualRegister(DstTy);
+ B.buildInsertSubvector(NewDst, TDst, Src, Idx);
+ Idx += SrcTy.getNumElements();
+ TDst = NewDst;
+ }
+ B.buildCopy(Dst, TDst);
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGUnmergeValues(LegalizerHelper &Helper, MachineInstr &MI,
+ MachineIRBuilder &B) {
+ auto *MRI = B.getMRI();
+
+ auto Src = MI.getOperand(MI.getNumOperands() - 1).getReg();
+ auto DstTy = MRI->getType(MI.getOperand(0).getReg());
+ assert(MRI->getType(Src).getScalarSizeInBits() == 32);
+
+ unsigned Idx = 0;
+ for (unsigned I = 0; I < MI.getNumOperands() - 1; I++) {
+ auto Dst = MI.getOperand(I).getReg();
+ if (DstTy.isVector()) {
+ // <2 x s32>, <2 x s32> = G_UNMERGE_VALUES <4 x s32>
+ B.buildExtractSubvector(Dst, Src, Idx);
+ Idx += DstTy.getNumElements();
+ } else {
+ // s32, s32, s32, s32 = G_UNMERGE_VALUES <4 x s32>
+ B.buildExtractVectorElementConstant(Dst, Src, Idx);
+ Idx += 1;
+ }
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGInsertSubvector(MachineInstr &MI, MachineIRBuilder &B) {
+ auto *MRI = B.getMRI();
+
+ auto DstReg = MI.getOperand(0).getReg();
+ auto VecReg = MI.getOperand(1).getReg();
+ auto SubVecReg = MI.getOperand(2).getReg();
+ auto Idx = MI.getOperand(3).getImm();
+
+ LLT DstTy = MRI->getType(DstReg);
+
+ if (DstTy.getScalarSizeInBits() == 64) {
+ LLT VecTy = MRI->getType(VecReg);
+ LLT SubVecTy = MRI->getType(SubVecReg);
+ LLT CastedVecTy =
+ LLT::fixed_vector(2 * VecTy.getNumElements(), LLT::integer(32));
+ LLT CastedSubVecTy =
+ LLT::fixed_vector(2 * SubVecTy.getNumElements(), LLT::integer(32));
+ LLT CastedDstTy =
+ LLT::fixed_vector(2 * DstTy.getNumElements(), LLT::integer(32));
+
+ // Update vec
+ auto CastedVecReg = MRI->createGenericVirtualRegister(CastedVecTy);
+ B.buildBitcast(CastedVecReg, VecReg);
+ MI.getOperand(1).setReg(CastedVecReg);
+
+ // Update subvec
+ auto CastedSubVecReg = MRI->createGenericVirtualRegister(CastedSubVecTy);
+ B.buildBitcast(CastedSubVecReg, SubVecReg);
+ MI.getOperand(2).setReg(CastedSubVecReg);
+
+ // Update dst
+ auto CastedDstReg = MRI->createGenericVirtualRegister(CastedDstTy);
+ MI.getOperand(0).setReg(CastedDstReg);
+
+ // Update index
+ MI.getOperand(3).setImm(2 * Idx);
+
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ B.buildBitcast(DstReg, CastedDstReg);
+ return legalizeGInsertSubvector(MI, B);
+ }
+
+ assert(DstTy.getScalarSizeInBits() == 32 && "Unexpected scalar size");
+
+ unsigned NumDstElems = DstTy.getNumElements();
+ if (NumDstElems <= 64)
+ return true; // Already legal
+
+ // Split destination vector into legal 32-element chunks
+ unsigned NumChunks = NumDstElems / 32;
+ LLT ChunkTy = LLT::vector(ElementCount::getFixed(32), DstTy.getScalarType());
+
+ SmallVector<Register, 4> Chunks;
+ for (unsigned I = 0; I < NumChunks; ++I)
+ Chunks.push_back(MRI->createGenericVirtualRegister(ChunkTy));
+
+ B.setInsertPt(*MI.getParent(), MI);
+ B.buildUnmerge(Chunks, VecReg);
+
+ // Determine which chunk the subvector belongs in
+ unsigned ChunkIdx = Idx / 32;
+ unsigned OffsetInChunk = Idx % 32;
+
+ // Per LLVM spec:
+ // "Idx must be a constant multiple of subvec’s known minimum vector length"
+ assert(ChunkIdx < Chunks.size() && "Subvector index out of bounds");
+ assert(OffsetInChunk + MRI->getType(SubVecReg).getNumElements() <= 32 &&
+ "Subvector spans multiple chunks");
+
+ // Insert subvector into the appropriate chunk
+ Register ModifiedChunk = MRI->createGenericVirtualRegister(ChunkTy);
+ B.buildInsertSubvector(ModifiedChunk, Chunks[ChunkIdx], SubVecReg,
+ OffsetInChunk);
+ Chunks[ChunkIdx] = ModifiedChunk;
+
+ // Rebuild the final vector
+ Register FinalVec = MRI->createGenericVirtualRegister(DstTy);
+ B.buildConcatVectors(FinalVec, Chunks);
+ B.buildCopy(DstReg, FinalVec);
+
+ MI.eraseFromParent();
+ return true;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGExtractSubvector(MachineInstr &MI, MachineIRBuilder &B) {
+ auto *MRI = B.getMRI();
+
+ auto DstReg = MI.getOperand(0).getReg();
+ auto SrcReg = MI.getOperand(1).getReg();
+ auto Idx = MI.getOperand(2).getImm();
+
+ LLT SrcTy = MRI->getType(SrcReg);
+
+ if (SrcTy.getScalarSizeInBits() == 64) {
+ LLT DstTy = MRI->getType(DstReg);
+ LLT CastedSrcTy =
+ LLT::fixed_vector(2 * SrcTy.getNumElements(), LLT::integer(32));
+ LLT CastedDstTy =
+ LLT::fixed_vector(2 * DstTy.getNumElements(), LLT::integer(32));
+
+ // Update src
+ auto CastedSrcReg = MRI->createGenericVirtualRegister(CastedSrcTy);
+ B.buildBitcast(CastedSrcReg, SrcReg);
+ MI.getOperand(1).setReg(CastedSrcReg);
+
+ // Update dst
+ auto CastedDstReg = MRI->createGenericVirtualRegister(CastedDstTy);
+ MI.getOperand(0).setReg(CastedDstReg);
+
+ // Update index
+ MI.getOperand(2).setImm(2 * Idx);
+
+ B.setInsertPt(B.getMBB(), ++B.getInsertPt());
+ B.buildBitcast(DstReg, CastedDstReg);
+ return legalizeGExtractSubvector(MI, B);
+ }
+
+ assert(SrcTy.getScalarSizeInBits() == 32 && "Unexpected scalar size");
+
+ unsigned NumSrcElems = SrcTy.getNumElements();
+ if (NumSrcElems <= 64)
+ return true; // Already legal
+
+ // Define 32-element legal vector type
+ unsigned NumChunks = NumSrcElems / 32;
+ LLT ChunkTy = LLT::vector(ElementCount::getFixed(32), SrcTy.getScalarType());
+
+ // Create registers for each chunk
+ SmallVector<Register, 4> Chunks;
+ for (unsigned I = 0; I < NumChunks; ++I)
+ Chunks.push_back(MRI->createGenericVirtualRegister(ChunkTy));
+
+ B.setInsertPt(*MI.getParent(), MI);
+ B.buildUnmerge(Chunks, SrcReg);
+
+ // Determine chunk index and offset
+ unsigned ChunkIdx = Idx / 32;
+ unsigned OffsetInChunk = Idx % 32;
+
+ // Per LLVM spec:
+ // "Idx must be a constant multiple of the known-minimum vector length of the
+ // result type"
+ assert(ChunkIdx < Chunks.size() && "Subvector index out of bounds");
+ assert(OffsetInChunk + MRI->getType(DstReg).getNumElements() <= 32 &&
+ "Subvector spans multiple legal vector chunks");
+
+ B.buildExtractSubvector(DstReg, Chunks[ChunkIdx], OffsetInChunk);
+ MI.eraseFromParent();
+ return true;
+}
+
+static StringRef getSyncScopeStr(LLVMContext &Ctx, SyncScope::ID ScopeID) {
+ // Map dynamically assigned PISA SyncScope ID to its scope name.
+ static DenseMap<SyncScope::ID, StringRef> ScopeID2Name;
+ auto InitializeScopeID2Name = [&]() {
+ static const StringMap<StringRef> ScopeName2EncodeName = {
+ {"workgroup", "workgroup"},
+ {"gpu", "gpu"},
+ {"system", "system"},
+ // using workgroup scope (see PISAScopeSelector pass).
+ {"subgroup", "workgroup"},
+ {"workitem", "workgroup"},
+ };
+ for (const auto &[Name, EncodeName] : ScopeName2EncodeName) {
+ auto ID = Ctx.getOrInsertSyncScopeID(Name);
+ ScopeID2Name.emplace_or_assign(ID, EncodeName);
+ }
+ };
+ static llvm::once_flag InitializeScopeID2NameFlag;
+ std::call_once(InitializeScopeID2NameFlag, InitializeScopeID2Name);
+
+ // Use the original SyncScope ID to look up its scope name in the map.
+ auto It = ScopeID2Name.find(ScopeID);
+ return It != ScopeID2Name.end() ? It->second : StringRef("gpu");
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGAtomicrmw(MachineInstr &MI, MachineIRBuilder &B) {
+ const MachineMemOperand *MemOp = *MI.memoperands_begin();
+ AtomicOrdering AO = MemOp->getSuccessOrdering();
+ AtomicOrdering AOF = MemOp->getFailureOrdering();
+ if (isAtLeastOrStrongerThan(AtomicOrdering::AcquireRelease, AO) &&
+ isAtLeastOrStrongerThan(AtomicOrdering::Release, AOF))
+ return true;
+
+ auto &Ctx = B.getMF().getFunction().getContext();
+ llvm::SmallString<16> FenceScopeStr =
+ getSyncScopeStr(Ctx, MemOp->getSyncScopeID());
+ unsigned AddressSpace = MemOp->getAddrSpace();
+ if (AddressSpace == unsigned(PISAAS::AddressSpace::SHARED))
+ FenceScopeStr += "-shared";
+ else if (AddressSpace == unsigned(PISAAS::AddressSpace::GENERIC))
+ FenceScopeStr += "-generic";
+ else
+ FenceScopeStr += "-global";
+
+ B.buildFence(
+ static_cast<unsigned>(llvm::AtomicOrdering::SequentiallyConsistent),
+ Ctx.getOrInsertSyncScopeID(FenceScopeStr));
+
+ if (!isAtLeastOrStrongerThan(AtomicOrdering::AcquireRelease, AO))
+ AO = AtomicOrdering::Monotonic;
+ if (!isAtLeastOrStrongerThan(AtomicOrdering::Release, AOF))
+ AOF = AtomicOrdering::Monotonic;
+
+ MachineMemOperand *NewMemOp = B.getMF().getMachineMemOperand(
+ MemOp->getPointerInfo(), MemOp->getFlags(), MemOp->getSize(),
+ MemOp->getAlign(),
+ MMOMetadata(MemOp->getAAInfo(), MemOp->getRanges(),
+ MemOp->getMemCacheHint()),
+ MemOp->getSyncScopeID(), AO, AOF);
+
+ if (MI.getOpcode() == TargetOpcode::G_ATOMIC_CMPXCHG)
+ B.buildAtomicCmpXchg(MI.getOperand(0), MI.getOperand(1), MI.getOperand(2),
+ MI.getOperand(3), *NewMemOp)
+ .setMIFlags(MI.getFlags());
+ else
+ B.buildAtomicRMW(MI.getOpcode(), MI.getOperand(0), MI.getOperand(1),
+ MI.getOperand(2), *NewMemOp)
+ .setMIFlags(MI.getFlags());
+
+ B.buildFence(
+ static_cast<unsigned>(llvm::AtomicOrdering::SequentiallyConsistent),
+ Ctx.getOrInsertSyncScopeID(FenceScopeStr));
+
+ MI.eraseFromParent();
+ return true;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGAtomicrmwXchg(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ auto &Dst = MI.getOperand(0);
+ LLT CurTy = MRI.getType(Dst.getReg());
+ if (CurTy.getScalarType().isPointer()) {
+ auto &Src = MI.getOperand(2);
+ LLT NewTy = LLT::integer(CurTy.getScalarSizeInBits());
+ Register NewSrc = MRI.createGenericVirtualRegister(NewTy);
+ Register NewDst = MRI.createGenericVirtualRegister(NewTy);
+ B.buildPtrToInt(NewSrc, Src);
+ B.buildAtomicRMWXchg(NewDst, MI.getOperand(1).getReg(), NewSrc,
+ **MI.memoperands_begin());
+ B.buildIntToPtr(Dst, NewDst);
+ MI.eraseFromParent();
+ } else {
+ legalizeGAtomicrmw(MI, B);
+ }
+ return true;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGShuffleVector(LegalizerHelper &Helper, MachineInstr &MI,
+ MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ auto &Dst = MI.getOperand(0);
+ auto &Src0 = MI.getOperand(1);
+ auto &Src1 = MI.getOperand(2);
+ ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
+ assert(MRI.getType(Dst.getReg()).getScalarSizeInBits() == 32);
+ assert(isPowerOf2_32(MRI.getType(Dst.getReg()).getNumElements()));
+
+ auto UseExtract = true;
+ // indices must be consecutive
+ int PrevIdx = -1;
+ for (int Idx : Mask) {
+ if ((PrevIdx != -1) && (Idx != (PrevIdx + 1)))
+ UseExtract = false;
+ PrevIdx = Idx;
+ }
+
+ // starting index must be aligned to destination size
+ if (Mask[0] % Mask.size())
+ UseExtract = false;
+
+ // indices can not straddle the arguments
+ auto SrcSize = MRI.getType(Src0.getReg()).getNumElements();
+ if ((Mask[0] < SrcSize) && ((Mask[0] + Mask.size()) > SrcSize))
+ UseExtract = false;
+
+ if (SrcSize > 8 && SrcSize != 16 && SrcSize != 32 && (SrcSize % 64))
+ UseExtract = false;
+
+ if (UseExtract) {
+ auto Src = (Mask[0] < SrcSize) ? Src0 : Src1;
+ auto Idx = (Mask[0] < SrcSize) ? Mask[0] : Mask[0] - SrcSize;
+ B.buildExtractSubvector(Dst, Src, Idx);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // lower if unable to use extract/insert
+ auto Res = Helper.lowerShuffleVector(MI);
+ return Res != LegalizerHelper::UnableToLegalize;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
+ MachineIRBuilder &MIRBuilder) {
+ auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ FPClassTest OriginalMask =
+ static_cast<FPClassTest>(MI.getOperand(2).getImm());
+ auto Mask = OriginalMask;
+ auto IsInvertedCheck = false;
+
+ if (Mask == fcNone) {
+ MIRBuilder.buildConstant(DstReg, 0);
+ MI.eraseFromParent();
+ return true;
+ }
+ if (Mask == fcAllFlags) {
+ MIRBuilder.buildConstant(DstReg, 1);
+ MI.eraseFromParent();
+ return true;
+ }
+
+ // support bfloat types
+ auto &Semantics = getFltSemanticForLLT(SrcTy.getScalarType());
+
+ unsigned BitSize = SrcTy.getScalarSizeInBits();
+ LLT IntTy = LLT::integer(BitSize);
+ if (SrcTy.isVector())
+ IntTy = LLT::vector(SrcTy.getElementCount(), IntTy);
+ auto AsInt = MIRBuilder.buildBitcast(IntTy, SrcReg);
+
+ // Various masks.
+ APInt SignBit = APInt::getSignMask(BitSize);
+ APInt ValueMask = APInt::getSignedMaxValue(BitSize); // All bits but sign.
+ APInt Inf = APFloat::getInf(Semantics).bitcastToAPInt(); // Exp and int bit.
+ APInt ExpMask = Inf;
+ APInt AllOneMantissa = APFloat::getLargest(Semantics).bitcastToAPInt() & ~Inf;
+ APInt QNaNBitMask =
+ APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);
+ APInt InvertionMask = APInt::getAllOnes(DstTy.getScalarSizeInBits());
+
+ auto SignBitC = MIRBuilder.buildConstant(IntTy, SignBit);
+ auto ValueMaskC = MIRBuilder.buildConstant(IntTy, ValueMask);
+ auto InfC = MIRBuilder.buildConstant(IntTy, Inf);
+ auto ExpMaskC = MIRBuilder.buildConstant(IntTy, ExpMask);
+ auto ZeroC = MIRBuilder.buildConstant(IntTy, 0);
+
+ auto Abs = MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC);
+ auto Sign =
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, AsInt, Abs);
+
+ auto Res = MIRBuilder.buildConstant(DstTy, 0);
+ // Clang doesn't support capture of structured bindings:
+ LLT DstTyCopy = DstTy;
+ const auto AppendToRes = [&](MachineInstrBuilder ToAppend) {
+ Res = MIRBuilder.buildOr(DstTyCopy, Res, ToAppend);
+ };
+
+ // Tests that involve more than one class should be processed first.
+ if ((Mask & fcFinite) == fcFinite) {
+ // finite(V) ==> abs(V) u< exp_mask
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs,
+ ExpMaskC));
+ Mask &= ~fcFinite;
+ } else if ((Mask & fcFinite) == fcPosFinite) {
+ // finite(V) && V > 0 ==> V u< exp_mask
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, AsInt,
+ ExpMaskC));
+ Mask &= ~fcPosFinite;
+ } else if ((Mask & fcFinite) == fcNegFinite) {
+ // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
+ auto Cmp = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs,
+ ExpMaskC);
+ auto And = MIRBuilder.buildAnd(DstTy, Cmp, Sign);
+ AppendToRes(And);
+ Mask &= ~fcNegFinite;
+ }
+
+ if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
+ // fcZero | fcSubnormal => test all exponent bits are 0
+ // TODO: Handle sign bit specific cases
+ // TODO: Handle inverted case
+ if (PartialCheck == (fcZero | fcSubnormal)) {
+ auto ExpBits = MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC);
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
+ ExpBits, ZeroC));
+ Mask &= ~PartialCheck;
+ }
+ }
+
+ if (Mask == OriginalMask) {
+ // combination of classes above did not yield any
+ // optimizations, see if inverse will be less ops
+ auto InvertedMask = (unsigned)~Mask;
+ if (llvm::popcount((unsigned)Mask) > llvm::popcount(InvertedMask)) {
+ Mask = ~Mask;
+ IsInvertedCheck = true;
+ }
+ }
+
+ // Check for individual classes.
+ if (FPClassTest PartialCheck = Mask & fcZero) {
+ if (PartialCheck == fcPosZero)
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
+ AsInt, ZeroC));
+ else if (PartialCheck == fcZero)
+ AppendToRes(
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, ZeroC));
+ else // fcNegZero
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
+ AsInt, SignBitC));
+ }
+
+ if (FPClassTest PartialCheck = Mask & fcSubnormal) {
+ // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
+ // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
+ auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
+ auto OneC = MIRBuilder.buildConstant(IntTy, 1);
+ auto VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC);
+ auto SubnormalRes =
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, VMinusOne,
+ MIRBuilder.buildConstant(IntTy, AllOneMantissa));
+ if (PartialCheck == fcNegSubnormal)
+ SubnormalRes = MIRBuilder.buildAnd(DstTy, SubnormalRes, Sign);
+ AppendToRes(SubnormalRes);
+ }
+
+ if (FPClassTest PartialCheck = Mask & fcInf) {
+ if (PartialCheck == fcPosInf)
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
+ AsInt, InfC));
+ else if (PartialCheck == fcInf)
+ AppendToRes(
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, InfC));
+ else { // fcNegInf
+ APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();
+ auto NegInfC = MIRBuilder.buildConstant(IntTy, NegInf);
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
+ AsInt, NegInfC));
+ }
+ }
+
+ if (FPClassTest PartialCheck = Mask & fcNan) {
+ auto InfWithQnanBitC =
+ MIRBuilder.buildConstant(IntTy, std::move(Inf) | QNaNBitMask);
+ if (PartialCheck == fcNan) {
+ // isnan(V) ==> abs(V) u> int(inf)
+ AppendToRes(
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC));
+ } else if (PartialCheck == fcQNan) {
+ // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
+ AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGE, DstTy, Abs,
+ InfWithQnanBitC));
+ } else { // fcSNan
+ // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
+ // abs(V) u< (unsigned(Inf) | quiet_bit)
+ auto IsNan =
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC);
+ auto IsNotQnan = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy,
+ Abs, InfWithQnanBitC);
+ AppendToRes(MIRBuilder.buildAnd(DstTy, IsNan, IsNotQnan));
+ }
+ }
+
+ if (FPClassTest PartialCheck = Mask & fcNormal) {
+ // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
+ // (max_exp-1))
+ APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));
+ auto ExpMinusOne = MIRBuilder.buildSub(
+ IntTy, Abs, MIRBuilder.buildConstant(IntTy, ExpLSB));
+ APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
+ auto NormalRes =
+ MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, ExpMinusOne,
+ MIRBuilder.buildConstant(IntTy, MaxExpMinusOne));
+ if (PartialCheck == fcNegNormal)
+ NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, Sign);
+ else if (PartialCheck == fcPosNormal) {
+ auto PosSign = MIRBuilder.buildXor(
+ DstTy, Sign, MIRBuilder.buildConstant(DstTy, InvertionMask));
+ NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, PosSign);
+ }
+ AppendToRes(NormalRes);
+ }
+
+ if (IsInvertedCheck)
+ MIRBuilder.buildNot(DstReg, Res);
+ else
+ MIRBuilder.buildCopy(DstReg, Res);
+ MI.eraseFromParent();
+ return true;
+}
+
+static bool legalizeGMulh(LegalizerHelper &Helper, MachineInstr &MI,
+ MachineIRBuilder &B) {
+ auto &MRI = *B.getMRI();
+ auto [Dst, Src0, Src1] = MI.getFirst3Regs();
+ auto IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
+ auto DstTy = MRI.getType(Dst);
+ assert(DstTy.getSizeInBits() == 64);
+
+ auto SourceA = MRI.createGenericVirtualRegister(DstTy);
+ auto SourceB = MRI.createGenericVirtualRegister(DstTy);
+
+ auto Const32 = MRI.createGenericVirtualRegister(DstTy);
+ auto Const63 = MRI.createGenericVirtualRegister(DstTy);
+ auto Const0 = MRI.createGenericVirtualRegister(DstTy);
+ auto Mask32 = MRI.createGenericVirtualRegister(DstTy);
+ B.buildConstant(Const32, 32);
+ B.buildConstant(Const63, 63);
+ B.buildConstant(Const0, 0);
+ B.buildConstant(Mask32, 0xFFFFFFFF);
+
+ auto ASign = MRI.createGenericVirtualRegister(DstTy);
+ auto BSign = MRI.createGenericVirtualRegister(DstTy);
+ auto ResultSign = MRI.createGenericVirtualRegister(DstTy);
+ B.buildAShr(ASign, Src0, Const63);
+ B.buildAShr(BSign, Src1, Const63);
+ B.buildXor(ResultSign, ASign, BSign);
+
+ if (IsSigned) {
+ auto ASignXor = MRI.createGenericVirtualRegister(DstTy);
+ auto BSignXor = MRI.createGenericVirtualRegister(DstTy);
+ B.buildXor(ASignXor, Src0, ASign);
+ B.buildXor(BSignXor, Src1, BSign);
+ B.buildSub(SourceA, ASignXor, ASign);
+ B.buildSub(SourceB, BSignXor, BSign);
+ } else {
+ B.buildCopy(SourceA, Src0);
+ B.buildCopy(SourceB, Src1);
+ }
+
+ auto LoSrc0 = MRI.createGenericVirtualRegister(DstTy);
+ auto HiSrc0 = MRI.createGenericVirtualRegister(DstTy);
+ auto LoSrc1 = MRI.createGenericVirtualRegister(DstTy);
+ auto HiSrc1 = MRI.createGenericVirtualRegister(DstTy);
+ B.buildLShr(HiSrc0, SourceA, Const32);
+ B.buildLShr(HiSrc1, SourceB, Const32);
+ B.buildAnd(LoSrc0, SourceA, Mask32);
+ B.buildAnd(LoSrc1, SourceB, Mask32);
+
+ auto ALobLo = MRI.createGenericVirtualRegister(DstTy);
+ auto ALobHi = MRI.createGenericVirtualRegister(DstTy);
+ auto AHibLo = MRI.createGenericVirtualRegister(DstTy);
+ auto AHibHi = MRI.createGenericVirtualRegister(DstTy);
+ B.buildMul(AHibHi, HiSrc0, HiSrc1);
+ B.buildMul(AHibLo, HiSrc0, LoSrc1);
+ B.buildMul(ALobHi, LoSrc0, HiSrc1);
+ B.buildMul(ALobLo, LoSrc0, LoSrc1);
+
+ auto ALobLoHi = MRI.createGenericVirtualRegister(DstTy);
+ auto ALobHiLo = MRI.createGenericVirtualRegister(DstTy);
+ auto AHibLoSum0 = MRI.createGenericVirtualRegister(DstTy);
+ auto AHibLoSum1 = MRI.createGenericVirtualRegister(DstTy);
+ B.buildLShr(ALobLoHi, ALobLo, Const32);
+ B.buildAnd(ALobHiLo, ALobHi, Mask32);
+ B.buildAdd(AHibLoSum0, ALobLoHi, ALobHiLo);
+ B.buildAdd(AHibLoSum1, AHibLo, AHibLoSum0);
+
+ auto ALobLoMasked = MRI.createGenericVirtualRegister(DstTy);
+ auto AHibLoShiftedL = MRI.createGenericVirtualRegister(DstTy);
+ auto ALobHiShiftedR = MRI.createGenericVirtualRegister(DstTy);
+ auto AHibLoShiftedR = MRI.createGenericVirtualRegister(DstTy);
+ auto ShiftedSum = MRI.createGenericVirtualRegister(DstTy);
+ auto DstLo = MRI.createGenericVirtualRegister(DstTy);
+ auto DstHi = MRI.createGenericVirtualRegister(DstTy);
+
+ B.buildAnd(ALobLoMasked, ALobLo, Mask32);
+ B.buildShl(AHibLoShiftedL, AHibLoSum1, Const32);
+ B.buildOr(DstLo, AHibLoShiftedL, ALobLoMasked);
+ B.buildLShr(ALobHiShiftedR, ALobHi, Const32);
+ B.buildLShr(AHibLoShiftedR, AHibLoSum1, Const32);
+ B.buildAdd(ShiftedSum, ALobHiShiftedR, AHibLoShiftedR);
+ B.buildAdd(DstHi, AHibHi, ShiftedSum);
+
+ if (IsSigned) {
+ // ulong mask = -resultSign;
+ // hi = hi ^ mask;
+ // lo = lo ^ mask;
+ // lo += resultSign; // Add 1 if resultSign is negative, otherwise add 0
+ // hi += (lo < resultSign); // Adjust hi if lo overflowed
+ auto Mask = MRI.createGenericVirtualRegister(DstTy);
+ B.buildNeg(Mask, ResultSign);
+ auto HiXorMask = MRI.createGenericVirtualRegister(DstTy);
+ auto LoXorMask = MRI.createGenericVirtualRegister(DstTy);
+ B.buildXor(HiXorMask, DstHi, Mask);
+ B.buildXor(LoXorMask, DstLo, Mask);
+
+ auto LoAddResult = MRI.createGenericVirtualRegister(DstTy);
+ auto LoAddShiftResult = MRI.createGenericVirtualRegister(DstTy);
+ B.buildAdd(LoAddResult, LoXorMask, ResultSign);
+ B.buildShl(LoAddShiftResult, LoAddResult, ResultSign);
+ B.buildAdd(Dst, HiXorMask, LoAddShiftResult);
+ } else {
+ B.buildCopy(Dst, DstHi);
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+// Legalizes an addrspacecast operation between pointers in non-generic address
+// spaces. Ensures that null pointers are preserved during the cast by replacing
+// the addrspacecast with a null pointer of the destination type.
+static bool legalizeGAddrspaceCast(MachineInstr &MI, MachineIRBuilder &B) {
+ auto &Dst = MI.getOperand(0);
+ LLT DstTy = B.getMRI()->getType(Dst.getReg());
+ B.buildConstant(
+ Dst, PISATargetMachine::getNullPointerValue(DstTy.getAddressSpace()));
+ MI.eraseFromParent();
+ return true;
+}
+
+// NOLINTNEXTLINE(readability-identifier-naming)
+static bool legalizeGBitcast(MachineInstr &MI, MachineIRBuilder &B) {
+ auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ auto *MRI = B.getMRI();
+
+ assert(SrcTy.getSizeInBits() == DstTy.getSizeInBits());
+ if (SrcTy.isPointer() != DstTy.isPointer()) {
+ // legalize bitcast between pointers and non-pointers
+ if (SrcTy.isPointer()) {
+ // <2 x i32> G_BITCAST (p1)
+ auto IntSize = SrcTy.getSizeInBits();
+ auto IntTy = LLT::integer(IntSize);
+ auto IntReg = MRI->createGenericVirtualRegister(IntTy);
+ B.buildPtrToInt(IntReg, SrcReg);
+ if (IntTy == DstTy)
+ B.buildCopy(DstReg, IntReg);
+ else
+ B.buildBitcast(DstReg, IntReg);
+ } else {
+ // (p1) G_BITCAST <2 x i32>
+ auto IntSize = DstTy.getSizeInBits();
+ auto IntTy = LLT::integer(IntSize);
+ auto IntReg = MRI->createGenericVirtualRegister(IntTy);
+ if (IntTy == SrcTy)
+ B.buildCopy(IntReg, SrcReg);
+ else
+ B.buildBitcast(IntReg, SrcReg);
+ B.buildIntToPtr(DstReg, IntReg);
+ }
+ } else {
+ // Legalizes a bitcast operation between vector types by decomposing the
+ // source vector into scalar elements and reassembling them into the
+ // destination vector type.
+ assert(SrcTy.isVector() && DstTy.isVector());
+ unsigned SrcScalarSize = SrcTy.getScalarSizeInBits();
+ unsigned DstScalarSize = DstTy.getScalarSizeInBits();
+
+ if (SrcScalarSize == DstScalarSize) {
+ // Same scalar size (e.g. <5 x f16> to <5 x i16>): extract each element
+ // and reinterpret as the destination scalar type.
+ SmallVector<Register, 4> DstElements;
+ for (unsigned I = 0; I < SrcTy.getNumElements(); I++) {
+ auto SrcElemReg =
+ MRI->createGenericVirtualRegister(SrcTy.getScalarType());
+ B.buildExtractVectorElementConstant(SrcElemReg, SrcReg, I);
+ auto DstElemReg =
+ MRI->createGenericVirtualRegister(DstTy.getScalarType());
+ B.buildBitcast(DstElemReg, SrcElemReg);
+ DstElements.push_back(DstElemReg);
+ }
+ B.buildBuildVector(DstReg, DstElements);
+ } else {
+ // Different scalar sizes with non-divisible element counts: decompose
+ // via GCD-sized pieces.
+ unsigned CommonPieceSize = std::gcd(DstScalarSize, SrcScalarSize);
+ SmallVector<Register, 4> ScalarPieces;
+ for (unsigned SrcElemIdx = 0; SrcElemIdx < SrcTy.getNumElements();
+ SrcElemIdx++) {
+ auto SrcElemReg =
+ MRI->createGenericVirtualRegister(SrcTy.getScalarType());
+ B.buildExtractVectorElementConstant(SrcElemReg, SrcReg, SrcElemIdx);
+ auto Unmerge =
+ B.buildUnmerge(LLT::integer(CommonPieceSize), SrcElemReg);
+ for (unsigned PieceIdx = 0; PieceIdx != Unmerge->getNumOperands() - 1;
+ PieceIdx++)
+ ScalarPieces.push_back(Unmerge.getReg(PieceIdx));
+ }
+ unsigned NumPiecesPerDstElem = DstScalarSize / CommonPieceSize;
+ SmallVector<Register, 4> DstElements;
+ for (unsigned PieceStartIdx = 0; PieceStartIdx < ScalarPieces.size();
+ PieceStartIdx += NumPiecesPerDstElem) {
+ auto DstElemReg =
+ MRI->createGenericVirtualRegister(DstTy.getScalarType());
+ SmallVector<Register> DstElemPieces(
+ ScalarPieces.begin() + PieceStartIdx,
+ ScalarPieces.begin() + PieceStartIdx + NumPiecesPerDstElem);
+ B.buildMergeValues(DstElemReg, DstElemPieces);
+ DstElements.push_back(DstElemReg);
+ }
+ B.buildBuildVector(DstReg, DstElements);
+ }
+ }
+ MI.eraseFromParent();
+ return true;
+}
+
+bool PISALegalizerInfo::legalizeCustom(
+ LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const {
+ MachineIRBuilder &B = Helper.MIRBuilder;
+ switch (MI.getOpcode()) {
+ case TargetOpcode::G_FLOG:
+ return legalizeGFlog(MI, B, numbers::ln2f);
+ case TargetOpcode::G_FLOG10:
+ return legalizeGFlog(MI, B, numbers::ln2f / numbers::ln10f);
+ case TargetOpcode::G_FEXP:
+ return legalizeGFexp(MI, B, numbers::log2e);
+ case TargetOpcode::G_FEXP10:
+ return legalizeGFexp(MI, B, numbers::ln10f / numbers::ln2f);
+ case TargetOpcode::G_FCMP:
+ return legalizeGFcmp(MI, B);
+ case TargetOpcode::G_TRUNC:
+ return legalizeGTrunc(MI, B);
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ANYEXT:
+ return legalizeGExt(MI, B);
+ case TargetOpcode::G_SITOFP:
+ case TargetOpcode::G_UITOFP:
+ return legalizeGItofp(MI, B);
+ case TargetOpcode::G_STORE:
+ case TargetOpcode::G_LOAD:
+ return legalizeGLoad(MI, B, Helper);
+ case TargetOpcode::G_SEXTLOAD:
+ case TargetOpcode::G_ZEXTLOAD:
+ return legalizeGExtload(MI, B);
+ case TargetOpcode::G_FDIV:
+ return legalizeGFdiv(MI, B);
+ case TargetOpcode::G_FREM:
+ return legalizeGFrem(MI, B);
+ case TargetOpcode::G_INSERT_VECTOR_ELT:
+ return legalizeGInsertVectorElt(Helper, MI, B);
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT:
+ return legalizeGExtractVectorElt(Helper, MI, B);
+ case TargetOpcode::G_INSERT_SUBVECTOR:
+ return legalizeGInsertSubvector(MI, B);
+ case TargetOpcode::G_EXTRACT_SUBVECTOR:
+ return legalizeGExtractSubvector(MI, B);
+ case TargetOpcode::G_CONCAT_VECTORS:
+ return legalizeGConcatVectors(MI, B);
+ case TargetOpcode::G_UNMERGE_VALUES:
+ return legalizeGUnmergeValues(Helper, MI, B);
+ case TargetOpcode::G_BSWAP:
+ return legalizeGBswap(MI, B);
+ case TargetOpcode::G_FPOW:
+ return legalizeGFpow(MI, B);
+ case TargetOpcode::G_FLDEXP:
+ case TargetOpcode::G_STRICT_FLDEXP:
+ return legalizeGFldexp(MI, B);
+ case TargetOpcode::G_ATOMICRMW_XCHG:
+ return legalizeGAtomicrmwXchg(MI, B);
+ case TargetOpcode::G_SHUFFLE_VECTOR:
+ return legalizeGShuffleVector(Helper, MI, B);
+ case TargetOpcode::G_IS_FPCLASS:
+ return legalizeGIsFpclass(Helper, MI, B);
+ case TargetOpcode::G_UMULH:
+ case TargetOpcode::G_SMULH:
+ return legalizeGMulh(Helper, MI, B);
+ case TargetOpcode::G_ADDRSPACE_CAST:
+ return legalizeGAddrspaceCast(MI, B);
+ case TargetOpcode::G_BITCAST:
+ return legalizeGBitcast(MI, B);
+ case TargetOpcode::G_ATOMIC_CMPXCHG:
+ case TargetOpcode::G_ATOMICRMW_ADD:
+ case TargetOpcode::G_ATOMICRMW_SUB:
+ case TargetOpcode::G_ATOMICRMW_AND:
+ case TargetOpcode::G_ATOMICRMW_OR:
+ case TargetOpcode::G_ATOMICRMW_XOR:
+ case TargetOpcode::G_ATOMICRMW_MIN:
+ case TargetOpcode::G_ATOMICRMW_MAX:
+ case TargetOpcode::G_ATOMICRMW_UMIN:
+ case TargetOpcode::G_ATOMICRMW_UMAX:
+ case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
+ case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
+ case TargetOpcode::G_ATOMICRMW_FADD:
+ case TargetOpcode::G_ATOMICRMW_FSUB:
+ case TargetOpcode::G_ATOMICRMW_FMIN:
+ case TargetOpcode::G_ATOMICRMW_FMAX:
+ return legalizeGAtomicrmw(MI, B);
+ case TargetOpcode::G_FABS:
+ return legalizeFAbs(MI, B);
+ }
+ assert(0 && "unhandled!");
+ return false;
+}
diff --git a/llvm/lib/Target/PISA/PISALegalizerInfo.h b/llvm/lib/Target/PISA/PISALegalizerInfo.h
new file mode 100644
index 0000000000000..53c8fb6b8b04b
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISALegalizerInfo.h
@@ -0,0 +1,30 @@
+//===-- PISALegalizerInfo.h --- PISA Legalization Rules -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISALEGALIZERINFO_H
+#define LLVM_LIB_TARGET_PISA_PISALEGALIZERINFO_H
+
+#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
+
+namespace llvm {
+
+class LLVMContext;
+class PISASubtarget;
+
+// This class provides the information for legalizing PISA instructions.
+class PISALegalizerInfo : public LegalizerInfo {
+
+public:
+ bool legalizeIntrinsic(LegalizerHelper &Helper,
+ MachineInstr &MI) const override;
+ bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
+ LostDebugLocObserver &LocObserver) const override;
+ PISALegalizerInfo(const PISASubtarget &ST);
+};
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_PISA_PISALEGALIZERINFO_H
diff --git a/llvm/lib/Target/PISA/PISAMachineFunctionInfo.cpp b/llvm/lib/Target/PISA/PISAMachineFunctionInfo.cpp
new file mode 100644
index 0000000000000..290f8a15d182b
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAMachineFunctionInfo.cpp
@@ -0,0 +1,17 @@
+//===-- PISAMachineFunctionInfo.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAMachineFunctionInfo.h"
+#include "PISASubtarget.h"
+
+using namespace llvm;
+
+PISAMachineFunctionInfo::PISAMachineFunctionInfo(const Function &F,
+ const PISASubtarget *STI) {}
+
+PISAMachineFunctionInfo::~PISAMachineFunctionInfo() = default;
diff --git a/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h b/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h
new file mode 100644
index 0000000000000..34c3a88176984
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h
@@ -0,0 +1,39 @@
+//===-- PISAMachineFunctionInfo.h -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAMACHINEFUNCTIONINFO_H
+#define LLVM_LIB_TARGET_PISA_PISAMACHINEFUNCTIONINFO_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/MachineFunction.h"
+
+namespace llvm {
+class PISASubtarget;
+
+class PISAMachineFunctionInfo : public MachineFunctionInfo {
+ using ArgInfo = std::pair<unsigned, bool>;
+ DenseMap<unsigned, ArgInfo> ArgInfos;
+
+public:
+ PISAMachineFunctionInfo(const Function &F, const PISASubtarget *STI);
+ ~PISAMachineFunctionInfo() override;
+
+ ArgInfo getArgInfo(unsigned Slot) const {
+ auto I = ArgInfos.find(Slot);
+ if (I == ArgInfos.end())
+ return {0, false};
+ return I->second;
+ }
+ void setArgInfo(unsigned Slot, unsigned ArgSize, bool IsByRef) {
+ ArgInfos[Slot] = {ArgSize, IsByRef};
+ }
+};
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISAMACHINEFUNCTIONINFO_H
diff --git a/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp b/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
new file mode 100644
index 0000000000000..b7ab3b214d7ee
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
@@ -0,0 +1,2005 @@
+//===-- lib/CodeGen/GlobalISel/PISAPostLegalizerCombiner.cpp --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "PISA.h"
+#include "PISALegalizerInfo.h"
+#include "PISATargetMachine.h"
+#include "PISAUtils.h"
+#include "llvm/CodeGen/GlobalISel/Combiner.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/IR/PISAIntrinsicUtils.h"
+#include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define GET_GICOMBINER_DEPS
+#include "PISAGenPostLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_DEPS
+
+#define DEBUG_TYPE "pisa-postlegalizer-combiner"
+
+using namespace llvm;
+using namespace llvm::MIPatternMatch;
+
+namespace {
+
+#define GET_GICOMBINER_TYPES
+#include "PISAGenPostLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_TYPES
+
+// integer types
+constexpr ElementCount EC0 = ElementCount::getFixed(0);
+constexpr LLT I16 = LLT(LLT::Kind::INTEGER, EC0, 16);
+constexpr LLT I32 = LLT(LLT::Kind::INTEGER, EC0, 32);
+
+class PISAPostLegalizerCombinerImpl : public Combiner {
+protected:
+ const PISAPostLegalizerCombinerImplRuleConfig &RuleConfig;
+ const PISASubtarget &STI;
+
+ // TODO: Make CombinerHelper methods const.
+ mutable CombinerHelper Helper;
+
+public:
+ PISAPostLegalizerCombinerImpl(
+ MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB,
+ GISelCSEInfo *CSEInfo,
+ const PISAPostLegalizerCombinerImplRuleConfig &RuleConfig,
+ const PISASubtarget &STI, MachineDominatorTree *MDT,
+ const LegalizerInfo *LI);
+
+ static const char *getName() { return "PISAGenPostLegalizeGICombiner"; }
+
+ bool tryCombineAllImpl(MachineInstr &MI) const;
+ bool tryCombineAll(MachineInstr &I) const override;
+
+ void applyBuildVectorWithConstants(MachineInstr &MI) const;
+
+ bool matchCompareSelect(MachineInstr *MI) const;
+ void applyCompareSelect(MachineInstr *MI) const;
+
+ bool
+ matchFCmpInvertedCond(MachineInstr &MI,
+ std::tuple<MachineInstr *, Register> &MatchInfo) const;
+ void
+ applyFCmpInvertedCond(MachineInstr &MI,
+ std::tuple<MachineInstr *, Register> &MatchInfo) const;
+
+ bool
+ matchShlAddToMad(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool
+ matchFDivToRcpFMul(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+ void applyShiftOfConstants(MachineInstr *MI) const;
+
+ bool matchRedundantMovesPost(MachineInstr &MI) const;
+ void applyRedundantMovesPost(MachineInstr &MI) const;
+
+ bool matchExtractAllToBuildVector(MachineInstr &MI,
+ Register &Replacement) const;
+ void applyExtractAllToBuildVector(MachineInstr &MI,
+ Register Replacement) const;
+ bool
+ matchOrAndToBfi(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+ bool
+ matchShiftSubToBfe(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchAndBitfieldToBitfield(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchV2i1ZextToBfeBfi(MachineInstr &BuildVectorMI,
+ Register &BitcastInput) const;
+ void applyV2i1ZextToBfeBfi(MachineInstr &MI, Register BitcastInput) const;
+
+ bool matchAndSelect(MachineInstr &MI, Register &Replacement) const;
+ void applyAndSelect(MachineInstr &MI, Register Replacement) const;
+
+ bool matchCmpAndAllOnes(MachineInstr &MI, GISelValueTracking *VT,
+ Register &MatchInfo) const;
+ void applyCmpAndAllOnes(MachineInstr &MI, Register &MatchInfo) const;
+
+ // Fix G_SHL/G_LSHR/G_ASHR where the shift amount is not i32.
+ // PISA legalizes shifts only as {I16,I32}, {I32,I32}, {I64,I32}; rules like
+ // mul_to_shl can introduce a shift with the value type as the amount type.
+ bool matchFixIllegalShiftAmt(MachineInstr &MI) const;
+ void applyFixIllegalShiftAmt(MachineInstr &MI) const;
+
+ bool matchSinkTrunc(MachineInstr &MI,
+ std::tuple<Register, int64_t> &MatchInfo) const;
+ void applySinkTrunc(MachineInstr &MI,
+ std::tuple<Register, int64_t> &MatchInfo) const;
+
+ bool matchTruncTrunc(MachineInstr &MI, Register &MatchInfo) const;
+ void applyTruncTrunc(MachineInstr &MI, Register &MatchInfo) const;
+
+ bool matchAbsRedMaxToRedAbsMax(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ struct Reg2MatchInfo {
+ Register Reg0, Reg1;
+ };
+ bool matchBuildRegFrom2(MachineInstr &MI, Reg2MatchInfo &MatchInfo) const;
+ void applyBuildRegFrom2(MachineInstr &MI, Reg2MatchInfo &MatchInfo) const;
+
+ bool matchBuildVectorFromUnmergeLanes(
+ MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const;
+ void applyBuildVectorFromUnmergeLanes(
+ MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const;
+
+ bool
+ matchBuildVectorConcatSubvectors(MachineInstr &MI,
+ SmallVector<Register, 4> &MatchInfo) const;
+ void
+ applyBuildVectorConcatSubvectors(MachineInstr &MI,
+ SmallVector<Register, 4> &MatchInfo) const;
+
+ bool
+ matchShiftTrueFalse(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchAddInt8Reduction(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchUnmergeBitcastBuildVectorToBitcast(MachineInstr &MI,
+ Register &MatchInfo) const;
+ void applyUnmergeBitcastBuildVectorToBitcast(MachineInstr &MI,
+ Register &MatchInfo) const;
+
+ bool matchRedundantFence(MachineInstr &MI, MachineInstr *&PrevFence) const;
+ void applyRedundantFence(MachineInstr &MI, MachineInstr *&PrevFence) const;
+
+ bool matchMinMaxIdentityFold(MachineInstr &MI, Register &MatchInfo) const;
+
+ bool
+ matchExtractSubvectorBuildVector(MachineInstr &MI,
+ SmallVector<Register, 8> &MatchInfo) const;
+ void
+ applyExtractSubvectorBuildVector(MachineInstr &MI,
+ SmallVector<Register, 8> &MatchInfo) const;
+
+ bool matchMergeAdjacentFences(MachineInstr &MI,
+ MachineInstr *&PrevFence) const;
+ void applyMergeAdjacentFences(MachineInstr &MI,
+ MachineInstr *&PrevFence) const;
+
+ bool
+ matchExtractSubvectorPartial(MachineInstr &MI,
+ SmallVector<MachineInstr *, 8> &MatchInfo) const;
+ void applyExtractSubvectorPartial(
+ MachineInstr &MI, const SmallVector<MachineInstr *, 8> &MatchInfo) const;
+
+private:
+#define GET_GICOMBINER_CLASS_MEMBERS
+#include "PISAGenPostLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_CLASS_MEMBERS
+};
+
+#define GET_GICOMBINER_IMPL
+#include "PISAGenPostLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_IMPL
+
+PISAPostLegalizerCombinerImpl::PISAPostLegalizerCombinerImpl(
+ MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB,
+ GISelCSEInfo *CSEInfo,
+ const PISAPostLegalizerCombinerImplRuleConfig &RuleConfig,
+ const PISASubtarget &STI, MachineDominatorTree *MDT,
+ const LegalizerInfo *LI)
+ : Combiner(MF, CInfo, &KB, CSEInfo), RuleConfig(RuleConfig), STI(STI),
+ Helper(Observer, B, /*IsPreLegalize=*/false, &KB, MDT, LI),
+#define GET_GICOMBINER_CONSTRUCTOR_INITS
+#include "PISAGenPostLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_CONSTRUCTOR_INITS
+{
+}
+
+bool PISAPostLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
+ return tryCombineAllImpl(MI);
+}
+
+void PISAPostLegalizerCombinerImpl::applyBuildVectorWithConstants(
+ MachineInstr &MI) const {
+ auto EltSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
+ auto Size = EltSize * (MI.getNumOperands() - 1);
+ assert((Size <= 64) && "vector size too large");
+
+ auto NewReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
+ uint64_t Value = 0;
+
+ for (unsigned I = MI.getNumOperands() - 1; I > 0; I--) {
+ auto Reg = MI.getOperand(I).getReg();
+ auto CValue = getAnyConstantVRegValWithLookThrough(Reg, MRI);
+ assert(CValue.has_value() && "expected const vreg val");
+ APInt IValue = CValue->Value;
+ Value <<= EltSize;
+ Value |= IValue.getZExtValue();
+ }
+
+ B.buildConstant(NewReg, Value);
+ B.buildBitcast(MI.getOperand(0), NewReg);
+ MI.eraseFromParent();
+}
+
+// i1 C = G_CMP ne i? A, 0
+// i? S = G_SELECT i1 C, i? LHS, i? RHS
+// => S = sel.? LHS, RHS, A
+// ... or ...
+// i1 C = G_CMP eq i? A, 0
+// i? S = G_SELECT i1 C, i? LHS, i? RHS
+// => S = sel.? RHS, LHS, A
+bool PISAPostLegalizerCombinerImpl::matchCompareSelect(MachineInstr *MI) const {
+ auto *SelectMI = MI;
+ auto *CmpMI = getDefIgnoringCopies(SelectMI->getOperand(1).getReg(), MRI);
+ if (CmpMI->getOpcode() == TargetOpcode::G_ICMP) {
+ auto CC = CmpMI->getOperand(1).getPredicate();
+ auto CReg = CmpMI->getOperand(3).getReg();
+ auto CVal = getIConstantVRegValWithLookThrough(CReg, MRI);
+ if (CVal.has_value() && (CVal->Value == 0) &&
+ ((CC == CmpInst::ICMP_EQ) || (CC == CmpInst::ICMP_NE))) {
+ auto CmpTy = MRI.getType(CmpMI->getOperand(2).getReg());
+ auto SelTy = MRI.getType(SelectMI->getOperand(2).getReg());
+ if ((CmpTy.isScalar() && !CmpTy.isPointer()) &&
+ (SelTy.isScalar() && !SelTy.isPointer()) &&
+ CmpTy.getScalarSizeInBits() == SelTy.getScalarSizeInBits()) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+void PISAPostLegalizerCombinerImpl::applyCompareSelect(MachineInstr *MI) const {
+ auto *SelectMI = MI;
+ auto *CmpMI = getDefIgnoringCopies(SelectMI->getOperand(1).getReg(), MRI);
+ auto CC = CmpMI->getOperand(1).getPredicate();
+ auto DstReg = SelectMI->getOperand(0).getReg();
+ auto MIB = B.buildInstr(PISA::G_PISA_SELECT)
+ .addDef(DstReg)
+ .add(CmpMI->getOperand(2));
+ if (CC == CmpInst::ICMP_NE)
+ MIB.add(SelectMI->getOperand(2)).add(SelectMI->getOperand(3));
+ else
+ MIB.add(SelectMI->getOperand(3)).add(SelectMI->getOperand(2));
+ MI->eraseFromParent();
+}
+
+// %9:_(s1) = G_FCMP floatpred(oge), %1:reg32b(s32), %2:reg32b
+// %22:_(s32) = G_CONSTANT i32 0
+// %23:_(s32) = G_CONSTANT i32 -1
+// %10:_(s32) = G_SELECT %9:_(s1), %23:_, %22:_
+// %11:_(s32) = G_CONSTANT i32 -1
+// %12:_(s32) = G_XOR %10:_, %11:_
+// %13:_(s32) = G_CONSTANT i32 -1
+// %6:_(s1) = G_ICMP intpred(eq), %12:_(s32), %13:_
+// G_BRCOND %6:_(s1), %bb.4
+// G_BR %bb.3
+// => %9:_(s1) = G_FCMP floatpred(oge), %1:reg32b(s32), %2:reg32b
+// => G_BRCOND %6:_(s1), %bb.3
+// => G_BR %bb.4
+bool PISAPostLegalizerCombinerImpl::matchFCmpInvertedCond(
+ MachineInstr &MI, std::tuple<MachineInstr *, Register> &MatchInfo) const {
+ assert(MI.getOpcode() == TargetOpcode::G_BR);
+
+ MachineInstr *BrCondMI;
+ MachineBasicBlock *MBB = MI.getParent();
+ MachineBasicBlock::iterator BrIt(MI);
+
+ if (BrIt == MBB->begin())
+ return false;
+ assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");
+
+ // G_BRCOND %6:_(s1), %bb.4
+ BrCondMI = &*std::prev(BrIt);
+ if (BrCondMI->getOpcode() != TargetOpcode::G_BRCOND)
+ return false;
+
+ auto *CmpMI = MRI.getVRegDef(BrCondMI->getOperand(0).getReg());
+
+ Register FCC;
+ CmpInst::Predicate Pred;
+ // Match either:
+ // icmp eq (xor (select fcc, -1, 0), -1), -1 => NOT fcc
+ // icmp eq (select fcc, 0, -1), -1 => NOT fcc
+ // icmp eq (select fcc, -1, 0), 0 => NOT fcc (produced by
+ // not_cmp_fold simplifying the xor form above)
+ bool Matched =
+ mi_match(
+ CmpMI, MRI,
+ m_GICmp(m_Pred(Pred),
+ m_GXor(m_GISelect(m_Reg(FCC), m_AllOnesInt(), m_ZeroInt()),
+ m_AllOnesInt()),
+ m_AllOnesInt())) ||
+ mi_match(CmpMI, MRI,
+ m_GICmp(m_Pred(Pred),
+ m_GISelect(m_Reg(FCC), m_ZeroInt(), m_AllOnesInt()),
+ m_AllOnesInt())) ||
+ mi_match(CmpMI, MRI,
+ m_GICmp(m_Pred(Pred),
+ m_GISelect(m_Reg(FCC), m_AllOnesInt(), m_ZeroInt()),
+ m_ZeroInt()));
+ if (!Matched || Pred != CmpInst::ICMP_EQ)
+ return false;
+
+ // %9:_(s1) = G_FCMP floatpred(oge), %1:reg32b(s32), %2:reg32b
+ auto *FCmpMI = MRI.getVRegDef(FCC);
+ if (FCmpMI->getOpcode() != TargetOpcode::G_FCMP)
+ return false;
+
+ auto FCCTy = MRI.getType(FCC);
+ if (!(FCCTy.isScalar() && FCCTy.getScalarSizeInBits() == 1))
+ return false;
+
+ MatchInfo = std::make_tuple(BrCondMI, FCC);
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyFCmpInvertedCond(
+ MachineInstr &MI, std::tuple<MachineInstr *, Register> &MatchInfo) const {
+ MachineInstr *BrCond;
+ Register FCC;
+ std::tie(BrCond, FCC) = MatchInfo;
+ auto *BrCondBB = BrCond->getOperand(1).getMBB();
+ auto *BrBB = MI.getOperand(0).getMBB();
+
+ Observer.changingInstr(MI);
+ MI.getOperand(0).setMBB(BrCondBB);
+ Observer.changedInstr(MI);
+
+ Observer.changingInstr(*BrCond);
+ BrCond->getOperand(0).setReg(FCC);
+ BrCond->getOperand(1).setMBB(BrBB);
+ Observer.changedInstr(*BrCond);
+}
+
+//%2:registers(s32) = G_SHL %0:reg32b, 2
+//%4:registers(s32) = G_ADD %2:registers, %3:registers
+// => %4:registers(s32) = G_PISA_SMAD %0:reg32b, 4, %3:registers
+bool PISAPostLegalizerCombinerImpl::matchShlAddToMad(
+ MachineInstr &AddMI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+
+ Register MulReg, AddReg;
+ APInt Shift;
+
+ if (!mi_match(&AddMI, MRI,
+ m_GAdd(m_GShl(m_Reg(MulReg), m_ICst(Shift)), m_Reg(AddReg))))
+ return false;
+
+ auto Ty = MRI.getType(MulReg);
+ if (Shift.getZExtValue() >= Ty.getSizeInBits())
+ return false;
+
+ MatchInfo = [Ty, &AddMI, AddReg, MulReg,
+ Shift = std::move(Shift)](MachineIRBuilder &B) {
+ auto MulOp2 = B.buildConstant(
+ Ty, APInt::getOneBitSet(Ty.getSizeInBits(), Shift.getZExtValue()));
+ B.buildIntrinsic(Intrinsic::pisa_smad, AddMI.getOperand(0).getReg())
+ .addUse(MulReg)
+ .addUse(MulOp2.getReg(0))
+ .addUse(AddReg);
+ };
+ return true;
+}
+
+// arcp G_FDIV %0, %1 that has at least one G_FADD user
+// =>
+// %rcp = arcp G_INTRINSIC intrinsic(@llvm.pisa.frcp), %1
+// %result = arcp G_FMUL %0, %rcp
+//
+// This transformation exposes G_FMUL for potential FMA fusion with the G_FADD
+// users.
+bool PISAPostLegalizerCombinerImpl::matchFDivToRcpFMul(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ // Check if this is a floating-point divide with FmArcp flag
+ if (MI.getOpcode() != TargetOpcode::G_FDIV)
+ return false;
+
+ // Must have the FmArcp flag to allow reciprocal approximation
+ if (!MI.getFlag(MachineInstr::FmArcp))
+ return false;
+
+ Register DstReg = MI.getOperand(0).getReg();
+ Register Src0Reg = MI.getOperand(1).getReg();
+ Register Src1Reg = MI.getOperand(2).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+
+ // Only handle 32-bit float for now
+ if (!DstTy.isScalar() || DstTy.getSizeInBits() != 32)
+ return false;
+
+ // Check if the result of this FDIV is used by at least one FADD/FSUB
+ // This ensures we only apply the transformation when FMA fusion is possible
+ if (!llvm::any_of(MRI.use_instructions(DstReg), [](const MachineInstr &Use) {
+ return Use.getOpcode() == TargetOpcode::G_FADD ||
+ Use.getOpcode() == TargetOpcode::G_FSUB;
+ }))
+ return false;
+
+ uint16_t Flags = MI.getFlags();
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ auto Rcp = B.buildIntrinsic(Intrinsic::pisa_frcp, {DstTy})
+ .addUse(Src1Reg)
+ .setMIFlags(Flags);
+
+ // A / B -> A * RCP(B)
+ B.buildFMul(DstReg, Src0Reg, Rcp, Flags);
+ };
+ return true;
+}
+
+void PISAPostLegalizerCombinerImpl::applyShiftOfConstants(
+ MachineInstr *MI) const {
+ auto [SrcMI, SrcRegIdx] =
+ PISA::getDefIgnoringBitcasts(MI->getOperand(1).getReg(), MRI);
+ auto [ShiftMI, ShiftRegIdx] =
+ PISA::getDefIgnoringBitcasts(MI->getOperand(2).getReg(), MRI);
+
+ const auto *CImm = SrcMI->getOperand(1).getCImm();
+ auto Shift = ShiftMI->getOperand(1).getCImm()->getZExtValue();
+ int64_t NewVal = 0;
+ switch (MI->getOpcode()) {
+ default:
+ assert(0 && "unhandled shift opcode");
+ break;
+ case TargetOpcode::G_SHL:
+ NewVal = CImm->getZExtValue() << Shift;
+ break;
+ case TargetOpcode::G_ASHR:
+ NewVal = CImm->getSExtValue() >> Shift;
+ break;
+ case TargetOpcode::G_LSHR:
+ NewVal = CImm->getZExtValue() >> Shift;
+ break;
+ }
+ B.buildConstant(MI->getOperand(0), NewVal);
+ MI->eraseFromParent();
+}
+
+// A(<2x16>) = G_BITCAST ARG(32)
+// B(16), C(16) = G_UNMERGE_VALUES A(<2x16)
+// D(<2x16>) = G_BUILD_VECTOR B, C
+// E(32) = G_BITCAST D(<2x16>)
+// => E(32) = COPY ARG(32)
+bool PISAPostLegalizerCombinerImpl::matchRedundantMovesPost(
+ MachineInstr &MI) const {
+ auto &BitcastMI = MI;
+ auto DstReg = BitcastMI.getOperand(0).getReg();
+ auto SrcReg = BitcastMI.getOperand(1).getReg();
+ if (MRI.getType(DstReg).isVector() || !MRI.getType(SrcReg).isVector())
+ return false;
+
+ auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ if (BuildVecMI.getOpcode() != TargetOpcode::G_BUILD_VECTOR)
+ return false;
+
+ Register UnmergeReg = 0;
+ for (unsigned I = 1; I < BuildVecMI.getNumOperands(); I++) {
+ auto &UnmergeMI =
+ *getDefIgnoringCopies(BuildVecMI.getOperand(I).getReg(), MRI);
+ if (UnmergeMI.getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+ return false;
+ if (I == 1) {
+ UnmergeReg =
+ UnmergeMI.getOperand(UnmergeMI.getNumOperands() - 1).getReg();
+ } else if (UnmergeReg !=
+ UnmergeMI.getOperand(UnmergeMI.getNumOperands() - 1).getReg()) {
+ // must extract indices from the same vector
+ return false;
+ }
+ // order to extracted operands should match
+ auto VecEltReg = BuildVecMI.getOperand(I).getReg();
+ if (UnmergeMI.getOperand(I - 1).getReg() != VecEltReg)
+ return false;
+ }
+
+ auto &SrcBitcastMI = *getDefIgnoringCopies(UnmergeReg, MRI);
+ if (SrcBitcastMI.getOpcode() != TargetOpcode::G_BITCAST)
+ return false;
+
+ auto SrcBitcastReg = SrcBitcastMI.getOperand(1).getReg();
+ if (MRI.getType(DstReg).getSizeInBits() !=
+ MRI.getType(SrcBitcastReg).getSizeInBits())
+ return false;
+ return true;
+}
+
+void PISAPostLegalizerCombinerImpl::applyRedundantMovesPost(
+ MachineInstr &MI) const {
+ auto &BitcastMI = MI;
+ auto DstReg = BitcastMI.getOperand(0).getReg();
+ auto SrcReg = BitcastMI.getOperand(1).getReg();
+ auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ auto &UnmergeMI =
+ *getDefIgnoringCopies(BuildVecMI.getOperand(1).getReg(), MRI);
+ auto &SrcBitcastMI = *getDefIgnoringCopies(
+ UnmergeMI.getOperand(UnmergeMI.getNumOperands() - 1).getReg(), MRI);
+ auto SrcBitcastReg = SrcBitcastMI.getOperand(1).getReg();
+ B.buildCopy(DstReg, SrcBitcastReg);
+ MI.eraseFromParent();
+}
+
+// A(s16) = G_EXTRACT_VECTOR_ELT ARG(<N x s16), 0
+// B(s16) = G_EXTRACT_VECTOR_ELT ARG(<N x s16), 1
+// ...
+// X(s16) = G_EXTRACT_VECTOR_ELT ARG(<N x s16>), N-1
+// Y(<N x s16>) = G_BUILD_VECTOR A, B, ..., X
+// => Y(<N x s16>) = COPY ARG(<N x s16>)
+//
+// Also matches the equivalent unpack-via-unmerge idiom:
+// A, B, ..., X = G_UNMERGE_VALUES ARG(<N x s16>)
+// Y(<N x s16>) = G_BUILD_VECTOR A, B, ..., X
+// => Y(<N x s16>) = COPY ARG(<N x s16>)
+bool PISAPostLegalizerCombinerImpl::matchExtractAllToBuildVector(
+ MachineInstr &MI, Register &Replacement) const {
+ auto &BuildVecMI = MI;
+ auto DstReg = BuildVecMI.getOperand(0).getReg();
+ auto SrcReg = BuildVecMI.getOperand(1).getReg();
+
+ if (MRI.getType(SrcReg).isVector())
+ return false;
+
+ // Branch A: all operands are G_EXTRACT_VECTOR_ELT from the same vector with
+ // sequential constant indices 0..N-1.
+ Register SrcVecReg;
+ bool AllExtracts = true;
+ for (unsigned I = 1; I < BuildVecMI.getNumOperands(); I++) {
+ auto EltReg = BuildVecMI.getOperand(I).getReg();
+ auto &ExtractMI = *getDefIgnoringCopies(EltReg, MRI);
+ if (ExtractMI.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT) {
+ AllExtracts = false;
+ break;
+ }
+
+ if (I == 1)
+ SrcVecReg = ExtractMI.getOperand(1).getReg();
+ else if (SrcVecReg != ExtractMI.getOperand(1).getReg()) {
+ AllExtracts = false;
+ break;
+ }
+
+ auto IndexReg = ExtractMI.getOperand(2).getReg();
+ auto CValue = getIConstantVRegValWithLookThrough(IndexReg, MRI);
+ if (!CValue.has_value() || CValue->Value != (I - 1)) {
+ AllExtracts = false;
+ break;
+ }
+ }
+ if (AllExtracts) {
+ if (MRI.getType(DstReg) != MRI.getType(SrcVecReg))
+ return false;
+ Replacement = SrcVecReg;
+ return true;
+ }
+
+ // Branch B: all operands are the sequential defs of a single G_UNMERGE_VALUES
+ // whose source vector type matches the G_BUILD_VECTOR dest type.
+ unsigned NumElts = BuildVecMI.getNumOperands() - 1;
+ auto FirstDefAndReg =
+ getDefSrcRegIgnoringCopies(BuildVecMI.getOperand(1).getReg(), MRI);
+ if (!FirstDefAndReg)
+ return false;
+ auto *UnmergeMI = FirstDefAndReg->MI;
+ if (UnmergeMI->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+ return false;
+ if (UnmergeMI->getNumOperands() - 1 != NumElts)
+ return false; // G_UNMERGE_VALUES produces a different number of elements
+ auto UnmergeSrcReg = UnmergeMI->getOperand(NumElts).getReg();
+ if (MRI.getType(DstReg) != MRI.getType(UnmergeSrcReg))
+ return false;
+
+ for (unsigned I = 0; I < NumElts; I++) {
+ auto DefAndReg =
+ getDefSrcRegIgnoringCopies(BuildVecMI.getOperand(I + 1).getReg(), MRI);
+ if (!DefAndReg || DefAndReg->MI != UnmergeMI)
+ return false; // different producer
+ if (DefAndReg->Reg != UnmergeMI->getOperand(I).getReg())
+ return false; // out-of-order element pickup
+ }
+
+ Replacement = UnmergeSrcReg;
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyExtractAllToBuildVector(
+ MachineInstr &MI, Register Replacement) const {
+ auto &BuildVecMI = MI;
+ auto DstReg = BuildVecMI.getOperand(0).getReg();
+ B.buildCopy(DstReg, Replacement);
+ MI.eraseFromParent();
+}
+
+// This function tries to match following pattern:
+// mask = ((1 << width) - 1) << offset;
+// dst = ((data << offset) & mask) | (base & ~mask);
+// -> bfi %data, %base, %width, %offset
+// Currently only the variant with constant masks is supported.
+bool PISAPostLegalizerCombinerImpl::matchOrAndToBfi(
+ MachineInstr &OrMI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ // BFI supports only 32 bits of width, start with this simple check to
+ // exclude unsupported shifts early.
+ auto BitWidth = MRI.getType(OrMI.getOperand(0).getReg()).getSizeInBits();
+ if (BitWidth != 32)
+ return false;
+
+ int64_t MaskA, MaskB;
+ Register A, B;
+ if (!mi_match(OrMI, MRI,
+ m_GOr(m_GAnd(m_Reg(A), m_ICst(MaskA)),
+ m_GAnd(m_Reg(B), m_ICst(MaskB)))))
+ return false;
+ if (MaskA != ~MaskB)
+ return false;
+
+ Register ShiftedData, Base;
+ unsigned Offset = 0, Width = 0;
+ if (isShiftedMask_32(MaskA, Offset, Width)) {
+ ShiftedData = A;
+ Base = B;
+ } else if (isShiftedMask_32(MaskB, Offset, Width)) {
+ ShiftedData = B;
+ Base = A;
+ } else {
+ return false;
+ }
+
+ auto [ShiftedDataMI, ShiftedDataRegIdx] =
+ PISA::getDefIgnoringBitcasts(ShiftedData, MRI);
+
+ Register Data;
+ if (!mi_match(ShiftedDataMI, MRI,
+ m_GShl(m_Reg(Data), m_SpecificICst(Offset))))
+ return false;
+
+ MatchInfo = [Data, Base, Width, Offset, &OrMI](MachineIRBuilder &B) {
+ auto WidthReg = B.buildConstant(I32, Width);
+ auto OffsetReg = B.buildConstant(I32, Offset);
+ B.buildIntrinsic(Intrinsic::pisa_bfi, {OrMI.getOperand(0)})
+ .addUse(Base)
+ .addUse(Data)
+ .addUse(WidthReg.getReg(0))
+ .addUse(OffsetReg.getReg(0));
+ };
+ return true;
+}
+
+// This function tries to match following pattern:
+// diff = bitwidth - width;
+// shift = diff - offset
+// dst = ((data << shift) >> diff);
+// -> bfe %data, %width, %offset
+bool PISAPostLegalizerCombinerImpl::matchShiftSubToBfe(
+ MachineInstr &ShrMI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ // BFE supports only 32 bits of width, start with this simple check to
+ // exclude unsupported shifts early.
+ auto BitWidth = MRI.getType(ShrMI.getOperand(0).getReg()).getSizeInBits();
+ if (BitWidth != 32)
+ return false;
+
+ auto MakeApplyHandler = [&ShrMI](Register Data, Register Width,
+ Register Offset) {
+ return [&ShrMI, Data, Width, Offset](MachineIRBuilder &B) {
+ B.buildIntrinsic(Intrinsic::pisa_ubfe, {ShrMI.getOperand(0)})
+ .addUse(Data)
+ .addUse(Width)
+ .addUse(Offset);
+ };
+ };
+
+ constexpr int64_t ShiftInstMask = 0x1f;
+
+ Register Data, Diff, Offset;
+ if (mi_match(ShrMI, MRI,
+ m_GLShr(m_GShl(m_Reg(Data), m_GSub(m_Reg(Diff), m_Reg(Offset))),
+ m_Reg(Diff)))) {
+ Register Width;
+ if (mi_match(Diff, MRI,
+ m_GSub(m_SpecificICst((int64_t)BitWidth), m_Reg(Width))) &&
+ VT->getKnownBits(Width).countMaxTrailingOnes() <= 6 &&
+ VT->getKnownBits(Offset).countMaxTrailingOnes() <= 5) {
+ // s32 %bitwidth = G_CONSTANT i32 32
+ // s32 %andwidth = G_AND i32 %width, <=63
+ // s32 %andoffset = G_AND i32 %offset, <=31
+ // s32 %diff = G_SUB i32 %bitwidth, %andwidth
+ // s32 %shift = G_SUB i32 %diff, %andoffset
+ // s32 %shl = G_SHL i32 %data, %shift
+ // s32 %shr = G_LSHR i32 %shl, %diff
+ // =>
+ // bfe %data %andwidth %andoffset
+ MatchInfo = MakeApplyHandler(Data, Width, Offset);
+ return true;
+ }
+ } else if (mi_match(
+ ShrMI, MRI,
+ m_GLShr(m_GShl(m_Reg(Data),
+ m_GAnd(m_GSub(m_Reg(Diff), m_Reg(Offset)),
+ m_SpecificICst(ShiftInstMask))),
+ m_GAnd(m_Reg(Diff), m_SpecificICst(ShiftInstMask))))) {
+ Register Width;
+ if (mi_match(Diff, MRI,
+ m_GSub(m_SpecificICst((int64_t)BitWidth), m_Reg(Width)))) {
+ if (VT->getKnownBits(Width).countMaxTrailingOnes() <= 5) {
+ // s32 %bitwidth = G_CONSTANT i32 32
+ // s32 %andwidth = G_AND i32 %width, <=31
+ // s32 %diff = G_SUB i32 %bitwidth, %andwidth
+ // s32 %shift = G_SUB i32 %diff, %offset
+ // s32 %andshift = G_AND i32 %shift, 31
+ // s32 %shl = G_SHL i32 %data, %andshift
+ // s32 %anddiff = G_AND i32 %diff, 31
+ // s32 %shr = G_LSHR i32 %shl, %anddiff
+ // =>
+ // s32 %andwidth = G_AND i32 %width, <=31
+ // bfe %data %andwidth %offset
+ MatchInfo = MakeApplyHandler(Data, Width, Offset);
+ return true;
+ }
+ // Masking %diff with 0x1f is equivalent to masking %width with 0x1f.
+ // We have to explicitly mask width with 0x1f to preserve behavior as
+ // bfe masks width with 0x3f by default.
+ // s32 %bitwidth = G_CONSTANT i32 32
+ // s32 %diff = G_SUB i32 %bitwidth, %width
+ // s32 %shift = G_SUB i32 %diff, %offset
+ // s32 %andshift = G_AND i32 %shift, 31
+ // s32 %shl = G_SHL i32 %data, %andshift
+ // s32 %anddiff = G_AND i32 %diff, 31
+ // s32 %shr = G_LSHR i32 %shl, %anddiff
+ // =>
+ // s32 %andwidth = G_AND i32 %width, 31
+ // bfe %data %andwidth %offset
+ auto WidthType = MRI.getType(Width);
+ MatchInfo = [Data, Width, WidthType, Offset,
+ &ShrMI](MachineIRBuilder &B) {
+ auto Mask = B.buildConstant(WidthType, ShiftInstMask);
+ auto MaskedWidth = B.buildAnd(WidthType, Width, Mask);
+ B.buildIntrinsic(Intrinsic::pisa_ubfe, {ShrMI.getOperand(0)})
+ .addUse(Data)
+ .addUse(MaskedWidth->getOperand(0).getReg())
+ .addUse(Offset);
+ };
+ return true;
+ }
+ }
+ return false;
+}
+
+// s32 %maskedwidth = G_AND %width, 63
+// s32 %maskedoffset = G_AND %offset, 31
+// bfe %data %maskedwidth %maskedoffset
+// =>
+// bfe %data %width %offset
+bool PISAPostLegalizerCombinerImpl::matchAndBitfieldToBitfield(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ Register Data, Width, Offset;
+ Intrinsic::ID II;
+ if (auto *MIIntrinsic = dyn_cast<GIntrinsic>(&MI)) {
+ II = MIIntrinsic->getIntrinsicID();
+ if (II == Intrinsic::pisa_ubfe) {
+ Data = MI.getOperand(2).getReg();
+ Width = MI.getOperand(3).getReg();
+ Offset = MI.getOperand(4).getReg();
+ } else if (II == Intrinsic::pisa_bfi) {
+ Data = MI.getOperand(2).getReg();
+ Width = MI.getOperand(4).getReg();
+ Offset = MI.getOperand(5).getReg();
+ } else {
+ return false;
+ }
+ } else {
+ // G_[SU]BFX, convert it to pisa_[su]bfe
+ II = MI.getOpcode() == TargetOpcode::G_UBFX ? Intrinsic::pisa_ubfe
+ : Intrinsic::pisa_sbfe;
+ Data = MI.getOperand(1).getReg();
+ Width = MI.getOperand(3).getReg();
+ Offset = MI.getOperand(2).getReg();
+ }
+
+ Register UnmaskedWidth{}, UnmaskedOffset{};
+ if (VT->getKnownBits(Width).countMaxTrailingOnes() >= 6 &&
+ mi_match(Width, MRI, m_GAnd(m_Reg(UnmaskedWidth), m_Reg()))) {
+ Width = UnmaskedWidth;
+ }
+ if (VT->getKnownBits(Offset).countMaxTrailingOnes() >= 5 &&
+ mi_match(Offset, MRI, m_GAnd(m_Reg(UnmaskedOffset), m_Reg()))) {
+ Offset = UnmaskedOffset;
+ }
+ if (!UnmaskedWidth.isValid() && !UnmaskedOffset.isValid())
+ return false; // No match, exit.
+
+ if (II == Intrinsic::pisa_ubfe || II == Intrinsic::pisa_sbfe) {
+ MatchInfo = [&MI, II, Data, Width, Offset](MachineIRBuilder &B) {
+ B.buildIntrinsic(II, {MI.getOperand(0)})
+ .addUse(Data)
+ .addUse(Width)
+ .addUse(Offset);
+ };
+ } else {
+ MatchInfo = [&MI, Data, Width, Offset](MachineIRBuilder &B) {
+ B.buildIntrinsic(Intrinsic::pisa_bfi, {MI.getOperand(0)})
+ .addUse(Data)
+ .addUse(MI.getOperand(3).getReg())
+ .addUse(Width)
+ .addUse(Offset);
+ };
+ }
+ return true;
+}
+
+// This transforms an unoptimal pattern for fcmp + zext i1->i8 on <2 x i8>
+// result types (e.g. from code that uses native i8 element vectors):
+// %14:_(<2 x s16>) = G_BITCAST %CmpRes:_(s32)
+// %20:_(s16), %21:_(s16) = G_UNMERGE_VALUES %14:_(<2 x s16>)
+// %22:_(s8) = G_TRUNC %20:_(s16)
+// %23:_(s8) = G_TRUNC %21:_(s16)
+// %17:_(<2 x s8>) = G_BUILD_VECTOR %22:_(s8), %23:_(s8)
+// %28:_(s16) = G_CONSTANT i16 257
+// %19:_(<2 x s8>) = G_BITCAST %28:_(s16)
+// %24:_(s16) = G_BITCAST %17:_(<2 x s8>)
+// %25:_(s16) = G_BITCAST %19:_(<2 x s8>)
+// %26:_(s16) = G_AND %24:_, %25:_
+// %res:_(<2 x s8>) = G_BITCAST %26:_(s16)
+// => bfe %1, %CmpRes, 1, 0
+// bfe %2, %CmpRes, 1, 16
+// bfi %3, %1, %2, 8
+// trunc.16.32 %res, %3
+// If the only user of the result is a G_STORE, we modify it to store
+// an s16 instead of <2xs8>. Otherwise, we insert a G_BITCAST.
+// Note: sub-byte types (i1, i4) are now promoted to i16, so this pattern
+// applies only to code that explicitly uses native <2 x i8> element types.
+bool PISAPostLegalizerCombinerImpl::matchV2i1ZextToBfeBfi(
+ MachineInstr &BitcastMI, Register &BitcastInput) const {
+
+ auto VectorElementSize =
+ MRI.getType(BitcastMI.getOperand(0).getReg()).getScalarSizeInBits();
+ // We only support an i8 zext
+ if (VectorElementSize != 8)
+ return false;
+
+ // Match automatically up to the G_AND instructions.
+ // The mask constant 257 (= 0x0101 = {1,1} as two i8 lanes) may appear in
+ // two forms depending on whether build_vector_with_constants has already
+ // folded G_BUILD_VECTOR(i8 1, i8 1) -> G_CONSTANT i16 257:
+ // original: G_BITCAST(G_BITCAST(G_CONSTANT i?? 257))
+ // folded: G_CONSTANT i16 257 (bare constant, no bitcasts)
+ Register UpperCmpRes, LowerCmpRes;
+ bool Matched =
+ mi_match(BitcastMI, MRI,
+ m_GBitcast(m_GAnd(
+ m_GBitcast(m_GBuildVector(m_GTrunc(m_Reg(UpperCmpRes)),
+ m_GTrunc(m_Reg(LowerCmpRes)))),
+ m_GBitcast(m_GBitcast(m_SpecificICst(257)))))) ||
+ mi_match(BitcastMI, MRI,
+ m_GBitcast(m_GAnd(
+ m_GBitcast(m_GBuildVector(m_GTrunc(m_Reg(UpperCmpRes)),
+ m_GTrunc(m_Reg(LowerCmpRes)))),
+ m_SpecificICst(257))));
+ if (!Matched)
+ return false;
+
+ /*
+ Verify that
+ - the extracted halves of the comparison result are part of
+ G_UNMERGE_VALUES instructions
+ - the extracted halves are at the correct position in the unmerge
+ instructions, e.g. the UpperCmpRes reg should be at index 0
+ - both G_UNMERGE_VALUES instructions use the same input
+ Note that this way, we both support two unmerges with two unused values (see
+ code example), and one unmerge instruction with both values being used
+ (i.e. UpperUnmerge =?= LowerUnmerge)
+ */
+ auto *UpperUnmergeInstr = MRI.getUniqueVRegDef(UpperCmpRes);
+ auto *LowerUnmergeInstr = MRI.getUniqueVRegDef(LowerCmpRes);
+ if (!UpperUnmergeInstr || !LowerUnmergeInstr ||
+ UpperUnmergeInstr->getOpcode() != TargetOpcode::G_UNMERGE_VALUES ||
+ LowerUnmergeInstr->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+ return false;
+
+ if (UpperUnmergeInstr->getOperand(0).getReg() != UpperCmpRes)
+ return false;
+
+ if (LowerUnmergeInstr->getOperand(1).getReg() != LowerCmpRes)
+ return false;
+
+ auto BitcastRes = UpperUnmergeInstr->getOperand(2);
+ if (!BitcastRes.isIdenticalTo(LowerUnmergeInstr->getOperand(2)))
+ return false;
+
+ // Verify that this inst is a G_BITCAST, and get the input register
+ auto *BitcastInst = MRI.getUniqueVRegDef(BitcastRes.getReg());
+ if (!BitcastInst || BitcastInst->getOpcode() != TargetOpcode::G_BITCAST)
+ return false;
+
+ BitcastInput = BitcastInst->getOperand(1).getReg();
+ return true;
+}
+
+void PISAPostLegalizerCombinerImpl::applyV2i1ZextToBfeBfi(
+ MachineInstr &BitcastMI, Register BitcastInput) const {
+
+ auto *MRI = B.getMRI();
+ auto Bfe1Res = MRI->createGenericVirtualRegister(I32);
+ auto Bfe2Res = MRI->createGenericVirtualRegister(I32);
+ auto BfiRes = MRI->createGenericVirtualRegister(I32);
+
+ auto Zero = B.buildConstant(I32, 0);
+ auto One = B.buildConstant(I32, 1);
+ auto Eight = B.buildConstant(I32, 8);
+ auto Sixteen = B.buildConstant(I32, 16);
+
+ B.buildIntrinsic(Intrinsic::pisa_ubfe, {DstOp(Bfe1Res)})
+ .addUse(BitcastInput)
+ .addUse(One.getReg(0)) // Width
+ .addUse(Zero.getReg(0)); // Offset
+
+ B.buildIntrinsic(Intrinsic::pisa_ubfe, {DstOp(Bfe2Res)})
+ .addUse(BitcastInput)
+ .addUse(One.getReg(0)) // Width
+ .addUse(Sixteen.getReg(0)); // Offset
+
+ B.buildIntrinsic(Intrinsic::pisa_bfi, {DstOp(BfiRes)})
+ .addUse(Bfe1Res)
+ .addUse(Bfe2Res)
+ .addUse(Eight.getReg(0)) // Width
+ .addUse(Eight.getReg(0)); // Offset
+
+ /*
+ Peephole optimisation:
+ The example input given above continues with a G_STORE instr
+ as the only user of our newly created G_TRUNC:
+ [...]
+ %52:_(s8) = G_TRUNC %48:_(s16)
+ %10:_(<2 x s8>) = G_BUILD_VECTOR %51:_(s8), %52:_(s8)
+ G_STORE %10:_(s16), %4:_(p1)
+
+ If we would just bitcast the G_TRUNC result back to
+ <2 x s8> to match all expected types in that case, the resulting
+ PISA code inserts an unnecessary mov into a different register
+
+ To avoid that, check if the next user is only a G_STORE instruction.
+ In that case, we can just modify the type of the stored register
+ to s16 and change the MachineMemOperand's type to s16 as well.
+
+ In all other cases, insert a G_BITCAST to <2 x s8>.
+ */
+
+ auto StoreInput = BitcastMI.getOperand(0).getReg();
+ if (MRI->hasOneNonDBGUse(StoreInput) &&
+ MRI->use_instr_nodbg_begin(StoreInput)->getOpcode() ==
+ TargetOpcode::G_STORE) {
+ // Our only use is a G_STORE
+ B.buildInstr(TargetOpcode::G_TRUNC).addDef(StoreInput).addUse(BfiRes);
+
+ MRI->setType(StoreInput, I16);
+ for (auto *MemOp : MRI->use_instr_nodbg_begin(StoreInput)->memoperands())
+ MemOp->setType(I16);
+ } else {
+ // We either have multiple users or the following user is not a G_STORE
+ // Insert a bitcast to <2 x s8> so that nothing breaks
+ auto TruncRes = MRI->createGenericVirtualRegister(I16);
+ B.buildInstr(TargetOpcode::G_TRUNC).addDef(TruncRes).addUse(BfiRes);
+
+ B.buildInstr(TargetOpcode::G_BITCAST).addDef(StoreInput).addUse(TruncRes);
+ }
+ BitcastMI.eraseFromParent();
+}
+
+// %374:_(s16) = G_CONSTANT i16 1
+// %375:_(s16) = G_AND %321:_, %374:_
+// %421:_(s16) = G_CONSTANT i16 0
+// %422:_(s16) = G_CONSTANT i16 1
+// %405:_(s16) = G_PISA_SELECT %375:_, %422:_, %421:_
+// => %374:_(s16) = G_CONSTANT i16 1
+// => %375:_(s16) = G_AND %321:_, %374:_
+// => %405:_(s16) = COPY %375
+bool PISAPostLegalizerCombinerImpl::matchAndSelect(
+ MachineInstr &MI, Register &Replacement) const {
+ auto &SelectMI = MI;
+
+ auto Const0 =
+ getIConstantVRegValWithLookThrough(SelectMI.getOperand(3).getReg(), MRI);
+ auto Const1 =
+ getIConstantVRegValWithLookThrough(SelectMI.getOperand(2).getReg(), MRI);
+ if (!Const0.has_value() || !Const1.has_value())
+ return false;
+ if ((Const0->Value != 0) || (Const1->Value != 1))
+ return false;
+
+ auto &AndMI = *getDefIgnoringCopies(SelectMI.getOperand(1).getReg(), MRI);
+ if (AndMI.getOpcode() != TargetOpcode::G_AND)
+ return false;
+ auto AndConst =
+ getIConstantVRegValWithLookThrough(AndMI.getOperand(2).getReg(), MRI);
+ if (!AndConst.has_value() || (AndConst->Value != 1))
+ return false;
+
+ if (MRI.getType(AndMI.getOperand(0).getReg()) !=
+ MRI.getType(SelectMI.getOperand(0).getReg()))
+ return false;
+
+ Replacement = AndMI.getOperand(0).getReg();
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyAndSelect(MachineInstr &MI,
+ Register Replacement) const {
+
+ B.buildCopy(MI.getOperand(0).getReg(), Replacement);
+ MI.eraseFromParent();
+}
+
+// %23:_(s32) = G_CONSTANT i32 8
+// %17:_(s8) = G_TRUNC %1:reg32b(s32)
+// %24:_(s32) = G_LSHR %1:reg32b, %23:_(s32)
+// %25:_(s8) = G_TRUNC %24:_(s32)
+// %28:_(s8) = G_TRUNC %2:reg32b(s32)
+// %31:_(s32) = G_LSHR %2:reg32b, %23:_(s32)
+// %32:_(s8) = G_TRUNC %31:_(s32)
+///%149:_(<4 x s8>) = G_BUILD_VECTOR %17:_(s8), %25:_(s8), %28:_(s8), %32:_(s8)
+// %141:_(s32) = G_BITCAST %149:_(<4 x s8>)
+// => %A = G_AND %1, 0xFFFF
+// => %B = G_SHL %2, 16
+// => %141 = G_OR %A, %B
+bool PISAPostLegalizerCombinerImpl::matchBuildRegFrom2(
+ MachineInstr &MI, Reg2MatchInfo &MatchInfo) const {
+ auto &BitcastMI = MI;
+
+ auto DstTy = MRI.getType(BitcastMI.getOperand(0).getReg());
+ if (DstTy.isVector() || (DstTy.getScalarSizeInBits() != 32))
+ return false;
+ auto SrcReg = BitcastMI.getOperand(1).getReg();
+ auto SrcTy = MRI.getType(SrcReg);
+ if (!SrcTy.isVector() || (SrcTy.getScalarSizeInBits() != 8))
+ return false;
+ auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ if (BuildVecMI.getOpcode() != TargetOpcode::G_BUILD_VECTOR)
+ return false;
+
+ for (auto I = 0; I < 2; I++) {
+ auto LoReg = BuildVecMI.getOperand(1 + (I * 2)).getReg();
+ auto HiReg = BuildVecMI.getOperand(2 + (I * 2)).getReg();
+ auto &LoTruncMI = *getDefIgnoringCopies(LoReg, MRI);
+ auto &HiTruncMI = *getDefIgnoringCopies(HiReg, MRI);
+ if (LoTruncMI.getOpcode() != TargetOpcode::G_TRUNC)
+ return false;
+ if (HiTruncMI.getOpcode() != TargetOpcode::G_TRUNC)
+ return false;
+ auto &HiShiftMI =
+ *getDefIgnoringCopies(HiTruncMI.getOperand(1).getReg(), MRI);
+ if (HiShiftMI.getOpcode() != TargetOpcode::G_LSHR)
+ return false;
+ if (LoTruncMI.getOperand(1).getReg() != HiShiftMI.getOperand(1).getReg())
+ return false;
+ auto ShiftConst = getIConstantVRegValWithLookThrough(
+ HiShiftMI.getOperand(2).getReg(), MRI);
+ if (!ShiftConst.has_value() || (ShiftConst->Value != 8))
+ return false;
+ if (I == 0)
+ MatchInfo.Reg0 = LoTruncMI.getOperand(1).getReg();
+ else
+ MatchInfo.Reg1 = LoTruncMI.getOperand(1).getReg();
+ }
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyBuildRegFrom2(
+ MachineInstr &MI, Reg2MatchInfo &MatchInfo) const {
+ auto MaskReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ auto Mask = B.buildConstant(MaskReg, 0xFFFF);
+ auto ShiftReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ auto Shift = B.buildConstant(ShiftReg, 16);
+
+ auto AReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ auto BReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+
+ auto RegSize = MRI.getType(MatchInfo.Reg0).getSizeInBits();
+ assert((RegSize == 16 || RegSize == 32 || RegSize == 64) &&
+ "only supporting 16/32/64bit registers");
+ Register ASrcReg, BSrcReg;
+ if (RegSize == 16) {
+ ASrcReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ BSrcReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ B.buildZExt(ASrcReg, MatchInfo.Reg0);
+ B.buildZExt(BSrcReg, MatchInfo.Reg1);
+ } else if (RegSize == 64) {
+ ASrcReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ BSrcReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ B.buildTrunc(ASrcReg, MatchInfo.Reg0);
+ B.buildTrunc(BSrcReg, MatchInfo.Reg1);
+ } else {
+ ASrcReg = MatchInfo.Reg0;
+ BSrcReg = MatchInfo.Reg1;
+ }
+ B.buildAnd(AReg, ASrcReg, Mask);
+ B.buildShl(BReg, BSrcReg, Shift);
+ B.buildOr(MI.getOperand(0).getReg(), AReg, BReg);
+ MI.eraseFromParent();
+}
+
+// Match a G_BUILD_VECTOR whose two elements are consecutive lanes (base,
+// base+1) of a single wider source vector, produced by G_UNMERGE_VALUES.
+// Such a build is really a sub-vector slice of that source; recording
+// (source, base) lets applyBuildVectorFromUnmergeLanes rewrite it to a
+// G_EXTRACT_SUBVECTOR, which ISel lowers to a composite sub-register COPY.
+bool PISAPostLegalizerCombinerImpl::matchBuildVectorFromUnmergeLanes(
+ MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
+ Register DstReg = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+ // Only nameable 2-element pairs (map to .xy / .zw) are handled.
+ if (!DstTy.isVector() || DstTy.getNumElements() != 2)
+ return false;
+ const unsigned NumElts = 2;
+
+ const MachineInstr *Unmerge = nullptr;
+ Register SrcReg;
+ int64_t BaseLane = -1;
+ for (unsigned I = 0; I < NumElts; ++I) {
+ auto Def = getDefSrcRegIgnoringCopies(MI.getOperand(1 + I).getReg(), MRI);
+ if (!Def)
+ return false;
+ const MachineInstr *D = Def->MI;
+ if (D->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+ return false;
+ // The source operand of G_UNMERGE_VALUES is its last operand; the ones
+ // before it are the per-lane results.
+ const unsigned NumLanes = D->getNumOperands() - 1;
+ int Lane = -1;
+ for (unsigned J = 0; J < NumLanes; ++J)
+ if (D->getOperand(J).getReg() == Def->Reg) {
+ Lane = (int)J;
+ break;
+ }
+ if (Lane < 0)
+ return false;
+ if (I == 0) {
+ Unmerge = D;
+ SrcReg = D->getOperand(NumLanes).getReg();
+ BaseLane = Lane;
+ } else if (D != Unmerge || Lane != BaseLane + (int)I) {
+ return false;
+ }
+ }
+
+ // Base must form a nameable composite sub-register (.xy at 0, .zw at 2).
+ if (BaseLane != 0 && BaseLane != 2)
+ return false;
+
+ LLT SrcTy = MRI.getType(SrcReg);
+ // The .xy/.zw composite sub-registers are only defined on 4-lane register
+ // classes (Reg*bx4). On wider vectors (v5-v8, v16, ...) the elements have
+ // per-lane sub-registers only; a .zw view there is not a real sub-register
+ // and would be mis-lowered. Restrict to <=4-lane sources (the intended
+ // 3-4-element scope) and let wider vectors keep the element-wise path.
+ if (!SrcTy.isVector() || SrcTy.getElementType() != DstTy.getElementType() ||
+ SrcTy.getNumElements() <= NumElts || SrcTy.getNumElements() > 4 ||
+ (uint64_t)BaseLane + NumElts > SrcTy.getNumElements())
+ return false;
+
+ MatchInfo = std::make_tuple(SrcReg, BaseLane);
+ return true;
+}
+
+void PISAPostLegalizerCombinerImpl::applyBuildVectorFromUnmergeLanes(
+ MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
+ auto [SrcReg, BaseLane] = MatchInfo;
+ B.buildExtractSubvector(MI.getOperand(0).getReg(), SrcReg, BaseLane);
+ MI.eraseFromParent();
+}
+
+// Match a 4-element G_BUILD_VECTOR that concatenates two whole 2-element
+// sub-vectors: elements [0,1] are lanes 0,1 of source A and elements [2,3] are
+// lanes 0,1 of source B (each produced by G_UNMERGE_VALUES). Such a build is a
+// concatenation; recording (A, B) lets the apply rewrite it to
+// G_CONCAT_VECTORS, which ISel writes into the .xy / .zw slices directly.
+bool PISAPostLegalizerCombinerImpl::matchBuildVectorConcatSubvectors(
+ MachineInstr &MI, SmallVector<Register, 4> &MatchInfo) const {
+ Register DstReg = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+ if (!DstTy.isVector() || DstTy.getNumElements() != 4)
+ return false;
+ const unsigned M = 2; // sub-vector width -> nameable .xy / .zw
+ const unsigned K = DstTy.getNumElements() / M;
+ MatchInfo.clear();
+ for (unsigned G = 0; G < K; ++G) {
+ const MachineInstr *Unmerge = nullptr;
+ Register SrcReg;
+ for (unsigned J = 0; J < M; ++J) {
+ auto Def = getDefSrcRegIgnoringCopies(
+ MI.getOperand(1 + G * M + J).getReg(), MRI);
+ if (!Def)
+ return false;
+ const MachineInstr *D = Def->MI;
+ if (D->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+ return false;
+ const unsigned NumLanes = D->getNumOperands() - 1;
+ int Lane = -1;
+ for (unsigned L = 0; L < NumLanes; ++L)
+ if (D->getOperand(L).getReg() == Def->Reg) {
+ Lane = (int)L;
+ break;
+ }
+ // Each source must be consumed whole and in order (lane J at position J).
+ if (Lane != (int)J)
+ return false;
+ if (J == 0) {
+ Unmerge = D;
+ SrcReg = D->getOperand(NumLanes).getReg();
+ } else if (D != Unmerge) {
+ return false;
+ }
+ }
+ LLT SrcTy = MRI.getType(SrcReg);
+ if (!SrcTy.isVector() || SrcTy.getNumElements() != M ||
+ SrcTy.getElementType() != DstTy.getElementType())
+ return false;
+ MatchInfo.push_back(SrcReg);
+ }
+ return MatchInfo.size() == K;
+}
+
+void PISAPostLegalizerCombinerImpl::applyBuildVectorConcatSubvectors(
+ MachineInstr &MI, SmallVector<Register, 4> &MatchInfo) const {
+ B.buildConcatVectors(MI.getOperand(0).getReg(), MatchInfo);
+ MI.eraseFromParent();
+}
+
+// if NumSignBits == BitWidth:
+// %24:_(s32) = G_CONSTANT i32 0
+// %25:_(s32) = G_CONSTANT i32 1
+// %26:_(s32) = G_AND %18:_, %25:_
+// %9:_(s1) = G_ICMP intpred(ne), %26:_(s32), %24:_
+// => %24:_(s32) = G_CONSTANT i32 0
+// %26:_(s32) = COPY %18:_(s32)
+// %9:_(s1) = G_ICMP intpred(ne), %26:_(s32), %24:_
+bool PISAPostLegalizerCombinerImpl::matchCmpAndAllOnes(
+ MachineInstr &MI, GISelValueTracking *VT, Register &MatchInfo) const {
+ Register SrcReg;
+ CmpInst::Predicate Pred;
+ if (!mi_match(MI, MRI,
+ m_GICmp(m_Pred(Pred), m_GAnd(m_Reg(SrcReg), m_SpecificICst(1)),
+ m_SpecificICst(0))) ||
+ (Pred != CmpInst::ICMP_NE))
+ return false;
+
+ auto BitWidth = MRI.getType(SrcReg).getSizeInBits();
+ if (BitWidth < 16)
+ return false;
+
+ if (VT->computeNumSignBits(SrcReg) < BitWidth)
+ return false;
+
+ MatchInfo = SrcReg;
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyCmpAndAllOnes(
+ MachineInstr &MI, Register &MatchInfo) const {
+ Observer.changingInstr(MI);
+ MI.getOperand(2).setReg(MatchInfo);
+ Observer.changedInstr(MI);
+}
+
+// %11:_(s16) = G_TRUNC %8:_(s32)
+// %12:_(s16) = G_CONSTANT i16 1
+// %13:_(s16) = G_AND %11:_, %12:_
+//=> %12:_(s32) = G_CONSTANT i32 1
+// %13:_(s32) = G_AND %8:_, %12:_
+// %11:_(s16) = G_TRUNC %13:_(s32)
+bool PISAPostLegalizerCombinerImpl::matchFixIllegalShiftAmt(
+ MachineInstr &MI) const {
+ assert(MI.getOpcode() == TargetOpcode::G_SHL ||
+ MI.getOpcode() == TargetOpcode::G_LSHR ||
+ MI.getOpcode() == TargetOpcode::G_ASHR);
+ return MRI.getType(MI.getOperand(2).getReg()) != I32;
+}
+
+void PISAPostLegalizerCombinerImpl::applyFixIllegalShiftAmt(
+ MachineInstr &MI) const {
+ Register ShAmtReg = MI.getOperand(2).getReg();
+ LLT ShAmtTy = MRI.getType(ShAmtReg);
+ MachineIRBuilder MIB(MI);
+ Register NewShAmt = (ShAmtTy.getSizeInBits() > 32)
+ ? MIB.buildTrunc(I32, ShAmtReg).getReg(0)
+ : MIB.buildZExt(I32, ShAmtReg).getReg(0);
+ Observer.changingInstr(MI);
+ MI.getOperand(2).setReg(NewShAmt);
+ Observer.changedInstr(MI);
+}
+
+bool PISAPostLegalizerCombinerImpl::matchSinkTrunc(
+ MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
+ const unsigned Opcode = MI.getOpcode();
+ assert(Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR ||
+ Opcode == TargetOpcode::G_XOR);
+
+ int64_t Mask;
+ Register SrcReg;
+ if (!mi_match(MI, MRI,
+ m_BinOp(Opcode, m_GTrunc(m_Reg(SrcReg)), m_ICst(Mask))))
+ return false;
+
+ // We don't want to create 64-bit boolean operations
+ auto BitWidth = MRI.getType(SrcReg).getSizeInBits();
+ if (BitWidth > 32)
+ return false;
+
+ MatchInfo = std::make_tuple(SrcReg, Mask);
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applySinkTrunc(
+ MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
+ int64_t Mask;
+ Register SrcReg;
+ std::tie(SrcReg, Mask) = MatchInfo;
+ auto SrcTy = MRI.getType(SrcReg);
+ auto MaskReg = MRI.createGenericVirtualRegister(SrcTy);
+ auto DstReg = MRI.createGenericVirtualRegister(SrcTy);
+ B.buildConstant(MaskReg, Mask);
+ B.buildInstr(MI.getOpcode()).addDef(DstReg).addUse(SrcReg).addUse(MaskReg);
+ B.buildInstr(TargetOpcode::G_TRUNC)
+ .addDef(MI.getOperand(0).getReg())
+ .addUse(DstReg);
+ MI.eraseFromParent();
+}
+
+// %22:_(s16) = G_TRUNC %26:_(s32)
+// %2:_(s8) = G_TRUNC %22:_(s16)
+//=> %2:_(s8) = G_TRUNC %26:_(s32)
+bool PISAPostLegalizerCombinerImpl::matchTruncTrunc(MachineInstr &MI,
+ Register &MatchInfo) const {
+ Register SrcReg;
+ if (!mi_match(MI, MRI, m_GTrunc(m_GTrunc(m_Reg(SrcReg)))))
+ return false;
+
+ MatchInfo = SrcReg;
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyTruncTrunc(MachineInstr &MI,
+ Register &MatchInfo) const {
+ Observer.changingInstr(MI);
+ MI.getOperand(1).setReg(MatchInfo);
+ Observer.changedInstr(MI);
+}
+
+// %1:registers(s32) = G_ABS %0:reg32b
+// %4:registers(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired),
+// IRedOp::UMAX, %1:registers(s32), %2:reg32b(s32), %3:reg32b(s32)
+// => %1:registers(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired),
+// IRedOp::ABSMAX, %0:registers(s32), %2:reg32b(s32), %3:reg32b(s32)
+bool PISAPostLegalizerCombinerImpl::matchAbsRedMaxToRedAbsMax(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ unsigned AbsOpcode;
+ unsigned TargetInstOpCode = 0;
+ unsigned TargetOpType = 0;
+ unsigned IntrID = 0;
+ switch (MI.getOpcode()) {
+ default:
+ return false;
+ case TargetOpcode::G_INTRINSIC_CONVERGENT: {
+ auto II = cast<GIntrinsic>(MI).getIntrinsicID();
+ if (II == Intrinsic::pisa_ired) {
+ if (MI.getOperand(2).getImm() != pisa::IRedOp::UMAX)
+ return false;
+ AbsOpcode = TargetOpcode::G_ABS;
+ IntrID = Intrinsic::pisa_ired;
+ TargetOpType = pisa::IRedOp::ABSMAX;
+ } else if (II == Intrinsic::pisa_fred) {
+ if (MI.getOperand(2).getImm() != pisa::FRedOp::MAX)
+ return false;
+ AbsOpcode = TargetOpcode::G_INTRINSIC;
+ IntrID = Intrinsic::pisa_fred;
+ TargetOpType = pisa::FRedOp::ABSMAX;
+ } else
+ return false;
+ break;
+ }
+ }
+ unsigned SrcOpIdx = IntrID ? 3 : 2;
+ auto [SrcMI, SrcRegIdx] =
+ PISA::getDefIgnoringBitcasts(MI.getOperand(SrcOpIdx).getReg(), MRI);
+ MachineInstr *SrcMIPtr = SrcMI;
+ if (SrcMIPtr->getOpcode() != AbsOpcode)
+ return false;
+ if (AbsOpcode == TargetOpcode::G_INTRINSIC &&
+ cast<GIntrinsic>(*SrcMIPtr).getIntrinsicID() != Intrinsic::pisa_fabs)
+ return false;
+ // For G_INTRINSIC, source reg is operand 2; for generic opcodes, operand 1.
+ unsigned AbsSrcIdx = AbsOpcode == TargetOpcode::G_INTRINSIC ? 2 : 1;
+ if (IntrID)
+ MatchInfo = [SrcMIPtr, AbsSrcIdx, &MI, IntrID,
+ TargetOpType](MachineIRBuilder &B) {
+ auto MIB = B.buildIntrinsic(IntrID, {MI.getOperand(0)});
+ MIB.addImm(TargetOpType);
+ MIB.addUse(SrcMIPtr->getOperand(AbsSrcIdx).getReg());
+ MIB.addUse(MI.getOperand(4).getReg());
+ MIB.addUse(MI.getOperand(5).getReg());
+ // pisa_fred carries a trailing i1 nanp operand; pisa_ired does not.
+ if (IntrID == Intrinsic::pisa_fred)
+ MIB.add(MI.getOperand(6)); // copy nanp from original
+ MIB.setMIFlags(MI.getFlags());
+ };
+ else
+ MatchInfo = [SrcMIPtr, &MI, TargetInstOpCode,
+ TargetOpType](MachineIRBuilder &B) {
+ B.buildInstr(TargetInstOpCode)
+ .addDef(MI.getOperand(0).getReg())
+ .addImm(TargetOpType)
+ .addUse(SrcMIPtr->getOperand(1).getReg())
+ .addUse(MI.getOperand(3).getReg())
+ .addUse(MI.getOperand(4).getReg())
+ .add(MI.getOperand(5)) // copy nanp from original
+ .setMIFlags(MI.getFlags());
+ };
+ return true;
+}
+
+// %45:_(s32) = G_CONSTANT i32 -1
+// %60:_(s32) = G_CONSTANT i32 0
+// %46:_(s32) = G_SELECT %10:_(s1), %45:_, %60:_
+// %47:_(s32) = G_SELECT %35:_(s1), %45:_, %60:_
+// %48:_(s32) = G_AND %46:_, %47:_
+// %59:_(s32) = G_CONSTANT i32 31
+// %58:_(s32) = G_SHL %48:_, %59:_(s32)
+// %14:_(s32) = G_ASHR %58:_, %59:_(s32)
+// => %14:_(s32) = G_AND %46:_, %47:_
+bool PISAPostLegalizerCombinerImpl::matchShiftTrueFalse(
+ MachineInstr &AShrMI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+
+ // Result can be only all zeros or all ones. But if sources are also only
+ // all zeros/ones, then shl/ashr is a redundant artifact from legalization.
+
+ auto DstReg = AShrMI.getOperand(0).getReg();
+ auto DstTy = MRI.getType(DstReg);
+ auto ShiftAmount = DstTy.getScalarSizeInBits() - 1;
+
+ Register SrcReg;
+ if (!mi_match(AShrMI, MRI,
+ m_GAShr(m_GShl(m_Reg(SrcReg), m_SpecificICst(ShiftAmount)),
+ m_SpecificICst(ShiftAmount))))
+ return false;
+
+ std::function<bool(Register)> Match = [&](Register Reg) {
+ auto CValue = getIConstantVRegValWithLookThrough(Reg, MRI);
+ if (CValue.has_value())
+ return CValue->Value.isZero() || CValue->Value.isAllOnes();
+
+ auto *MI = getDefIgnoringCopies(Reg, MRI);
+ if (!MI)
+ return false;
+
+ switch (MI->getOpcode()) {
+ case TargetOpcode::G_TRUNC:
+ return Match(MI->getOperand(1).getReg());
+ case TargetOpcode::G_AND:
+ case TargetOpcode::G_OR:
+ case TargetOpcode::G_XOR:
+ return Match(MI->getOperand(1).getReg()) &&
+ Match(MI->getOperand(2).getReg());
+ case TargetOpcode::G_SELECT:
+ case PISA::G_PISA_SELECT:
+ return Match(MI->getOperand(2).getReg()) &&
+ Match(MI->getOperand(3).getReg());
+ default:
+ return false;
+ }
+ };
+
+ if (!Match(SrcReg))
+ return false;
+
+ MatchInfo = [DstReg, SrcReg, this](MachineIRBuilder &B) {
+ if (MRI.hasOneNonDBGUse(SrcReg)) {
+ auto *MI = getDefIgnoringCopies(SrcReg, MRI);
+ if (MI) {
+ MI->getOperand(0).setReg(DstReg);
+ return;
+ }
+ }
+
+ B.buildCopy(DstReg, SrcReg);
+ };
+ return true;
+}
+
+// %54:_(s8), %55:_(s8), %56:_(s8), %57:_(s8) = G_UNMERGE_VALUES %21:_(<4 x s8>)
+// %39:_(s16) = G_ANYEXT %54:_(s8)
+// %40:_(s16) = G_ANYEXT %55:_(s8)
+// %41:_(s16) = G_ADD %39:_, %40:_
+// %36:_(s16) = G_ANYEXT %56:_(s8)
+// %37:_(s16) = G_ANYEXT %57:_(s8)
+// %38:_(s16) = G_ADD %36:_, %37:_
+// %35:_(s16) = G_ADD %41:_, %38:_
+// => %40:_(s32) = G_INTRINSIC intrinsic(@llvm.pisa.dp4a.uu), 0, %21, 0x01010101
+// %35:_(s16) = G_TRUNC %40:_(s32)
+bool PISAPostLegalizerCombinerImpl::matchAddInt8Reduction(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+
+ auto Dst = MI.getOperand(0).getReg();
+ auto DstTy = MRI.getType(Dst);
+ if (!DstTy.isScalar() || DstTy.getSizeInBits() > 32)
+ return false;
+
+ std::array<Register, 4> Srcs;
+ // Helper to match anyext or zext of a register
+ auto MatchExt = [](Register &Reg) {
+ return m_any_of(m_GAnyExt(m_Reg(Reg)), m_GZExt(m_Reg(Reg)));
+ };
+ if (!mi_match(MI, MRI,
+ m_any_of(m_GAdd(m_OneNonDBGUse(m_GAdd(MatchExt(Srcs[0]),
+ MatchExt(Srcs[1]))),
+ m_OneNonDBGUse(m_GAdd(MatchExt(Srcs[2]),
+ MatchExt(Srcs[3])))),
+ m_GAdd(m_OneNonDBGUse(m_GAdd(
+ m_OneNonDBGUse(m_GAdd(MatchExt(Srcs[0]),
+ MatchExt(Srcs[1]))),
+ MatchExt(Srcs[2]))),
+ MatchExt(Srcs[3])))))
+ return false;
+
+ // All registers must be unique.
+ SmallSet<Register, 4> Unique(Srcs.begin(), Srcs.end());
+ if (Unique.size() < Srcs.size())
+ return false;
+
+ // All register must be i8 from the same unmerge instruction.
+ auto GetUnmerge = [=](Register &Reg) {
+ auto *UnmergeMI = MRI.getVRegDef(Reg);
+ return UnmergeMI->getOpcode() == TargetOpcode::G_UNMERGE_VALUES ? UnmergeMI
+ : nullptr;
+ };
+
+ MachineInstr *UnmergeMI = GetUnmerge(Srcs[0]);
+ if (!UnmergeMI)
+ return false;
+ for (unsigned I = 1; I < Srcs.size(); ++I) {
+ if (UnmergeMI != GetUnmerge(Srcs[I]))
+ return false;
+ }
+
+ auto VectorReg =
+ UnmergeMI->getOperand(UnmergeMI->getNumOperands() - 1).getReg();
+ if (MRI.getType(VectorReg) !=
+ LLT::vector(ElementCount::getFixed(4), LLT::integer(8)))
+ return false;
+
+ MatchInfo = [=](MachineIRBuilder &B) {
+ auto Acc = MRI.createGenericVirtualRegister(I32);
+ auto X = MRI.createGenericVirtualRegister(I32);
+ auto Y = MRI.createGenericVirtualRegister(I32);
+
+ B.buildConstant(Acc, 0);
+ B.buildBitcast(X, VectorReg);
+ B.buildConstant(Y, 0x01010101);
+
+ // Check if dst can be directly used, or result must be truncated.
+ if (DstTy == I32) {
+ B.buildIntrinsic(Intrinsic::pisa_dp4a_uu, {DstOp(Dst)})
+ .addUse(Acc)
+ .addUse(X)
+ .addUse(Y)
+ .addImm(0);
+ } else {
+ auto Dst32 = MRI.createGenericVirtualRegister(I32);
+ B.buildIntrinsic(Intrinsic::pisa_dp4a_uu, {DstOp(Dst32)})
+ .addUse(Acc)
+ .addUse(X)
+ .addUse(Y)
+ .addImm(0);
+ B.buildTrunc(Dst, Dst32);
+ }
+ };
+ return true;
+}
+
+// %370:_(i16), %371:_(i16) = G_UNMERGE_VALUES %366:_(<2 x i16>)
+// %378:_(f16) = G_BITCAST %370:_(i16)
+// %379:_(f16) = G_BITCAST %371:_(i16)
+// %37:_(<2 x f16>) = G_BUILD_VECTOR %378:_(f16), %379:_(f16)
+// => %37:_(<2 x f16>) = G_BITCAST %366:_(<2 x i16>)
+bool PISAPostLegalizerCombinerImpl::matchUnmergeBitcastBuildVectorToBitcast(
+ MachineInstr &MI, Register &MatchInfo) const {
+ Register UnmergeSrc;
+ for (unsigned I = 1, E = MI.getNumOperands(); I < E; I++) {
+ auto [SrcMI, SrcRegIdx] =
+ PISA::getDefIgnoringBitcasts(MI.getOperand(I).getReg(), MRI, true);
+ if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+ return false;
+ if (SrcMI->getNumOperands() != MI.getNumOperands())
+ return false;
+ auto SrcReg = SrcMI->getOperand(SrcMI->getNumOperands() - 1).getReg();
+ if (SrcRegIdx != (I - 1))
+ return false;
+ if (I == 1)
+ UnmergeSrc = SrcReg;
+ else if (UnmergeSrc != SrcReg)
+ return false;
+ }
+ MatchInfo = UnmergeSrc;
+ return true;
+}
+void PISAPostLegalizerCombinerImpl::applyUnmergeBitcastBuildVectorToBitcast(
+ MachineInstr &MI, Register &MatchInfo) const {
+ auto DstReg = MI.getOperand(0).getReg();
+ if (MRI.getType(DstReg) != MRI.getType(MatchInfo))
+ B.buildBitcast(MI.getOperand(0).getReg(), MatchInfo);
+ else
+ B.buildCopy(MI.getOperand(0).getReg(), MatchInfo);
+ MI.eraseFromParent();
+}
+
+// Return true if the G_FENCE corresponds to a subgroup or workitem scope.
+// These are execution barriers, not memory ordering fences, and should
+// be preserved.
+static bool isSubgroupFence(const MachineInstr &MI) {
+ auto ScopeID = static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
+ if (ScopeID == SyncScope::SingleThread)
+ return true;
+ auto &Ctx = MI.getMF()->getFunction().getContext();
+ if (auto Name = Ctx.getSyncScopeName(ScopeID))
+ return Name->starts_with("subgroup") || Name->starts_with("workitem");
+ return false;
+}
+
+// Extract the base scope (e.g. "workgroup") from a sync scope name that may
+// include an address space suffix (e.g. "workgroup-shared", "workgroup-global",
+// "workgroup-generic", or just "workgroup").
+static StringRef getBaseScopeName(const MachineInstr &MI) {
+ auto ScopeID = static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
+ auto &Ctx = MI.getMF()->getFunction().getContext();
+ auto ScopeName = Ctx.getSyncScopeName(ScopeID);
+ if (!ScopeName)
+ return StringRef();
+
+ for (StringRef Suffix : {"-shared", "-global", "-generic"})
+ if (ScopeName->ends_with(Suffix))
+ return ScopeName->drop_back(Suffix.size());
+ return *ScopeName;
+}
+
+bool PISAPostLegalizerCombinerImpl::matchRedundantFence(
+ MachineInstr &MI, MachineInstr *&PrevFence) const {
+ assert(MI.getOpcode() == TargetOpcode::G_FENCE);
+
+ if (isSubgroupFence(MI))
+ return false;
+
+ unsigned Scope = MI.getOperand(1).getImm();
+
+ auto It = MI.getIterator();
+ const MachineBasicBlock &MBB = *MI.getParent();
+ while (It != MBB.begin()) {
+ --It;
+ if (It->getOpcode() == TargetOpcode::G_FENCE) {
+ if (It->getOperand(1).getImm() == Scope) {
+ PrevFence = &*It;
+ return true;
+ }
+ // TODO: A wider scope subsumes a narrower one on the same addrspace.
+ // E.g. for `fence.global.gpu; fence.global.workgroup` we could drop
+ // the workgroup fence.
+ return false;
+ }
+ // TODO: Skip memory instructions with scope that do not interfere.
+ if (It->mayLoadOrStore() || It->hasUnmodeledSideEffects())
+ return false;
+ }
+ return false;
+}
+
+void PISAPostLegalizerCombinerImpl::applyRedundantFence(
+ MachineInstr &MI, MachineInstr *&PrevFence) const {
+ assert(PrevFence && "Expected a preceding compatible fence");
+
+ auto CurOrd = static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
+ auto PrevOrd = static_cast<AtomicOrdering>(PrevFence->getOperand(0).getImm());
+
+ // Pick the stronger ordering (e.g. choose seq_cst over acquire).
+ // Acquire and release cannot be compared; use acq_rel for the
+ // resulting fence.
+ auto Merged = getMergedAtomicOrdering(PrevOrd, CurOrd);
+ PrevFence->getOperand(0).setImm(static_cast<int64_t>(Merged));
+ MI.eraseFromParent();
+}
+
+// %28:_(<8 x i32>) = G_BUILD_VECTOR %91:_(i32), %92:_(i32), ...
+// %37:_(<4 x i32>) = G_EXTRACT_SUBVECTOR %28:_(<8 x i32>), 0
+// => %37:_(<4 x i32>) = G_BUILD_VECTOR %91:_(i32), %92:_(i32), ...
+bool PISAPostLegalizerCombinerImpl::matchExtractSubvectorBuildVector(
+ MachineInstr &MI, SmallVector<Register, 8> &MatchInfo) const {
+ assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
+ if (!MI.getOperand(2).isImm())
+ return false;
+ auto *BV = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ if (!BV || BV->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
+ return false;
+ uint64_t Offset = MI.getOperand(2).getImm();
+ unsigned NumDstElts = MRI.getType(MI.getOperand(0).getReg()).getNumElements();
+ if (NumDstElts != 4)
+ return false;
+ unsigned NumSrcElts = BV->getNumOperands() - 1; // operand 0 is the dst
+ if (Offset + NumDstElts > NumSrcElts)
+ return false;
+ MatchInfo.clear();
+ for (unsigned I = 0; I < NumDstElts; ++I)
+ MatchInfo.push_back(BV->getOperand(1 + Offset + I).getReg());
+ return true;
+}
+
+void PISAPostLegalizerCombinerImpl::applyExtractSubvectorBuildVector(
+ MachineInstr &MI, SmallVector<Register, 8> &MatchInfo) const {
+ Register DstReg = MI.getOperand(0).getReg();
+ B.setInstrAndDebugLoc(MI);
+ B.buildBuildVector(DstReg, MatchInfo);
+ MI.eraseFromParent();
+}
+
+// Match adjacent G_FENCE instructions that have the same memory ordering and
+// same base scope but differ only in address space. These can be merged into
+// a single generic-address-space fence.
+bool PISAPostLegalizerCombinerImpl::matchMergeAdjacentFences(
+ MachineInstr &MI, MachineInstr *&PrevFence) const {
+ assert(MI.getOpcode() == TargetOpcode::G_FENCE);
+
+ if (isSubgroupFence(MI))
+ return false;
+
+ auto Order = static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
+ auto BaseScopeNameA = getBaseScopeName(MI);
+ if (BaseScopeNameA.empty())
+ return false;
+
+ auto It = MI.getIterator();
+ const MachineBasicBlock &MBB = *MI.getParent();
+ while (It != MBB.begin()) {
+ --It;
+ if (It->getOpcode() == TargetOpcode::G_FENCE) {
+ auto ItOrder = static_cast<AtomicOrdering>(It->getOperand(0).getImm());
+ if (ItOrder != Order)
+ return false;
+
+ if (It->getOperand(1).getImm() == MI.getOperand(1).getImm())
+ return false;
+
+ auto BaseScopeNameIt = getBaseScopeName(*It);
+ if (BaseScopeNameIt.empty() || BaseScopeNameIt != BaseScopeNameA)
+ return false;
+
+ PrevFence = &*It;
+ return true;
+ }
+ if (It->mayLoadOrStore() || It->hasUnmodeledSideEffects())
+ return false;
+ }
+ return false;
+}
+
+// Merge two fences with same ordering and same base scope but different
+// address spaces into a single fence with generic address space.
+void PISAPostLegalizerCombinerImpl::applyMergeAdjacentFences(
+ MachineInstr &MI, MachineInstr *&PrevFence) const {
+ assert(PrevFence && "Expected a preceding compatible fence");
+
+ StringRef BaseScopeName = getBaseScopeName(*PrevFence);
+ assert(!BaseScopeName.empty() && "Expected a named sync scope");
+ std::string GenericName = (BaseScopeName + "-generic").str();
+ auto &Ctx = MI.getMF()->getFunction().getContext();
+ auto GenericID = Ctx.getOrInsertSyncScopeID(GenericName);
+ PrevFence->getOperand(1).setImm(static_cast<int64_t>(GenericID));
+
+ MI.eraseFromParent();
+}
+
+// %29:_(<64 x s32>) = IMPLICIT_DEF
+// %30:_(<64 x s32>) = G_INSERT_SUBVECTOR %29:_, %7:_(<8 x s32>), 0
+// %31:_(<64 x s32>) = G_INSERT_SUBVECTOR %30:_, %10:_(<8 x s32>), 8
+// %32:_(<64 x s32>) = G_INSERT_SUBVECTOR %31:_, %13:_(<8 x s32>), 16
+// %33:_(<64 x s32>) = G_INSERT_SUBVECTOR %32:_, %16:_(<8 x s32>), 24
+// %34:_(<64 x s32>) = G_INSERT_SUBVECTOR %33:_, %19:_(<8 x s32>), 32
+// %35:_(<64 x s32>) = G_INSERT_SUBVECTOR %34:_, %22:_(<8 x s32>), 40
+// %36:_(<64 x s32>) = G_INSERT_SUBVECTOR %35:_, %25:_(<8 x s32>), 48
+// %37:_(<64 x s32>) = G_INSERT_SUBVECTOR %36:_, %28:_(<8 x s32>), 56
+// %4:_(<32 x s32>) = G_EXTRACT_SUBVECTOR %37:_(<64 x s32>), 32
+// =>
+// %IMP:_(<32 x s32>) = IMPLICIT_DEF
+// %34:_(<32 x s32>) = G_INSERT_SUBVECTOR %IMP:_, %19:_(<8 x s32>), 0
+// %35:_(<32 x s32>) = G_INSERT_SUBVECTOR %34:_, %22:_(<8 x s32>), 8
+// %36:_(<32 x s32>) = G_INSERT_SUBVECTOR %35:_, %25:_(<8 x s32>), 16
+// %37:_(<32 x s32>) = G_INSERT_SUBVECTOR %36:_, %28:_(<8 x s32>), 24
+// %4:_(<32 x s32>) = COPY %37:_(<32 x s32>)
+bool PISAPostLegalizerCombinerImpl::matchExtractSubvectorPartial(
+ MachineInstr &MI, SmallVector<MachineInstr *, 8> &MatchInfo) const {
+ assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
+
+ Register SrcReg = MI.getOperand(1).getReg();
+ uint64_t ExtractIdx = MI.getOperand(2).getImm();
+ LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+ LLT SrcTy = MRI.getType(SrcReg);
+ unsigned DstNumElts = DstTy.getNumElements();
+
+ if (DstTy.getScalarSizeInBits() != 32)
+ return false;
+ if (DstNumElts < 8 || DstNumElts == SrcTy.getNumElements())
+ return false;
+
+ // Walk the chain of G_INSERT_SUBVECTOR instructions feeding the source.
+ // Collect inserts that are fully contained within the extracted range.
+ MatchInfo.clear();
+ MachineInstr *Cur = MRI.getVRegDef(SrcReg);
+ while (Cur && Cur->getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR) {
+ uint64_t InsIdx = Cur->getOperand(3).getImm();
+ Register InsSubReg = Cur->getOperand(2).getReg();
+ unsigned InsNumElts = MRI.getType(InsSubReg).getNumElements();
+
+ // Reject if insert starts before/extend past the extract window.
+ if (InsIdx < ExtractIdx && InsIdx + InsNumElts > ExtractIdx)
+ return false;
+ if (InsIdx >= ExtractIdx && InsIdx + InsNumElts > ExtractIdx + DstNumElts)
+ return false;
+
+ if (InsIdx >= ExtractIdx)
+ MatchInfo.push_back(Cur);
+ Cur = MRI.getVRegDef(Cur->getOperand(1).getReg());
+ }
+
+ // Base of the chain must be IMPLICIT_DEF and we need at least one insert.
+ if (MatchInfo.empty() || !Cur ||
+ Cur->getOpcode() != TargetOpcode::IMPLICIT_DEF)
+ return false;
+ return true;
+}
+
+void PISAPostLegalizerCombinerImpl::applyExtractSubvectorPartial(
+ MachineInstr &MI, const SmallVector<MachineInstr *, 8> &MatchInfo) const {
+ Register DstReg = MI.getOperand(0).getReg();
+ uint64_t ExtractIdx = MI.getOperand(2).getImm();
+ LLT DstTy = MRI.getType(DstReg);
+
+ // Rebuild in reverse order (innermost/earliest insert first).
+ B.setInstrAndDebugLoc(MI);
+ Register AccReg = B.buildUndef(DstTy).getReg(0);
+
+ for (const MachineInstr *Ins : llvm::reverse(MatchInfo)) {
+ Register InsSubReg = Ins->getOperand(2).getReg();
+ uint64_t InsIdx = Ins->getOperand(3).getImm();
+ uint64_t NewIdx = InsIdx - ExtractIdx;
+ AccReg = B.buildInsertSubvector(DstTy, AccReg, InsSubReg, NewIdx).getReg(0);
+ }
+
+ B.buildCopy(DstReg, AccReg);
+ MI.eraseFromParent();
+}
+
+// umin(x, UINT_MAX) -> x, umax(x, 0) -> x
+// smin(x, INT_MAX) -> x, smax(x, INT_MIN) -> x
+bool PISAPostLegalizerCombinerImpl::matchMinMaxIdentityFold(
+ MachineInstr &MI, Register &MatchInfo) const {
+ assert(MI.getOpcode() == TargetOpcode::G_UMIN ||
+ MI.getOpcode() == TargetOpcode::G_UMAX ||
+ MI.getOpcode() == TargetOpcode::G_SMIN ||
+ MI.getOpcode() == TargetOpcode::G_SMAX);
+ Register LHS = MI.getOperand(1).getReg();
+ Register RHS = MI.getOperand(2).getReg();
+ auto IsIdentity = [&](const APInt &Val) -> bool {
+ switch (MI.getOpcode()) {
+ case TargetOpcode::G_UMIN:
+ return Val.isAllOnes();
+ case TargetOpcode::G_UMAX:
+ return Val.isZero();
+ case TargetOpcode::G_SMIN:
+ return Val.isMaxSignedValue();
+ case TargetOpcode::G_SMAX:
+ return Val.isMinSignedValue();
+ default:
+ return false;
+ }
+ };
+ if (auto C = getIConstantVRegValWithLookThrough(RHS, MRI))
+ if (IsIdentity(C->Value)) {
+ MatchInfo = LHS;
+ return true;
+ }
+ if (auto C = getIConstantVRegValWithLookThrough(LHS, MRI))
+ if (IsIdentity(C->Value)) {
+ MatchInfo = RHS;
+ return true;
+ }
+ return false;
+}
+
+// Pass boilerplate
+// ================
+
+class PISAPostLegalizerCombiner : public MachineFunctionPass {
+ PISAPostLegalizerCombinerImplRuleConfig RuleConfig;
+
+public:
+ static char ID;
+
+ PISAPostLegalizerCombiner();
+
+ StringRef getPassName() const override { return "PISAPostLegalizerCombiner"; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+} // end anonymous namespace
+
+void PISAPostLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ getSelectionDAGFallbackAnalysisUsage(AU);
+ AU.addRequired<GISelValueTrackingAnalysisLegacy>();
+ AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
+ AU.addRequired<MachineDominatorTreeWrapperPass>();
+ AU.addPreserved<MachineDominatorTreeWrapperPass>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+PISAPostLegalizerCombiner::PISAPostLegalizerCombiner()
+ : MachineFunctionPass(ID) {
+ initializePISAPostLegalizerCombinerPass(*PassRegistry::getPassRegistry());
+ if (!RuleConfig.parseCommandLineOption())
+ report_fatal_error("Invalid rule identifier");
+}
+
+bool PISAPostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
+ if (MF.getProperties().hasProperty(
+ MachineFunctionProperties::Property::FailedISel))
+ return false;
+
+ const Function &F = MF.getFunction();
+ bool EnableOpt =
+ MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
+
+ GISelValueTracking *KB =
+ &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
+ MachineDominatorTree *MDT =
+ &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ CombinerInfo CInfo(
+ /*AllowIllegalOps=*/true, /*ShouldLegalizeIllegal=*/false,
+ /*LegalizerInfo=*/nullptr, EnableOpt, F.hasOptSize(), F.hasMinSize());
+
+ const PISASubtarget &STI = MF.getSubtarget<PISASubtarget>();
+ PISAPostLegalizerCombinerImpl Impl(MF, CInfo, *KB, /*CSEInfo=*/nullptr,
+ RuleConfig, STI, MDT,
+ STI.getLegalizerInfo());
+ return Impl.combineMachineInstrs();
+}
+
+char PISAPostLegalizerCombiner::ID = 0;
+INITIALIZE_PASS_BEGIN(PISAPostLegalizerCombiner, DEBUG_TYPE,
+ "Combine PISA machine instrs after legalization", false,
+ false)
+INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(PISAPostLegalizerCombiner, DEBUG_TYPE,
+ "Combine PISA machine instrs after legalization", false,
+ false)
+
+namespace llvm {
+FunctionPass *createPISAPostLegalizerCombiner() {
+ return new PISAPostLegalizerCombiner();
+}
+} // end namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp b/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
new file mode 100644
index 0000000000000..7a9cf7a4b3205
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
@@ -0,0 +1,1664 @@
+//===-- lib/CodeGen/GlobalISel/PISAPreLegalizerCombiner.cpp ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "PISA.h"
+#include "PISALegalizerInfo.h"
+#include "PISATargetMachine.h"
+#include "PISAUtils.h"
+#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
+#include "llvm/CodeGen/GlobalISel/Combiner.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define GET_GICOMBINER_DEPS
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_DEPS
+
+#define DEBUG_TYPE "pisa-prelegalizer-combiner"
+
+using namespace llvm;
+using namespace llvm::MIPatternMatch;
+
+namespace {
+
+#define GET_GICOMBINER_TYPES
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_TYPES
+
+class PISAPreLegalizerCombinerImpl : public Combiner {
+protected:
+ const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig;
+ const PISASubtarget &STI;
+ MachineDominatorTree *MDT;
+
+ // TODO: Make CombinerHelper methods const.
+ mutable CombinerHelper Helper;
+
+public:
+ PISAPreLegalizerCombinerImpl(
+ MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB,
+ GISelCSEInfo *CSEInfo,
+ const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig,
+ const PISASubtarget &STI, MachineDominatorTree *MDT,
+ const LegalizerInfo *LI);
+
+ static const char *getName() { return "PISAGenPreLegalizeGICombiner"; }
+
+ bool tryCombineAllImpl(MachineInstr &MI) const;
+ bool tryCombineAll(MachineInstr &I) const override;
+
+ void applyTruncatedLoad(MachineInstr &MI) const;
+
+ bool matchTruncatedStore(MachineInstr &MI) const;
+ void applyTruncatedStore(MachineInstr &MI) const;
+
+ bool matchExtendedLoad(MachineInstr &MI) const;
+ void applyExtendedLoad(MachineInstr &MI) const;
+
+ bool matchSimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const;
+ void applySimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const;
+
+ bool matchExpandNonPowerOf2LoadStore(MachineInstr &MI) const;
+ void applyExpandNonPowerOf2LoadStore(MachineInstr &MI) const;
+
+ bool matchTruncatedShift(MachineInstr &MI) const;
+ void applyTruncatedShift(MachineInstr &MI) const;
+
+ bool matchRedundantMovesPre(MachineInstr &MI) const;
+ void applyRedundantMovesPre(MachineInstr &MI) const;
+
+ bool
+ matchRcpSqrtToRsqrt(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+ bool
+ matchSubFloorToFrc(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchLaneIdLeftShiftChain(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchZExtAndToAndZExt(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchExtractInsertToBitcast(MachineInstr &MI, Register &) const;
+ void applyExtractInsertToBitcast(MachineInstr &MI, Register) const;
+
+ bool matchExtractBuildVectorToBitcast(MachineInstr &MI, Register &) const;
+ void applyExtractBuildVectorToBitcast(MachineInstr &MI, Register) const;
+
+ bool matchReducePredicates(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchCmpInt1(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchSelectTruncOneZero(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+private:
+#define GET_GICOMBINER_CLASS_MEMBERS
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_CLASS_MEMBERS
+};
+
+#define GET_GICOMBINER_IMPL
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_IMPL
+
+PISAPreLegalizerCombinerImpl::PISAPreLegalizerCombinerImpl(
+ MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB,
+ GISelCSEInfo *CSEInfo,
+ const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig,
+ const PISASubtarget &STI, MachineDominatorTree *MDT,
+ const LegalizerInfo *LI)
+ : Combiner(MF, CInfo, &KB, CSEInfo), RuleConfig(RuleConfig), STI(STI),
+ MDT(MDT), Helper(Observer, B, /*IsPreLegalize=*/true, &KB, MDT, LI),
+#define GET_GICOMBINER_CONSTRUCTOR_INITS
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_CONSTRUCTOR_INITS
+{
+}
+
+bool PISAPreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
+ // G_FPTRUNC to bf16 needs special constant folding because
+ // getFltSemanticForLLT() doesn't support BFloat16 yet.
+ if (MI.getOpcode() == TargetOpcode::G_FPTRUNC &&
+ MRI.getType(MI.getOperand(0).getReg()).getScalarType().isBFloat16()) {
+ const ConstantFP *Cst = nullptr;
+ if (mi_match(MI.getOperand(1).getReg(), MRI, m_GFCst(Cst))) {
+ APFloat Result(Cst->getValue());
+ bool Unused;
+ Result.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &Unused);
+ const ConstantFP *NewCst = ConstantFP::get(B.getContext(), Result);
+ MachineIRBuilder Builder(MI);
+ Builder.buildFConstant(MI.getOperand(0), *NewCst);
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ if (tryCombineAllImpl(MI))
+ return true;
+
+ return false;
+}
+
+void PISAPreLegalizerCombinerImpl::applyTruncatedLoad(MachineInstr &MI) const {
+ auto *LoadMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ auto *MMO = LoadMI->memoperands()[0];
+ auto Dst = MI.getOperand(0);
+ auto Addr = LoadMI->getOperand(1);
+ auto *NewMMO = MI.getMF()->getMachineMemOperand(MMO, MMO->getOffset(),
+ MRI.getType(Dst.getReg()));
+ B.buildLoad(Dst, Addr, *NewMMO);
+ MI.eraseFromParent();
+}
+
+bool PISAPreLegalizerCombinerImpl::matchExpandNonPowerOf2LoadStore(
+ MachineInstr &MI) const {
+ assert(MI.getOpcode() == TargetOpcode::G_LOAD ||
+ MI.getOpcode() == TargetOpcode::G_STORE);
+ GLoadStore &LS = cast<GLoadStore>(MI);
+
+ auto Size = LS.getMemSizeInBits().getValue();
+
+ if (isPowerOf2_32(Size))
+ return false;
+
+ if (LS.getMMO().getMemoryType().isVector())
+ return false;
+
+ // Check if this is a load and only has zext uses that are handled by the
+ // extended load pattern - if yes, we want to handle it via said pattern.
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ for (auto &Use : MRI.use_operands(MI.getOperand(0).getReg())) {
+ auto *Inst = Use.getParent();
+ if (Inst->getOpcode() != TargetOpcode::G_ZEXT ||
+ !matchExtendedLoad(*Inst))
+ return true;
+ }
+ return false;
+ }
+
+ return true;
+}
+void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
+ MachineInstr &MI) const {
+ assert(MI.getOpcode() == TargetOpcode::G_LOAD ||
+ MI.getOpcode() == TargetOpcode::G_STORE);
+
+ GLoadStore &LS = cast<GLoadStore>(MI);
+
+ auto PointerReg = LS.getPointerReg();
+ auto ValueReg = LS.getOperand(0).getReg();
+ auto &MMO = LS.getMMO();
+
+ /// The remaining size in Bits that still has to be loaded/stored
+ ssize_t Size = LS.getMemSizeInBits().getValue();
+ // Support sizes that are not a multiple of 8 by "promoting" them to the next
+ // multiple of 8. If the size is already a multiple of 8, it is not modified
+ Size = (Size + 7) & ~7;
+
+ unsigned Offset = 0;
+ const auto SizeTy = LLT::integer(Size);
+
+ // If the size was changed to the next power of 8 and we are storing a value,
+ // we need to modify the register's type as well.
+ // A similar check is needed for loads, but this is done at the end
+ if (MI.getOpcode() == TargetOpcode::G_STORE &&
+ SizeTy != MRI.getType(ValueReg)) {
+ auto NewValueReg = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildZExt(NewValueReg, ValueReg);
+ ValueReg = NewValueReg;
+ }
+
+ /// The register holding the loaded value at the end
+ Register LoadRes;
+ while (Size > 0) {
+ auto OpSize = bit_floor(static_cast<size_t>(Size));
+ const auto ShiftAmount = Offset * 8;
+
+ const LLT OpTy = LLT::integer(OpSize);
+
+ auto *NewMMO =
+ MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset() + Offset, OpTy);
+
+ /// Stores the (potentially modified) pointer register
+ auto AddrReg = PointerReg;
+ // We might need to change the store offset
+ if (Offset != 0) {
+ auto NewAddr = MRI.cloneVirtualRegister(AddrReg);
+
+ // Get the pointer size from the pointer register type
+ const LLT PtrTy = MRI.getType(AddrReg);
+ const LLT IntTy = LLT::integer(PtrTy.getSizeInBits());
+ auto Const = MRI.createGenericVirtualRegister(IntTy);
+ B.buildConstant(Const, Offset);
+ B.buildPtrAdd(NewAddr, AddrReg, Const);
+
+ AddrReg = NewAddr;
+ }
+
+ if (MI.getOpcode() == TargetOpcode::G_STORE) {
+ auto Res = ValueReg;
+ // If we're not storing from the start, we need to shift our value first
+ if (Offset != 0) {
+ auto ShrRes = MRI.createGenericVirtualRegister(SizeTy);
+ auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildConstant(ShiftConst, ShiftAmount);
+ B.buildLShr(ShrRes, ValueReg, ShiftConst);
+ Res = ShrRes;
+ }
+
+ // Next, truncate it to the correct size, if needed
+ if (SizeTy != OpTy) {
+ auto TruncRes = MRI.createGenericVirtualRegister(OpTy);
+ B.buildTrunc(TruncRes, Res);
+ Res = TruncRes;
+ }
+
+ B.buildStore(Res, AddrReg, *NewMMO);
+ } else {
+ // When loading, do the same thing but basically in reverse
+ // First, load the value
+ auto LoadedValReg = MRI.createGenericVirtualRegister(OpTy);
+ B.buildLoad(LoadedValReg, AddrReg, *NewMMO);
+ auto Res = LoadedValReg;
+
+ // Extend it to the correct size, if needed
+ if (SizeTy != OpTy) {
+ auto ZextRes = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildZExt(ZextRes, LoadedValReg);
+ Res = ZextRes;
+ }
+
+ // If we're not loading the first Bytes, then we need to shift it
+ if (Offset != 0) {
+ auto ShiftRes = MRI.createGenericVirtualRegister(SizeTy);
+
+ auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildConstant(ShiftConst, ShiftAmount);
+
+ B.buildShl(ShiftRes, Res, ShiftConst);
+ Res = ShiftRes;
+ }
+
+ if (!LoadRes.isValid()) {
+ LoadRes = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildConstant(LoadRes, 0);
+ }
+
+ auto NewLoadRes = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildOr(NewLoadRes, LoadRes, Res);
+ LoadRes = NewLoadRes;
+ }
+
+ Offset += OpSize / 8;
+ Size -= OpSize;
+ }
+
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ // If we modified the original size of the load (to support sizes that are
+ // not a multiple of 8), we need to truncate it to the correct size again,
+ // in order not to break any following instructions
+ if (SizeTy != MRI.getType(ValueReg)) {
+ auto TruncRes = MRI.createGenericVirtualRegister(MRI.getType(ValueReg));
+ B.buildTrunc(TruncRes, LoadRes);
+ LoadRes = TruncRes;
+ }
+ MRI.replaceRegWith(ValueReg, LoadRes);
+ }
+
+ MI.eraseFromParent();
+}
+
+/// If the value we store resulted from a G_LOAD that the rule above expanded,
+/// then we can use the individual load values directly instead of merging into
+/// one integer, and then splitting it again. Size modifications between the
+/// loads and store (i.e. SEXT/ZEXT/TRUNC) are also supported by
+/// truncating/extending the relevant load results (or, in the case of
+/// truncation, ignoring some loads entirely)
+///
+/// Below is a simple example where we simply load and store an i56:
+///
+/// bb.1.entry:
+/// %0:reg32b(p0) = functionParameter_32b 0
+/// -> %3:_(s56) = G_ZEXTLOAD %0:reg32b(p0) ::
+/// (load (s32) from %ir.dst)
+/// %4:_(s56) = G_CONSTANT i56 0
+/// %7:_(s32) = G_CONSTANT i32 4
+/// %6:reg32b(p0) = G_PTR_ADD %0:reg32b, %7:_(s32)
+/// -> %9:_(s56) = G_ZEXTLOAD %6:reg32b(p0) ::
+/// (load (s16) from %ir.dst + 4, align 4)
+/// %11:_(s56) = G_CONSTANT i56 32
+/// %10:_(s56) = G_SHL %9:_, %11:_(s56)
+/// %12:_(s56) = G_OR %3:_, %10:_
+/// %14:_(s32) = G_CONSTANT i32 6
+/// %13:reg32b(p0) = G_PTR_ADD %0:reg32b, %14:_(s32)
+/// -> %16:_(s56) = G_ZEXTLOAD %13:reg32b(p0) ::
+/// (load (s8) from %ir.dst + 6, align 2, basealign 4)
+/// %18:_(s56) = G_CONSTANT i56 48
+/// %17:_(s56) = G_SHL %16:_, %18:_(s56)
+/// %19:_(s56) = G_OR %12:_, %17:_
+/// G_STORE %19:_(s56), %0:reg32b(p0) ::
+/// (store (s56) into %ir.dst, align 4)
+/// ret
+///
+/// @param Loads stores all the load instructions that we find along the way.
+/// The second parameter in the pair is the offset from which the value was
+/// loaded
+/// @param SizeModificationOp stores the opcode to the instruction that modified
+/// the size of the integer after loading and before storing, if it exists.
+/// Possible opcode values are null (none), G_SEXT, G_SEXT_INREG, G_ZEXT, and
+/// G_TRUNC
+bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const {
+ GLoadStore &StoreInst = cast<GLoadStore>(MI);
+
+ auto ValueReg = StoreInst.getOperand(0).getReg();
+
+ /// This has the integer size at the beginning, and each individual load
+ /// decreases the value by its load size. At the end, this must be zero.
+ /// NB: We use the size of the store here, promoted to the next multiple
+ /// of 8. This size might be modified later if we find a G_*EXT/G_TRUNC.
+ unsigned ValueSize = StoreInst.getMemSize().getValue() * 8;
+
+ /// This stores the size of the last load instruction in Bytes, s.t. we can
+ /// verify that the current load is larger than the previous one
+ unsigned LastSize = 0;
+
+ SizeModificationOp = nullptr;
+
+ // First, we need to verify the entire "chain" of loads, s.t. we know that
+ // there are no further modifications to the value that we want to store.
+ // If there is a truncation/extension instruction in between, make note of
+ // that, and continue while verifying the load chain using the load size
+ // (i.e. size before *ext/trunc)
+ MachineInstr *NextChainInst = MRI.getVRegDef(ValueReg);
+ if (NextChainInst->getOpcode() == TargetOpcode::G_TRUNC ||
+ NextChainInst->getOpcode() == TargetOpcode::G_ZEXT ||
+ NextChainInst->getOpcode() == TargetOpcode::G_SEXT ||
+ NextChainInst->getOpcode() == TargetOpcode::G_SEXT_INREG) {
+ /// The "actual" size that was used during loading
+ unsigned LoadSize = MRI.getType(NextChainInst->getOperand(1).getReg())
+ .getScalarSizeInBits();
+
+ // We also use G_TRUNC in case the loaded integer was not a multiple of 8.
+ // In that case, ValueSize (which was already promoted to the next
+ // multiple of 8) would be equal to the total number of Bytes loaded from
+ // memory. If not, we need to remember this instruction, and change the
+ // expected size to the one that is actually used during loading.
+ if (ValueSize != LoadSize ||
+ NextChainInst->getOpcode() != TargetOpcode::G_TRUNC) {
+ SizeModificationOp = NextChainInst;
+ ValueSize = LoadSize;
+ }
+
+ // Either way, skip this instruction
+ NextChainInst = MRI.getVRegDef(NextChainInst->getOperand(1).getReg());
+
+ // If we loaded an integer that's not a multiple of 8, after which an
+ // *ext/trunc operation is done before storing, then there is another
+ // G_TRUNC in the way. Simply ignore that one.
+ // Example when doing load i38, zext -> i40, store i40:
+ // [...]
+ // %15:_(s38) = G_TRUNC %14:_(s40)
+ // %3:_(s40) = G_ZEXT %15:_(s38)
+ // G_STORE %3:_(s40), %1:reg32b(p0)
+ // [...]
+ if (NextChainInst->getOpcode() == TargetOpcode::G_TRUNC) {
+ if (LoadSize % 8 == 0)
+ return false;
+ ValueSize = MRI.getType(NextChainInst->getOperand(1).getReg())
+ .getScalarSizeInBits();
+ NextChainInst = MRI.getVRegDef(NextChainInst->getOperand(1).getReg());
+ }
+ }
+
+ while (ValueSize != 0) {
+ // We either start at an G_OR, or at a G_LOAD/G_ZEXTLOAD (meaning we're
+ // done)
+ if (NextChainInst->getOpcode() == TargetOpcode::G_OR) {
+ MachineInstr *NextOr, *ZextLoadMI;
+ int64_t LoadShift;
+ // First, match the Or-masking
+ if (!mi_match(NextChainInst, MRI,
+ m_GOr(m_MInstr(NextOr),
+ m_GShl(m_MInstr(ZextLoadMI), m_ICst(LoadShift)))))
+ return false;
+
+ if (ZextLoadMI->getOpcode() != TargetOpcode::G_ZEXTLOAD)
+ return false;
+
+ // The shift amount matches the offset used in the load instruction
+ MachineInstr *PtrAddInst =
+ MRI.getVRegDef(ZextLoadMI->getOperand(1).getReg());
+ if (PtrAddInst->getOpcode() != TargetOpcode::G_PTR_ADD)
+ return false;
+
+ auto LoadOffset =
+ getIConstantVRegVal(PtrAddInst->getOperand(2).getReg(), MRI);
+ if (!LoadOffset.has_value() || LoadOffset.value() != LoadShift / 8)
+ return false;
+
+ // Verify the LLT that was loaded
+ LLT LoadTy = cast<GZExtLoad>(ZextLoadMI)->getMMO().getType();
+ if (!LoadTy.isScalar() || LoadTy.getScalarSizeInBits() <= LastSize ||
+ ValueSize <= LastSize)
+ return false;
+
+ LastSize = LoadTy.getScalarSizeInBits();
+ ValueSize -= LastSize;
+ NextChainInst = NextOr;
+
+ Loads.push_back({ZextLoadMI, static_cast<unsigned>(
+ LoadOffset.value().getZExtValue())});
+ } else if (NextChainInst->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
+ NextChainInst->getOpcode() == TargetOpcode::G_LOAD) {
+ const auto &LoadInst = cast<GLoadStore>(*NextChainInst);
+ const LLT LoadTy = LoadInst.getMMO().getType();
+ if (!LoadTy.isScalar())
+ return false;
+
+ // This is the last load instruction of the chain, so it must match the
+ // remaining value in ValueSize
+ auto LoadSize = LoadTy.getScalarSizeInBits();
+ if (ValueSize % 8 == 0 && LoadSize != ValueSize)
+ return false;
+
+ // To avoid endless loops with single loads, sort out the bad cases here
+ if (Loads.empty()) {
+ // We just load and store a value, there is nothing to optimize here
+ if (!SizeModificationOp)
+ return false;
+
+ // There is no optimization potential for zero extensions or simple
+ // truncations, only for sign extension
+ if (SizeModificationOp->getOpcode() == TargetOpcode::G_ZEXT ||
+ SizeModificationOp->getOpcode() == TargetOpcode::G_TRUNC)
+ return false;
+
+ auto SizeAfterModificationOp =
+ MRI.getType(SizeModificationOp->getOperand(0).getReg())
+ .getScalarSizeInBits();
+
+ // If the size does not significantly change after sign extension, then
+ // there's nothing to do for us here, the code is already optimal
+ if ((isPowerOf2_32(SizeAfterModificationOp)
+ ? SizeAfterModificationOp
+ : NextPowerOf2(SizeAfterModificationOp)) == LoadSize)
+ return false;
+
+ // If the G_SEXT source size equals the load size and the source is
+ // byte-aligned, applying this transformation would reconstruct the
+ // same G_SEXT(G_LOAD) pattern, causing an infinite loop. When the
+ // source size is not byte-aligned, the apply inserts in-place
+ // shl/ashr to align it first, producing a different pattern.
+ auto SextSrcSize =
+ MRI.getType(SizeModificationOp->getOperand(1).getReg())
+ .getScalarSizeInBits();
+ if (LoadSize == SextSrcSize && SextSrcSize % 8 == 0)
+ return false;
+
+ // TODO: revisit
+ // store(sext i63 (load i8)) results in infinite loop.
+ if (!isPowerOf2_32(SizeAfterModificationOp) &&
+ (SizeAfterModificationOp != LoadSize))
+ return false;
+ }
+
+ // Done
+ Loads.push_back({NextChainInst, 0});
+ return true;
+ } else {
+ return false;
+ }
+ }
+ return false;
+}
+void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const {
+ // If the chain is valid, erase our single G_STORE and convert it into
+ // multiple G_STOREs using the individually loaded values
+ GLoadStore &StoreInst = cast<GLoadStore>(MI);
+
+ auto PointerReg = StoreInst.getPointerReg();
+ auto &MMO = StoreInst.getMMO();
+
+ unsigned StoreSizeInBytes = StoreInst.getMemSize().getValue();
+
+ for (auto [Index, Pair] : enumerate(Loads)) {
+ auto [LoadMI, Offset] = Pair;
+
+ Register Value = LoadMI->getOperand(0).getReg();
+ LLT LoadTy = cast<GLoadStore>(LoadMI)->getMMO().getType();
+
+ Register Res = Value;
+ if (LoadTy.getSizeInBytes() + Offset <= StoreSizeInBytes) {
+ if (LoadMI->getOpcode() == TargetOpcode::G_ZEXTLOAD) {
+ // Since they are ZextLoad instructions, we need to change the size
+ // back to the loaded size. This instruction will not be visible in
+ // PISA later, but is required here for type correctness.
+ Res = MRI.createGenericVirtualRegister(LoadTy);
+ B.buildTrunc(Res, Value);
+ }
+
+ // This is the last load (i.e. the first entry in the vector) and the
+ // load was extended, so we need to sext/zext this value before
+ // storing
+ if (Index == 0 && SizeModificationOp != nullptr) {
+ auto Opcode = SizeModificationOp->getOpcode();
+ assert(Opcode == TargetOpcode::G_ZEXT ||
+ Opcode == TargetOpcode::G_SEXT ||
+ Opcode == TargetOpcode::G_SEXT_INREG);
+
+ if (Opcode == TargetOpcode::G_SEXT_INREG ||
+ Opcode == TargetOpcode::G_SEXT) {
+ // SEXT is a little trickier
+ unsigned StoreSizeInBits = StoreInst.getMemSizeInBits().getValue();
+ unsigned OriginalSize =
+ Opcode == TargetOpcode::G_SEXT_INREG
+ ? (SizeModificationOp->getOperand(2).getImm())
+ : MRI.getType(SizeModificationOp->getOperand(1).getReg())
+ .getScalarSizeInBits();
+
+ // First, do we have to sign extend this value in-place using
+ // shifts?
+ if (OriginalSize % 8 != 0) {
+ auto ExtendBy = 8 - (OriginalSize % 8);
+
+ auto ConstReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ B.buildConstant(ConstReg, ExtendBy);
+
+ if (LoadTy.getSizeInBits() < OriginalSize + ExtendBy) {
+ LoadTy = LLT::integer(OriginalSize + ExtendBy);
+ auto ZExtRes = MRI.createGenericVirtualRegister(LoadTy);
+ B.buildZExt(ZExtRes, Res);
+ Res = ZExtRes;
+ }
+
+ auto ShlRes = MRI.cloneVirtualRegister(Res);
+ B.buildShl(ShlRes, Res, ConstReg);
+
+ auto AShrRes = MRI.cloneVirtualRegister(ShlRes);
+ B.buildAShr(AShrRes, ShlRes, ConstReg);
+
+ Res = AShrRes;
+ // Update the size to reflect the extension
+ OriginalSize = (OriginalSize + 7) & ~7;
+ }
+
+ // Second, do we still have to SEXT it further?
+ if (StoreSizeInBits > OriginalSize) {
+ LoadTy = LLT::integer(StoreSizeInBits);
+ auto SextRes = MRI.createGenericVirtualRegister(LoadTy);
+ B.buildSExt(SextRes, Res);
+
+ Res = SextRes;
+ }
+ } else {
+ // G_ZEXT, simple
+ auto NewSize = StoreSizeInBytes - Offset;
+ assert(NewSize > LoadTy.getSizeInBytes());
+
+ LoadTy = LLT::integer(NewSize * 8);
+ auto ExtRes = MRI.createGenericVirtualRegister(LoadTy);
+ B.buildZExt(ExtRes, Res);
+
+ Res = ExtRes;
+ }
+ }
+ } else {
+ assert(SizeModificationOp->getOpcode() == TargetOpcode::G_TRUNC);
+
+ // The value was truncated before storing. There are two cases now:
+ // - we need to truncate this value to the correct size
+ // - we need to ignore this one, as it is "out of range"
+ if (Offset >= StoreSizeInBytes) {
+ continue;
+ }
+ LoadTy = LLT::integer((StoreSizeInBytes - Offset) * 8);
+
+ Res = MRI.createGenericVirtualRegister(LoadTy);
+ B.buildTrunc(Res, Value);
+ }
+
+ auto *NewMMO = MI.getMF()->getMachineMemOperand(
+ &MMO, MMO.getOffset() + Offset, LoadTy);
+
+ /// Stores the (potentially modified) pointer register
+ auto AddrReg = PointerReg;
+
+ // Add the offset to the pointer reg if the offset is not zero
+ if (Offset != 0) {
+ auto NewPointerReg =
+ MRI.createGenericVirtualRegister(MRI.getType(AddrReg));
+
+ // Get the pointer size from the pointer register type
+ const LLT PtrTy = MRI.getType(AddrReg);
+ const LLT IntTy = LLT::integer(PtrTy.getSizeInBits());
+ auto CstReg = MRI.createGenericVirtualRegister(IntTy);
+ B.buildConstant(CstReg, Offset);
+
+ B.buildPtrAdd(NewPointerReg, AddrReg, CstReg);
+
+ AddrReg = NewPointerReg;
+ }
+ B.buildStore(Res, AddrReg, *NewMMO);
+ }
+
+ MI.eraseFromParent();
+ return;
+}
+
+// Matches following pattern:
+// (s24) = G_TRUNC (s32)
+// G_STORE (s24), ptr
+bool PISAPreLegalizerCombinerImpl::matchTruncatedStore(MachineInstr &MI) const {
+ auto &Store = cast<GStore>(MI);
+ auto *TruncMI = getDefIgnoringCopies(Store.getValueReg(), MRI);
+ if (!TruncMI)
+ return false;
+ if (TruncMI->getOpcode() != TargetOpcode::G_TRUNC)
+ return false;
+ if (!isPowerOf2_32(
+ MRI.getType(TruncMI->getOperand(1).getReg()).getSizeInBits()))
+ return false;
+
+ auto Size = Store.getMemSizeInBits().getValue();
+ auto Align = Store.getMMO().getAlign().value();
+
+ if (isPowerOf2_32(Size))
+ return false;
+
+ auto NextPow2 = NextPowerOf2(Size);
+ auto TruncSize = MRI.getType(TruncMI->getOperand(1).getReg()).getSizeInBits();
+ if (TruncSize != NextPow2)
+ return false;
+ auto Remainder = NextPow2 - Size;
+ if (NextPow2 % Remainder != 0)
+ return false;
+
+ if (Remainder % 8 != 0 || Remainder > 64 || !isPowerOf2_32(Remainder))
+ return false;
+
+ // We will create a vector store of smaller elements, which has to be aligned
+ // to the bit width of the vector element.
+ if (Align % (Remainder / 8) != 0)
+ return false;
+
+ auto VecSize = NextPow2 / Remainder;
+ if (VecSize > 4)
+ return false;
+
+ return true;
+}
+
+// Changes the type of a non-power-of-2 store to a vector of smaller elements,
+// if it comes from a trunc instruction.
+void PISAPreLegalizerCombinerImpl::applyTruncatedStore(MachineInstr &MI) const {
+ auto &Store = cast<GStore>(MI);
+
+ auto *TruncMI = getDefIgnoringCopies(Store.getValueReg(), MRI);
+ assert(TruncMI && TruncMI->getOpcode() == TargetOpcode::G_TRUNC);
+
+ auto TruncVal = TruncMI->getOperand(1);
+ auto Size = Store.getMemSizeInBits().getValue();
+ auto NextPow2 = NextPowerOf2(Size);
+ auto Remainder = NextPow2 - Size;
+ auto VecSize = NextPow2 / Remainder;
+ auto VecSizeSmall = Size / Remainder;
+ auto VecType = LLT::fixed_vector(VecSize, LLT::integer(Remainder));
+ auto VecTypeSmall = LLT::fixed_vector(VecSizeSmall, LLT::integer(Remainder));
+ auto TmpDst = MRI.createGenericVirtualRegister(VecType);
+ auto TmpDstSmall = MRI.createGenericVirtualRegister(VecTypeSmall);
+ B.buildBitcast(TmpDst, TruncVal);
+ B.buildShuffleVector(TmpDstSmall, TmpDst, B.buildUndef(VecType), {0, 1, 2});
+
+ auto Addr = Store.getPointerReg();
+ auto &MMO = Store.getMMO();
+ auto *NewMMO =
+ MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset(), VecTypeSmall);
+ B.buildStore(TmpDstSmall, Addr, *NewMMO);
+ MI.eraseFromParent();
+}
+
+// Matches a G_ZEXT that extends the result of a load that has a non-power-of-2
+// type. It might be profitable to do a wider load and mask out the bits.
+// This saves us from using second load instruction and few arithmetic
+// instructions (shl, and, or) to zero extend.
+bool PISAPreLegalizerCombinerImpl::matchExtendedLoad(MachineInstr &MI) const {
+ auto *DefMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ if (!DefMI)
+ return false;
+ if (DefMI->getOpcode() != TargetOpcode::G_LOAD)
+ return false;
+ GLoad &Load = cast<GLoad>(*DefMI);
+
+ auto Size = Load.getMemSizeInBits().getValue();
+ auto Align = Load.getMMO().getAlign().value();
+
+ if (isPowerOf2_32(Size))
+ return false;
+
+ auto ZextSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ if (!isPowerOf2_32(ZextSize))
+ return false;
+
+ if (ZextSize > 64)
+ return false;
+
+ if (Align % (ZextSize / 8) != 0)
+ return false;
+
+ return true;
+}
+
+void PISAPreLegalizerCombinerImpl::applyExtendedLoad(MachineInstr &MI) const {
+ GLoad *LoadMI =
+ cast<GLoad>(getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI));
+ assert(LoadMI);
+
+ auto DestSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ auto LoadSize = LoadMI->getMemSizeInBits().getValue();
+
+ APInt Mask = APInt::getLowBitsSet(DestSize, LoadSize);
+
+ auto NewLoadSize = LLT::integer(DestSize);
+ auto LoadDst = MRI.createGenericVirtualRegister(NewLoadSize);
+
+ auto Addr = LoadMI->getOperand(1);
+ auto &MMO = LoadMI->getMMO();
+ auto *NewMMO = MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset(),
+ MRI.getType(LoadDst));
+
+ B.buildLoad(LoadDst, Addr, *NewMMO);
+ auto MaskReg = MRI.createGenericVirtualRegister(LLT::integer(DestSize));
+ B.buildConstant(MaskReg, Mask.getZExtValue());
+ B.buildAnd(MI.getOperand(0).getReg(), LoadDst, MaskReg);
+ MI.eraseFromParent();
+}
+
+// i32 %lo = G_TRUNC i32 a 16
+// i32 %shift = G_LSHR i32 a 16
+// %hi = G_TRUNC i32 %shift to i16
+// => %1 = G_BITCAST i32 a to <2 x 16>
+// => %lo = G_EXTRACT_VECTOR_ELT <2 x 16> %1, 0
+// => %hi = G_EXTRACT_VECTOR_ELT <2 x 16> %1, 1
+bool PISAPreLegalizerCombinerImpl::matchTruncatedShift(MachineInstr &MI) const {
+ auto &TruncMI = MI;
+ auto &LshMI = *getDefIgnoringCopies(TruncMI.getOperand(1).getReg(), MRI);
+ if (LshMI.getOpcode() == TargetOpcode::G_LSHR) {
+ auto WideTy = MRI.getType(TruncMI.getOperand(1).getReg());
+ auto NarrowTy = MRI.getType(TruncMI.getOperand(0).getReg());
+ auto WideSize = WideTy.getScalarSizeInBits();
+ auto NarrowSize = NarrowTy.getScalarSizeInBits();
+ if (WideTy.isScalar() && NarrowTy.isScalar()) {
+ if (isPowerOf2_32(WideSize) && isPowerOf2_32(NarrowSize)) {
+ auto ShiftReg = LshMI.getOperand(2).getReg();
+ if (auto Shift = getIConstantVRegValWithLookThrough(ShiftReg, MRI)) {
+ if (Shift.has_value()) {
+ auto ShiftVal = Shift->Value.getZExtValue();
+ if ((NarrowSize + ShiftVal) == WideSize)
+ return true;
+ }
+ }
+ }
+ }
+ }
+ return false;
+}
+void PISAPreLegalizerCombinerImpl::applyTruncatedShift(MachineInstr &MI) const {
+ auto &TruncMI = MI;
+ auto &LshMI = *getDefIgnoringCopies(TruncMI.getOperand(1).getReg(), MRI);
+ auto WideTy = MRI.getType(TruncMI.getOperand(1).getReg());
+ auto NarrowTy = MRI.getType(TruncMI.getOperand(0).getReg());
+ auto WideSize = WideTy.getScalarSizeInBits();
+ auto NarrowSize = NarrowTy.getScalarSizeInBits();
+
+ auto VecLen = WideSize / NarrowSize;
+ auto VecTy = LLT::fixed_vector(VecLen, NarrowTy);
+ auto VecReg = MRI.createGenericVirtualRegister(VecTy);
+
+ // trunc to extract high part
+ auto BitCast = B.buildBitcast(VecReg, LshMI.getOperand(1));
+ B.buildExtractVectorElementConstant(TruncMI.getOperand(0), VecReg,
+ VecLen - 1);
+
+ // find trunc to extract low part (if any)
+ auto ShiftSrc = LshMI.getOperand(1).getReg();
+ for (auto &UseMI : MRI.use_instructions(ShiftSrc)) {
+ if (UseMI.getOpcode() == TargetOpcode::G_TRUNC) {
+ auto DstSize = MRI.getType(UseMI.getOperand(0).getReg()).getSizeInBits();
+ if (DstSize == NarrowSize) {
+ assert(MDT && "machine dominator pass must be available");
+ if (!MDT->dominates(&UseMI, &TruncMI) &&
+ !MDT->dominates(&TruncMI, &UseMI))
+ continue; // can not optimize out low part
+ MachineIRBuilder MIB(UseMI);
+ auto Lo = MIB.buildExtractVectorElementConstant(UseMI.getOperand(0),
+ VecReg, 0);
+ if (MDT->dominates(&UseMI, &TruncMI))
+ BitCast->moveBefore(Lo);
+ UseMI.eraseFromParent();
+ }
+ }
+ }
+ TruncMI.eraseFromParent();
+}
+
+// A(16) = G_EXTRACT_VECTOR_ELT ARG(32), 0
+// B(16) = G_EXTRACT_VECTOR_ELT ARG(32), 1
+// C(<2x16>) = G_BUILD_VECTOR A, B
+// D(32) = G_BITCAST C(<2x16>)
+// => D(32) = COPY ARG(32)
+bool PISAPreLegalizerCombinerImpl::matchRedundantMovesPre(
+ MachineInstr &MI) const {
+ auto &BitcastMI = MI;
+ auto DstReg = BitcastMI.getOperand(0).getReg();
+ auto SrcReg = BitcastMI.getOperand(1).getReg();
+ if (MRI.getType(DstReg).isVector() || !MRI.getType(SrcReg).isVector())
+ return false;
+
+ auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ if (BuildVecMI.getOpcode() != TargetOpcode::G_BUILD_VECTOR)
+ return false;
+
+ unsigned Mask = 0;
+ Register VecReg = 0;
+ for (unsigned I = 1; I < BuildVecMI.getNumOperands(); I++) {
+ auto &ExtractMI =
+ *getDefIgnoringCopies(BuildVecMI.getOperand(I).getReg(), MRI);
+ if (ExtractMI.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
+ return false;
+ if (I == 1) {
+ VecReg = ExtractMI.getOperand(1).getReg();
+ } else if (VecReg != ExtractMI.getOperand(1).getReg()) {
+ // must extract indices from the same vector
+ return false;
+ }
+ auto IndexReg = ExtractMI.getOperand(2).getReg();
+ auto Index = getIConstantVRegValWithLookThrough(IndexReg, MRI);
+ if (!Index.has_value())
+ return false;
+ // indices must be in the same order
+ if (Index->Value.getZExtValue() != (I - 1))
+ return false;
+ Mask |= 1 << Index->Value.getZExtValue();
+ }
+ // check that all indices have been extracted
+ if (Mask != (1u << MRI.getType(SrcReg).getNumElements()) - 1)
+ return false;
+
+ // do not consider creating <2x16> from e.g. <3x16>
+ if (MRI.getType(DstReg).getSizeInBits() !=
+ MRI.getType(VecReg).getSizeInBits())
+ return false;
+
+ return true;
+}
+void PISAPreLegalizerCombinerImpl::applyRedundantMovesPre(
+ MachineInstr &MI) const {
+ auto &BitcastMI = MI;
+ auto DstReg = BitcastMI.getOperand(0).getReg();
+ auto SrcReg = BitcastMI.getOperand(1).getReg();
+ auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ auto &ExtractMI =
+ *getDefIgnoringCopies(BuildVecMI.getOperand(1).getReg(), MRI);
+ auto VecReg = ExtractMI.getOperand(1).getReg();
+
+ if (MRI.getType(DstReg) == MRI.getType(VecReg))
+ B.buildCopy(DstReg, VecReg);
+ else
+ B.buildBitcast(DstReg, VecReg);
+ MI.eraseFromParent();
+}
+
+/// Returns divisor if operation is frcp intrinsic or fdiv with dividend equal
+/// to constant one.
+static MachineInstr *getReciprocalDivisor(MachineInstr *MI,
+ MachineRegisterInfo &MRI) {
+ if (MI->getOpcode() == TargetOpcode::G_FDIV) {
+ // Check for 1.0 / x pattern. Avoid m_GFCstOrSplat because
+ // getFConstantVRegValWithLookThrough loses bfloat16 semantics.
+ Register Dividend = MI->getOperand(1).getReg();
+ auto *DivDefMI = getDefIgnoringCopies(Dividend, MRI);
+ if (!DivDefMI || DivDefMI->getOpcode() != TargetOpcode::G_FCONSTANT)
+ return nullptr;
+ if (!DivDefMI->getOperand(1).getFPImm()->isExactlyValue(1.0))
+ return nullptr;
+ return getDefIgnoringCopies(MI->getOperand(2).getReg(), MRI);
+ }
+
+ auto *GI = dyn_cast<GIntrinsic>(MI);
+ if (GI && GI->is(Intrinsic::pisa_frcp))
+ return getDefIgnoringCopies(MI->getOperand(2).getReg(), MRI);
+
+ return nullptr;
+}
+
+// 1/(sqrt(x)) -> frsqrt(x)
+// frcp(sqrt(x)) -> frsqrt(x)
+// sqrt(1/(x)) -> frsqrt(x)
+// sqrt(frcp(x)) -> frsqrt(x)
+// 1/(fabs(sqrt(x))) -> fabs(frsqrt(x))
+// frcp(fabs(sqrt(x))) -> fabs(frsqrt(x))
+bool PISAPreLegalizerCombinerImpl::matchRcpSqrtToRsqrt(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ auto GetFrcpSrc = [=](MachineInstr *MI) -> MachineInstr * {
+ if (!MI || !MI->getFlag(MachineInstr::FmContract))
+ return nullptr;
+ return getReciprocalDivisor(MI, MRI);
+ };
+ auto GetSqrtSrc = [=](MachineInstr *MI) -> MachineInstr * {
+ if (!MI || !MI->getFlag(MachineInstr::FmContract))
+ return nullptr;
+ if (MI->getOpcode() == TargetOpcode::G_FSQRT)
+ return getDefIgnoringCopies(MI->getOperand(1).getReg(), MRI);
+ return nullptr;
+ };
+
+ bool WrapInFAbs = false;
+ MachineInstr *Divisor = GetFrcpSrc(&MI);
+ if (Divisor) {
+ // result of sqrt cannot be snan, so Intrinsic::fabs is replaced with
+ // Intrinsic::pisa_fabs. We won't get G_FABS here.
+ auto *GI = dyn_cast<GIntrinsic>(Divisor);
+ if (GI && GI->is(Intrinsic::pisa_fabs)) {
+ WrapInFAbs = true;
+ Divisor = getDefIgnoringCopies(GI->getOperand(2).getReg(), MRI);
+ }
+ }
+
+ MachineInstr *InnerMI = GetSqrtSrc(Divisor);
+ if (!InnerMI) {
+ WrapInFAbs = false;
+ InnerMI = GetFrcpSrc(GetSqrtSrc(&MI));
+ }
+ if (!InnerMI)
+ return false;
+
+ LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+ MatchInfo = [InnerMI, WrapInFAbs, DstTy, &MI](MachineIRBuilder &B) {
+ Register Src = InnerMI->getOperand(0).getReg();
+ if (!WrapInFAbs) {
+ B.buildIntrinsic(Intrinsic::pisa_frsqrt, {MI.getOperand(0)})
+ .addUse(Src)
+ .setMIFlags(MI.getFlags());
+ return;
+ }
+ // 1/|sqrt(x)| == |frsqrt(x)|
+ auto Frsqrt = B.buildIntrinsic(Intrinsic::pisa_frsqrt, {DstTy})
+ .addUse(Src)
+ .setMIFlags(MI.getFlags());
+ B.buildIntrinsic(Intrinsic::pisa_fabs, {MI.getOperand(0)})
+ .addUse(Frsqrt.getReg(0))
+ .setMIFlags(MI.getFlags());
+ };
+ return true;
+}
+
+// s32 %floor = G_FFLOOR %x
+// s32 %dst = G_FSUB %x, %floor
+// => s32 %dst = frc %x
+bool PISAPreLegalizerCombinerImpl::matchSubFloorToFrc(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ Register XReg, FloorReg;
+
+ bool IsPlainFSub = MI.getOpcode() == TargetOpcode::G_FSUB;
+ if (IsPlainFSub) {
+ XReg = MI.getOperand(1).getReg();
+ FloorReg = MI.getOperand(2).getReg();
+ } else if (auto *GI = dyn_cast<GIntrinsic>(&MI);
+ GI && GI->is(Intrinsic::pisa_fsub)) {
+ unsigned NumOps = MI.getNumOperands();
+ auto Round = static_cast<RoundingMode>(MI.getOperand(NumOps - 2).getImm());
+ int64_t Sat = MI.getOperand(NumOps - 1).getImm();
+ if (Round != RoundingMode::TowardZero || Sat)
+ return false;
+ XReg = MI.getOperand(2).getReg();
+ FloorReg = MI.getOperand(3).getReg();
+ } else {
+ return false;
+ }
+
+ auto BitWidth = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ if (BitWidth != 32)
+ return false;
+
+ MachineInstr *FloorMI = getDefIgnoringCopies(FloorReg, MRI);
+ if (!FloorMI || FloorMI->getOpcode() != TargetOpcode::G_FFLOOR)
+ return false;
+
+ if (IsPlainFSub && (!MI.getFlag(MachineInstr::FmContract) ||
+ !FloorMI->getFlag(MachineInstr::FmContract)))
+ return false;
+
+ if (FloorMI->getOperand(1).getReg() != XReg)
+ return false;
+
+ MatchInfo = [&MI, XReg](MachineIRBuilder &B) {
+ B.buildIntrinsic(Intrinsic::pisa_frc, {MI.getOperand(0)})
+ .addUse(XReg)
+ .setMIFlags(MI.getFlags());
+ };
+ return true;
+}
+
+// Fold `shl (zext (shl lane_id, S1)), S2` to `zext (shl lane_id, S1+S2)`.
+// lane_id currently fits in 5 bits.
+// The second shift could only appear from irtranslator pass.
+//
+// Pattern:
+// %5:_(s32) = G_CONSTANT i32 3
+// %4:_(s32) = G_INTRINSIC intrinsic(@llvm.pisa.lane.id)
+// %6:_(s32) = nuw nsw G_SHL %4:_, %5:_(s32)
+// %7:_(s64) = nneg G_ZEXT %6:_(s32)
+// %31:_(s64) = G_CONSTANT i64 2
+// %9:_(s64) = nuw nsw G_SHL %7:_, %31:_(s64)
+// =>
+// %4:_(s32) = G_INTRINSIC intrinsic(@llvm.pisa.lane.id)
+// %5:_(s32) = G_CONSTANT i32 5
+// %6:_(s32) = nuw nsw G_SHL %4:_, %5:_(s32)
+// %7:_(s64) = nneg G_ZEXT %6:_(s32)
+bool PISAPreLegalizerCombinerImpl::matchLaneIdLeftShiftChain(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ int64_t ShiftImm1, ShiftImm2;
+ MachineInstr *ZExt = nullptr;
+ MachineInstr *Intr = nullptr;
+ if (!mi_match(MI, MRI, m_GShl(m_MInstr(ZExt), m_ICst(ShiftImm1))))
+ return false;
+ if (!mi_match(ZExt, MRI, m_GZExt(m_GShl(m_MInstr(Intr), m_ICst(ShiftImm2)))))
+ return false;
+ auto *LaneId = dyn_cast<GIntrinsic>(Intr);
+ if (!LaneId || LaneId->getIntrinsicID() != Intrinsic::pisa_lane_id)
+ return false;
+
+ auto LaneIdTy = MRI.getType(LaneId->getOperand(0).getReg());
+ constexpr int MaxNumBitsInLaneId = 5;
+ if ((ShiftImm1 + ShiftImm2) >=
+ (static_cast<int>(LaneIdTy.getSizeInBits()) - MaxNumBitsInLaneId - 1))
+ return false;
+
+ MatchInfo = [=, &MI](MachineIRBuilder &B) {
+ auto NewShlReg = MRI.createGenericVirtualRegister(LaneIdTy);
+ B.buildShl(NewShlReg, LaneId->getOperand(0),
+ B.buildConstant(LaneIdTy, ShiftImm1 + ShiftImm2), MI.getFlags());
+ B.buildZExt(MI.getOperand(0), NewShlReg, ZExt->getFlags());
+ };
+ return true;
+}
+
+// G_AND (G_ZEXT x), C -> G_ZEXT (G_AND x, trunc(C)).
+// CodeGenPrepare can form the widened mask, but keeping the mask narrow lets
+// address folding see the original arithmetic shape.
+bool PISAPreLegalizerCombinerImpl::matchZExtAndToAndZExt(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ Register ZExtReg = MI.getOperand(1).getReg();
+ Register MaskReg = MI.getOperand(2).getReg();
+ auto *ZExtMI = getDefIgnoringCopies(ZExtReg, MRI);
+ auto Mask = getIConstantVRegValWithLookThrough(MaskReg, MRI);
+
+ if ((!ZExtMI || ZExtMI->getOpcode() != TargetOpcode::G_ZEXT ||
+ !Mask.has_value())) {
+ std::swap(ZExtReg, MaskReg);
+ ZExtMI = getDefIgnoringCopies(ZExtReg, MRI);
+ Mask = getIConstantVRegValWithLookThrough(MaskReg, MRI);
+ }
+
+ if (!ZExtMI || ZExtMI->getOpcode() != TargetOpcode::G_ZEXT ||
+ !Mask.has_value())
+ return false;
+
+ Register DstReg = MI.getOperand(0).getReg();
+ Register SrcReg = ZExtMI->getOperand(1).getReg();
+ LLT DstTy = MRI.getType(DstReg);
+ LLT SrcTy = MRI.getType(SrcReg);
+ if (!DstTy.isScalar() || !SrcTy.isScalar() || SrcTy.getSizeInBits() != 32 ||
+ DstTy.getSizeInBits() != 64)
+ return false;
+
+ APInt NarrowMask = Mask->Value.zextOrTrunc(SrcTy.getSizeInBits());
+ unsigned ZExtFlags = ZExtMI->getFlags();
+ MatchInfo = [=](MachineIRBuilder &B) {
+ auto NarrowMaskReg = B.buildConstant(SrcTy, NarrowMask);
+ auto NarrowAnd = B.buildAnd(SrcTy, SrcReg, NarrowMaskReg);
+ B.buildZExt(DstReg, NarrowAnd, ZExtFlags);
+ };
+ return true;
+}
+
+// Given the following pattern
+//
+// %144:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<32 x s8>), %145:_(s32)
+// %146:_(s32) = G_ZEXT %144:_(s8)
+// %147:_(s32) = nuw G_SHL %146:_, %23:_(s32)
+// %148:_(s32) = disjoint G_OR %143:_, %147:
+//
+// record the following info using BaseReg (%148) as input
+// - SrcVecReg = %4
+// - SrcVecIdx = 1 << %145
+// - bits of s32 (typeof(BaseReg)) written to by above - return value
+static uint64_t getCoveredBits(Register BaseReg, Register &SrcVecReg,
+ uint64_t *SrcVecIdx, MachineRegisterInfo &MRI) {
+ auto &SrcMI = *getDefIgnoringCopies(BaseReg, MRI);
+ switch (SrcMI.getOpcode()) {
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
+ auto VecReg = SrcMI.getOperand(1).getReg();
+ if (MRI.getType(VecReg).getNumElements() > 64)
+ return 0; // not supported
+ auto IdxReg = SrcMI.getOperand(2).getReg();
+ if (SrcVecReg && (SrcVecReg != VecReg))
+ return 0; // extracting from different source vector ?
+ SrcVecReg = VecReg;
+ auto Index = getIConstantVRegValWithLookThrough(IdxReg, MRI);
+ if (!Index.has_value())
+ return 0; // not a constant
+ *SrcVecIdx |= (1ull << Index->Value.getZExtValue());
+ return ~0;
+ } break;
+ case TargetOpcode::G_ZEXT: {
+ auto SrcReg = SrcMI.getOperand(1).getReg();
+ auto &ExtractMI = *getDefIgnoringCopies(SrcReg, MRI);
+ if (ExtractMI.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
+ return 0;
+ if (getCoveredBits(SrcReg, SrcVecReg, SrcVecIdx, MRI))
+ return (1ull << MRI.getType(SrcReg).getScalarSizeInBits()) - 1;
+ } break;
+ case TargetOpcode::G_SHL: {
+ auto SrcReg = SrcMI.getOperand(1).getReg();
+ auto ShiftReg = SrcMI.getOperand(2).getReg();
+ auto &ExtMI = *getDefIgnoringCopies(SrcReg, MRI);
+ if (ExtMI.getOpcode() != TargetOpcode::G_ZEXT)
+ return 0;
+ auto Shift = getIConstantVRegValWithLookThrough(ShiftReg, MRI);
+ if (!Shift.has_value())
+ return 0; // not a constant
+ auto ShiftValue = Shift->Value.getZExtValue();
+ // make sure we are not swapping bits
+ auto OldSrcVecIdx = *SrcVecIdx;
+ auto Bits = getCoveredBits(SrcReg, SrcVecReg, SrcVecIdx, MRI);
+ auto BitSet = llvm::countr_zero(*SrcVecIdx & ~OldSrcVecIdx);
+ auto Offset = (BitSet * MRI.getType(SrcVecReg).getScalarSizeInBits()) %
+ MRI.getType(SrcReg).getSizeInBits();
+ if (ShiftValue != Offset)
+ return 0; // will not insert at proper offset
+ return Bits << ShiftValue;
+ } break;
+ case TargetOpcode::G_OR: {
+ auto LHSReg = SrcMI.getOperand(1).getReg();
+ auto RHSReg = SrcMI.getOperand(2).getReg();
+ auto LHSOp = getDefIgnoringCopies(LHSReg, MRI)->getOpcode();
+ auto RHSOp = getDefIgnoringCopies(RHSReg, MRI)->getOpcode();
+ if ((LHSOp != TargetOpcode::G_SHL) && (LHSOp != TargetOpcode::G_ZEXT) &&
+ (LHSOp != TargetOpcode::G_OR))
+ return 0;
+ if ((RHSOp != TargetOpcode::G_SHL) && (RHSOp != TargetOpcode::G_ZEXT) &&
+ (RHSOp != TargetOpcode::G_OR))
+ return 0;
+ auto LHSBits = getCoveredBits(LHSReg, SrcVecReg, SrcVecIdx, MRI);
+ auto RHSBits = getCoveredBits(RHSReg, SrcVecReg, SrcVecIdx, MRI);
+ if (LHSBits && RHSBits)
+ return LHSBits | RHSBits;
+ } break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+bool PISAPreLegalizerCombinerImpl::matchExtractInsertToBitcast(
+ MachineInstr &MI, Register &SaveReg) const {
+ auto &BuildVectorMI = MI;
+ auto DstReg = BuildVectorMI.getOperand(0).getReg();
+ auto BuildVectorTy = MRI.getType(DstReg);
+ auto NumElts = BuildVectorTy.getNumElements();
+
+ Register SrcVecReg; // G_EXTRACT_VECTOR_ELT
+ uint64_t SrcVecIdx = 0; // index of G_EXTRACT_VECTOR_ELT
+ for (unsigned I = 0; I < NumElts; I++) {
+ auto BuildVectorEltReg = BuildVectorMI.getOperand(I + 1).getReg();
+ auto Bits = getCoveredBits(BuildVectorEltReg, SrcVecReg, &SrcVecIdx, MRI);
+ if (!Bits)
+ return false; // did not match
+ if (BuildVectorTy.getSizeInBits() != MRI.getType(SrcVecReg).getSizeInBits())
+ return false;
+ if (!((1ull << I) & SrcVecIdx))
+ return false; // indices are not in ascending order
+ auto EltSize = BuildVectorTy.getScalarSizeInBits();
+ uint64_t Mask = (EltSize == 64) ? (uint64_t)-1ll : (1ull << EltSize) - 1;
+ if (Mask != Bits)
+ return false; // did not cover full element
+ }
+ auto SrcVecTy = MRI.getType(SrcVecReg);
+ uint64_t Mask = (1ull << SrcVecTy.getNumElements()) - 1;
+ if (Mask != SrcVecIdx)
+ return false; // did not extract all indices
+
+ SaveReg = SrcVecReg;
+ return true;
+}
+void PISAPreLegalizerCombinerImpl::applyExtractInsertToBitcast(
+ MachineInstr &MI, Register SrcVecReg) const {
+ auto DstReg = MI.getOperand(0).getReg();
+ if (MRI.getType(DstReg) == MRI.getType(SrcVecReg))
+ B.buildCopy(DstReg, SrcVecReg);
+ else
+ B.buildBitcast(DstReg, SrcVecReg);
+ MI.eraseFromParent();
+}
+
+// %23:_(<4 x s8>) = G_BITCAST %1:_(s32)
+// %2:_(s8) = G_EXTRACT_VECTOR_ELT %23:_(<4 x s8>), %25:_(s32)
+// %7:_(s32) = G_LSHR %1:_, %6:_(s32)
+// %8:_(s8) = G_TRUNC %7:_(s32)
+// %12:_(s32) = G_LSHR %1:_, %11:_(s32)
+// %13:_(s8) = G_TRUNC %12:_(s32)
+// %24:_(s32) = G_CONSTANT i32 3
+// %18:_(s8) = G_EXTRACT_VECTOR_ELT %23:_(<4 x s8>), %24:_(s32)
+// %19:_(<4 x s8>) = G_BUILD_VECTOR %2:_(s8), %8:_(s8), %13:_(s8), %18:_(s8)
+// => %19:_(<4 x s8>) = G_BITCAST %1:_(s32)
+bool PISAPreLegalizerCombinerImpl::matchExtractBuildVectorToBitcast(
+ MachineInstr &MI, Register &SaveReg) const {
+ auto &BuildMI = MI;
+ auto DstTy = MRI.getType(BuildMI.getOperand(0).getReg());
+
+ Register CastSrcReg; // G_BITCAST %1
+ for (unsigned I = 1; I < BuildMI.getNumOperands(); I++) {
+ auto EltReg = BuildMI.getOperand(I).getReg();
+ auto &ExtMI = *getDefIgnoringCopies(EltReg, MRI);
+ if (ExtMI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) {
+ auto NumReg = ExtMI.getOperand(2).getReg();
+ auto NumValue = getIConstantVRegValWithLookThrough(NumReg, MRI);
+ if (!NumValue.has_value() || (NumValue->Value != (I - 1)))
+ return false;
+ auto &CastMI = *getDefIgnoringCopies(ExtMI.getOperand(1).getReg(), MRI);
+ if (CastMI.getOpcode() != TargetOpcode::G_BITCAST)
+ return false;
+ auto CastReg = CastMI.getOperand(1).getReg();
+ if (CastSrcReg && (CastSrcReg != CastReg))
+ return false;
+ if (DstTy.getSizeInBits() != MRI.getType(CastReg).getSizeInBits())
+ return false;
+ CastSrcReg = CastReg;
+ } else if (ExtMI.getOpcode() == TargetOpcode::G_TRUNC) {
+ auto &ShiftMI = *getDefIgnoringCopies(ExtMI.getOperand(1).getReg(), MRI);
+ if (ShiftMI.getOpcode() != TargetOpcode::G_LSHR)
+ return false;
+ auto ShiftValue = getIConstantVRegValWithLookThrough(
+ ShiftMI.getOperand(2).getReg(), MRI);
+ if (!ShiftValue.has_value() ||
+ (ShiftValue->Value != ((I - 1) * DstTy.getScalarSizeInBits())))
+ return false;
+ auto ShiftReg = ShiftMI.getOperand(1).getReg();
+ if (CastSrcReg && (CastSrcReg != ShiftReg))
+ return false;
+ CastSrcReg = ShiftReg;
+ } else
+ return false;
+ }
+ SaveReg = CastSrcReg;
+ return true;
+}
+void PISAPreLegalizerCombinerImpl::applyExtractBuildVectorToBitcast(
+ MachineInstr &MI, Register CastSrcReg) const {
+ auto DstReg = MI.getOperand(0).getReg();
+ B.buildBitcast(DstReg, CastSrcReg);
+ MI.eraseFromParent();
+}
+
+// Reduce predicates in s32.
+// %12:_(s1) = G_CONSTANT i1 true
+// %3:_(<3 x s1>) = G_FCMP floatpred(oeq), %0:regv3_32b(<3 x s32>), %1:_
+// %4:_(s1) = G_EXTRACT_VECTOR_ELT %3:_(<3 x s1>), %5:_(s32)
+// %6:_(s1) = G_EXTRACT_VECTOR_ELT %3:_(<3 x s1>), %7:_(s32)
+// %8:_(s1) = G_EXTRACT_VECTOR_ELT %3:_(<3 x s1>), %9:_(s32)
+// %10:_(s1) = G_AND %4:_, %6:_
+// %11:_(s1) = G_AND %10:_, %8:_
+// %13:_(s1) = G_ICMP intpred(eq), %11:_(s1), %12:_
+// => %3:_(<3 x s1>) = G_FCMP floatpred(oeq), %0:regv3_32b(<3 x s32>), %1:_
+// %18:_(<3 x s32>) = G_SEXT %3:_(<3 x s1>)
+// %20:_(s32) = G_CONSTANT i32 0
+// %19:_(s32) = G_EXTRACT_VECTOR_ELT %18:_(<3 x s32>), %20:_(s32)
+// %23:_(s32) = G_CONSTANT i32 1
+// %21:_(s32) = G_EXTRACT_VECTOR_ELT %18:_(<3 x s32>), %23:_(s32)
+// %22:_(s32) = G_AND %19:_, %21:_
+// %26:_(s32) = G_CONSTANT i32 2
+// %24:_(s32) = G_EXTRACT_VECTOR_ELT %18:_(<3 x s32>), %26:_(s32)
+// %25:_(s32) = G_AND %22:_, %24:_
+// %27:_(s32) = G_CONSTANT i32 -1
+// %13:_(s1) = G_ICMP intpred(eq), %25:_(s32), %27:_
+// Supports patterns with sext too:
+// %13:_(s8) = G_CONSTANT i8 -1
+// %3:_(<3 x s1>) = G_FCMP floatpred(oeq), %0:regv3_32b(<3 x s32>), %1:_
+// %4:_(<3 x s8>) = G_SEXT %3:_(<3 x s1>)
+// %5:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<3 x s8>), %6:_(s32)
+// %7:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<3 x s8>), %8:_(s32)
+// %9:_(s8) = G_EXTRACT_VECTOR_ELT %4:_(<3 x s8>), %10:_(s32)
+// %11:_(s8) = G_AND %5:_, %7:_
+// %12:_(s8) = G_AND %11:_, %9:_
+// %14:_(s1) = G_ICMP intpred(eq), %12:_(s8), %13:_
+bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+
+ auto DstReg = MI.getOperand(0).getReg();
+ if (!MRI.hasOneUse(DstReg))
+ return false;
+
+ auto DstTy = MRI.getType(DstReg);
+ if (!DstTy.isScalar() || (DstTy.getSizeInBits() == 32))
+ return false;
+
+ // If type is greater than s1, it must be interpreted as true/false.
+ if ((DstTy.getSizeInBits() > 1) &&
+ !(mi_match(*MRI.use_instr_begin(DstReg), MRI,
+ m_GICmp(m_Pred(), m_Reg(),
+ m_any_of(m_SpecificICst(0), m_SpecificICst(-1))))))
+ return false;
+
+ MachineInstr *VecCmpMI = nullptr;
+ APInt Mask;
+ auto ReductionOpcode = MI.getOpcode();
+
+ std::function<bool(MachineInstr *)> Match = [&](MachineInstr *MI) {
+ if (!MI)
+ return false;
+ // Next step in reduction?
+ Register LHS, RHS, Dst = MI->getOperand(0).getReg();
+ if (mi_match(Dst, MRI,
+ m_OneUse(m_BinOp(ReductionOpcode, m_Reg(LHS), m_Reg(RHS))))) {
+ return Match(getDefIgnoringCopies(LHS, MRI)) &&
+ Match(getDefIgnoringCopies(RHS, MRI));
+ }
+ // Extract from vector?
+ int64_t Idx;
+ if (mi_match(Dst, MRI,
+ m_OneUse(m_BinOp(TargetOpcode::G_EXTRACT_VECTOR_ELT,
+ m_Reg(LHS), m_ICst(Idx))))) {
+ if (!Match(getDefIgnoringCopies(LHS, MRI)))
+ return false;
+ Mask.setBit(Idx);
+ return true;
+ }
+ // Sext?
+ if (MI->getOpcode() == TargetOpcode::G_SEXT) {
+ return Match(getDefIgnoringCopies(MI->getOperand(1).getReg(), MRI));
+ }
+ // Cmp?
+ if (mi_match(Dst, MRI,
+ m_any_of(m_GICmp(m_Pred(), m_Reg(), m_Reg()),
+ m_GFCmp(m_Pred(), m_Reg(), m_Reg())))) {
+ if (VecCmpMI)
+ return VecCmpMI == MI;
+ auto VecType = MRI.getType(MI->getOperand(0).getReg());
+ if (!VecType.isFixedVector())
+ return false;
+ // Found cmp producing vector of predicates.
+ VecCmpMI = MI;
+ Mask = APInt::getZero(VecType.getNumElements());
+ return true;
+ }
+ return false;
+ };
+
+ if (!Match(&MI) || !Mask.isAllOnes())
+ return false;
+
+ MatchInfo = [DstReg, Mask = std::move(Mask), VecCmpMI, ReductionOpcode,
+ this](MachineIRBuilder &B) {
+ const LLT S32 = LLT::integer(32);
+
+ // Build reduction in s32.
+ auto ExtendedVector = MRI.createGenericVirtualRegister(
+ LLT::fixed_vector(Mask.getBitWidth(), S32));
+ B.buildSExt(ExtendedVector, VecCmpMI->getOperand(0).getReg());
+
+ Register Reduction = MRI.createGenericVirtualRegister(S32);
+ B.buildExtractVectorElement(Reduction, ExtendedVector,
+ B.buildConstant(S32, 0));
+
+ for (unsigned I = 1; I < Mask.getBitWidth(); ++I) {
+ auto SecondSrc = MRI.createGenericVirtualRegister(S32);
+ auto Dst = MRI.createGenericVirtualRegister(S32);
+ B.buildExtractVectorElement(SecondSrc, ExtendedVector,
+ B.buildConstant(S32, I));
+ B.buildInstr(ReductionOpcode, {Dst}, {Reduction, SecondSrc});
+ Reduction = Dst;
+ }
+
+ // Replace next use if possible.
+ auto UseMI = MRI.use_instr_begin(DstReg);
+ switch (UseMI->getOpcode()) {
+ case TargetOpcode::G_ICMP: {
+ auto Pred = (CmpInst::Predicate)UseMI->getOperand(1).getPredicate();
+ if (mi_match(UseMI->getOperand(3).getReg(), MRI, m_SpecificICst(-1)))
+ Pred = CmpInst::getInversePredicate(Pred);
+ B.buildICmp(Pred, UseMI->getOperand(0).getReg(), Reduction,
+ B.buildConstant(S32, 0));
+ UseMI->eraseFromParent();
+ } break;
+ default: {
+ B.buildICmp(CmpInst::ICMP_NE, DstReg, Reduction, B.buildConstant(S32, 0));
+ } break;
+ }
+ };
+ return true;
+}
+
+// %9:_(s1) = G_CONSTANT i1 true
+// %8:_(s1) = G_ICMP intpred(eq), %15:_(s32), %22:_
+// %10:_(s1) = G_ICMP intpred(eq), %8:_(s1), %9:_
+// %11:_(s32) = G_SEXT %10:_(s1)
+// => %8:_(s1) = G_ICMP intpred(eq), %15:_(s32), %22:_
+// %11:_(s32) = G_SEXT %8:_(s1)
+// or
+// %12:_(s1) = G_CONSTANT i1 true
+// %11:_(s1) = G_AND %10:_, %35:_
+// %13:_(s1) = G_ICMP intpred(eq), %11:_(s1), %12:_
+// %14:_(s32) = G_SEXT %13:_(s1)
+// => %11:_(s1) = G_AND %10:_, %35:_
+// %14:_(s32) = G_SEXT %11:_(s1)
+bool PISAPreLegalizerCombinerImpl::matchCmpInt1(
+ MachineInstr &ICmpMI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+
+ MachineInstr *PrevMI = nullptr;
+ CmpInst::Predicate Pred;
+ int64_t IsTrue;
+
+ if (!mi_match(
+ ICmpMI, MRI,
+ m_GICmp(m_Pred(Pred),
+ m_all_of(m_SpecificType(LLT::integer(1)), m_MInstr(PrevMI)),
+ m_ICst(IsTrue))))
+ return false;
+
+ if (Pred != CmpInst::ICMP_EQ)
+ return false;
+
+ if (!PrevMI)
+ return false;
+
+ auto DstReg = ICmpMI.getOperand(0).getReg();
+ MatchInfo = [DstReg, PrevMI, IsTrue, this](MachineIRBuilder &B) {
+ auto PrevDstReg = PrevMI->getOperand(0).getReg();
+
+ if (IsTrue) {
+ if (MRI.hasOneUse(PrevDstReg))
+ PrevMI->getOperand(0).setReg(DstReg);
+ else
+ B.buildCopy(DstReg, PrevDstReg);
+ } else {
+ B.buildNot(DstReg, PrevDstReg);
+ }
+ };
+ return true;
+}
+
+// Pass boilerplate
+// ================
+
+class PISAPreLegalizerCombiner : public MachineFunctionPass {
+ PISAPreLegalizerCombinerImplRuleConfig RuleConfig;
+
+public:
+ static char ID;
+
+ PISAPreLegalizerCombiner();
+
+ StringRef getPassName() const override { return "PISAPreLegalizerCombiner"; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+} // end anonymous namespace
+
+void PISAPreLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<TargetPassConfig>();
+ AU.setPreservesCFG();
+ getSelectionDAGFallbackAnalysisUsage(AU);
+ AU.addRequired<GISelValueTrackingAnalysisLegacy>();
+ AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
+ AU.addRequired<MachineDominatorTreeWrapperPass>();
+ AU.addPreserved<MachineDominatorTreeWrapperPass>();
+
+ AU.addRequired<GISelCSEAnalysisWrapperPass>();
+ AU.addPreserved<GISelCSEAnalysisWrapperPass>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+PISAPreLegalizerCombiner::PISAPreLegalizerCombiner() : MachineFunctionPass(ID) {
+ initializePISAPreLegalizerCombinerPass(*PassRegistry::getPassRegistry());
+ if (!RuleConfig.parseCommandLineOption())
+ report_fatal_error("Invalid rule identifier");
+}
+
+bool PISAPreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
+ if (MF.getProperties().hasProperty(
+ MachineFunctionProperties::Property::FailedISel))
+ return false;
+
+ auto &TPC = getAnalysis<TargetPassConfig>();
+ const Function &F = MF.getFunction();
+ bool EnableOpt =
+ MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
+ GISelValueTracking *KB =
+ &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
+ MachineDominatorTree *MDT =
+ &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ CombinerInfo CInfo(
+ /*AllowIllegalOps=*/true, /*ShouldLegalizeIllegal=*/false,
+ /*LegalizerInfo=*/nullptr, EnableOpt, F.hasOptSize(), F.hasMinSize());
+ // Enable CSE.
+ GISelCSEAnalysisWrapper &Wrapper =
+ getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
+ auto *CSEInfo = &Wrapper.get(TPC.getCSEConfig());
+
+ const PISASubtarget &STI = MF.getSubtarget<PISASubtarget>();
+ PISAPreLegalizerCombinerImpl Impl(MF, CInfo, *KB, CSEInfo, RuleConfig, STI,
+ MDT, STI.getLegalizerInfo());
+ return Impl.combineMachineInstrs();
+}
+
+// select(trunc(wide -> i1), N 1, N 0) => trunc(and(wide, 1), N)
+// Avoids introducing an illegal i1 compare when lowering bool-to-int.
+bool PISAPreLegalizerCombinerImpl::matchSelectTruncOneZero(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const {
+ assert(MI.getOpcode() == TargetOpcode::G_SELECT);
+ GSelect &Sel = cast<GSelect>(MI);
+
+ // Condition must be i1.
+ Register Cond = Sel.getCondReg();
+ if (MRI.getType(Cond) != LLT::scalar(1))
+ return false;
+
+ // Condition must come from a G_TRUNC.
+ MachineInstr *TruncMI = getDefIgnoringCopies(Cond, MRI);
+ if (!TruncMI || TruncMI->getOpcode() != TargetOpcode::G_TRUNC)
+ return false;
+
+ Register Wide = TruncMI->getOperand(1).getReg();
+ LLT WideTy = MRI.getType(Wide);
+ LLT DstTy = MRI.getType(Sel.getReg(0));
+
+ // Only handle scalar integers where dst fits in the wide source.
+ if (!DstTy.isScalar() || !WideTy.isScalar())
+ return false;
+ if (DstTy.getScalarSizeInBits() >= WideTy.getScalarSizeInBits())
+ return false;
+
+ // True value must be 1, false value must be 0.
+ auto TrueOpt = getIConstantVRegValWithLookThrough(Sel.getTrueReg(), MRI);
+ auto FalseOpt = getIConstantVRegValWithLookThrough(Sel.getFalseReg(), MRI);
+ if (!TrueOpt || !FalseOpt)
+ return false;
+ if (!TrueOpt->Value.isOne() || !FalseOpt->Value.isZero())
+ return false;
+
+ Register DstReg = Sel.getReg(0);
+ MatchInfo = [=](MachineIRBuilder &B) {
+ auto One = B.buildConstant(WideTy, 1);
+ auto And = B.buildAnd(WideTy, Wide, One);
+ B.buildTrunc(DstReg, And);
+ };
+ return true;
+}
+
+char PISAPreLegalizerCombiner::ID = 0;
+INITIALIZE_PASS_BEGIN(PISAPreLegalizerCombiner, DEBUG_TYPE,
+ "Combine PISA machine instrs before legalization", false,
+ false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
+INITIALIZE_PASS_END(PISAPreLegalizerCombiner, DEBUG_TYPE,
+ "Combine PISA machine instrs before legalization", false,
+ false)
+
+namespace llvm {
+FunctionPass *createPISAPreLegalizerCombiner() {
+ return new PISAPreLegalizerCombiner();
+}
+} // end namespace llvm
diff --git a/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp b/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
new file mode 100644
index 0000000000000..fb8fa3f69ed17
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
@@ -0,0 +1,53 @@
+//===-- PISARegisterBankInfo.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISARegisterBankInfo.h"
+#include "PISARegisterInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterBank.h"
+
+#define GET_REGINFO_ENUM
+#include "PISAGenRegisterInfo.inc"
+
+#define GET_TARGET_REGBANK_IMPL
+#include "PISAGenRegisterBank.inc"
+
+using namespace llvm;
+
+const RegisterBankInfo::InstructionMapping &
+PISARegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
+ const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
+
+ if (Mapping.isValid())
+ return Mapping;
+
+ const MachineFunction &MF = *MI.getParent()->getParent();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+
+ SmallVector<const ValueMapping *, 8> OpdsMapping(MI.getNumOperands());
+
+ for (unsigned Idx = 0; Idx < MI.getNumOperands(); ++Idx) {
+ auto &MO = MI.getOperand(Idx);
+
+ if (MO.isReg() && MO.getReg().isValid()) {
+ unsigned Size = getSizeInBits(MO.getReg(), MRI, *TRI);
+ OpdsMapping[Idx] = &getValueMapping(0, Size, PISA::RegistersRegBank);
+ }
+ }
+
+ return getInstructionMapping(DefaultMappingID, 1,
+ getOperandsMapping(OpdsMapping),
+ MI.getNumOperands());
+}
+
+const RegisterBank &PISARegisterBankInfo::getRegBankFromRegClass(
+ const TargetRegisterClass & /* RC */, LLT /* Ty */) const {
+ return PISA::RegistersRegBank;
+}
diff --git a/llvm/lib/Target/PISA/PISARegisterBankInfo.h b/llvm/lib/Target/PISA/PISARegisterBankInfo.h
new file mode 100644
index 0000000000000..705fa93c8c500
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISARegisterBankInfo.h
@@ -0,0 +1,37 @@
+//===-- PISARegisterBankInfo.h --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAREGISTERBANKINFO_H
+#define LLVM_LIB_TARGET_PISA_PISAREGISTERBANKINFO_H
+
+#include "llvm/CodeGen/RegisterBankInfo.h"
+
+#define GET_REGBANK_DECLARATIONS
+#include "PISAGenRegisterBank.inc"
+
+namespace llvm {
+
+class TargetRegisterInfo;
+
+class PISAGenRegisterBankInfo : public RegisterBankInfo {
+protected:
+#define GET_TARGET_REGBANK_CLASS
+#include "PISAGenRegisterBank.inc"
+};
+
+// This class provides the information for the target register banks.
+class PISARegisterBankInfo final : public PISAGenRegisterBankInfo {
+public:
+ const RegisterBank &getRegBankFromRegClass(const TargetRegisterClass &RC,
+ LLT Ty) const override;
+
+ const InstructionMapping &
+ getInstrMapping(const MachineInstr &MI) const override;
+};
+} // namespace llvm
+#endif // LLVM_LIB_TARGET_PISA_PISAREGISTERBANKINFO_H
diff --git a/llvm/lib/Target/PISA/PISASubtarget.cpp b/llvm/lib/Target/PISA/PISASubtarget.cpp
index 84d48180cae2e..bfc7c7390c2cc 100644
--- a/llvm/lib/Target/PISA/PISASubtarget.cpp
+++ b/llvm/lib/Target/PISA/PISASubtarget.cpp
@@ -8,7 +8,11 @@
#include "PISASubtarget.h"
#include "PISA.h"
+#include "PISALegalizerInfo.h"
+#include "PISARegisterBankInfo.h"
#include "PISATargetMachine.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/TargetParser/Host.h"
using namespace llvm;
@@ -21,10 +25,28 @@ using namespace llvm;
PISASubtarget::PISASubtarget(const Triple &TT, const std::string &CPU,
const std::string &FS, const PISATargetMachine &TM)
: PISAGenSubtargetInfo(TT, CPU, /*TuneCPU=*/CPU, FS), InstrInfo(*this),
- FrameLowering(initSubtargetDependencies(CPU, FS)) {}
+ FrameLowering(initSubtargetDependencies(CPU, FS)), TLInfo(TM, *this) {
+
+ CallLoweringInfo = std::make_unique<PISACallLowering>(TLInfo);
+ InlineAsmLoweringInfo = std::make_unique<InlineAsmLowering>(&TLInfo);
+ Legalizer = std::make_unique<PISALegalizerInfo>(*this);
+ RegBankInfo = std::make_unique<PISARegisterBankInfo>();
+ LLT::setUseExtended(true); // enable bfloat support
+ // The instruction selector is created in a subsequent change.
+}
PISASubtarget &PISASubtarget::initSubtargetDependencies(StringRef CPU,
StringRef FS) {
ParseSubtargetFeatures(CPU, /*TuneCPU=*/CPU, FS);
+ if (CPU.empty())
+ CPU = PISA::stripCPUPrefix(PISA::getDefaultCPUName());
+ PISATarget = PISA::getPISATargetInfo(CPU);
return *this;
}
+
+// Determine compatibility of instruction's PISA target, specified via
+// "let Predicates = []", vs. platform's target, specified via -mcpu=
+bool PISASubtarget::supportsPISATarget(StringRef Name) const {
+ auto InstrPISATarget = PISA::getPISATargetInfo(Name);
+ return isCompatiblePISATargetInfo(PISATarget, InstrPISATarget);
+}
diff --git a/llvm/lib/Target/PISA/PISASubtarget.h b/llvm/lib/Target/PISA/PISASubtarget.h
index 9f64ffa7f7ef0..857be0afadf0a 100644
--- a/llvm/lib/Target/PISA/PISASubtarget.h
+++ b/llvm/lib/Target/PISA/PISASubtarget.h
@@ -9,11 +9,21 @@
#ifndef LLVM_LIB_TARGET_PISA_PISASUBTARGET_H
#define LLVM_LIB_TARGET_PISA_PISASUBTARGET_H
+#include "PISACallLowering.h"
#include "PISAFrameLowering.h"
+#include "PISAISelLowering.h"
#include "PISAInstrInfo.h"
+#include "llvm/CodeGen/GlobalISel/CallLowering.h"
+#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
+#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
+#include "llvm/CodeGen/RegisterBankInfo.h"
+#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/DataLayout.h"
+#include "llvm/Support/PISAAddrSpace.h"
#include "llvm/Target/TargetMachine.h"
+#include "llvm/TargetParser/PISATargetParser.h"
#define GET_SUBTARGETINFO_HEADER
#include "PISAGenSubtargetInfo.inc"
@@ -21,9 +31,10 @@
namespace llvm {
class StringRef;
class PISATargetMachine;
-
class PISASubtarget : public PISAGenSubtargetInfo {
private:
+ PISA::PISATargetInfo PISATarget;
+
// Bool members for features defined in PISAFeatures.td.
#define GET_SUBTARGETINFO_MACRO(ATTRIBUTE, DEFAULT, GETTER) \
bool ATTRIBUTE = DEFAULT;
@@ -31,6 +42,14 @@ class PISASubtarget : public PISAGenSubtargetInfo {
PISAInstrInfo InstrInfo;
PISAFrameLowering FrameLowering;
+ PISATargetLowering TLInfo;
+
+ // GlobalISel related APIs.
+ std::unique_ptr<CallLowering> CallLoweringInfo;
+ std::unique_ptr<InlineAsmLowering> InlineAsmLoweringInfo;
+ std::unique_ptr<RegisterBankInfo> RegBankInfo;
+ std::unique_ptr<LegalizerInfo> Legalizer;
+ std::unique_ptr<InstructionSelector> InstSelector;
public:
// This constructor initializes the data members to match that
@@ -44,10 +63,35 @@ class PISASubtarget : public PISAGenSubtargetInfo {
// PISAFeatures.td.
void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
+ StringRef getPISATargetName() const { return PISATarget.Name; };
+ bool supportsPISATarget(StringRef Name) const;
+
+ bool shouldPrefetchAddressSpace(unsigned AS) const override {
+ return AS == static_cast<unsigned>(PISAAS::AddressSpace::GLOBAL);
+ }
+
+ const CallLowering *getCallLowering() const override {
+ return CallLoweringInfo.get();
+ }
+ const InlineAsmLowering *getInlineAsmLowering() const override {
+ return InlineAsmLoweringInfo.get();
+ }
+ const RegisterBankInfo *getRegBankInfo() const override {
+ return RegBankInfo.get();
+ }
+ const LegalizerInfo *getLegalizerInfo() const override {
+ return Legalizer.get();
+ }
+ InstructionSelector *getInstructionSelector() const override {
+ return InstSelector.get();
+ }
const PISAInstrInfo *getInstrInfo() const override { return &InstrInfo; }
const PISAFrameLowering *getFrameLowering() const override {
return &FrameLowering;
}
+ const PISATargetLowering *getTargetLowering() const override {
+ return &TLInfo;
+ }
const PISARegisterInfo *getRegisterInfo() const override {
return &InstrInfo.getRegisterInfo();
}
diff --git a/llvm/lib/Target/PISA/PISATargetMachine.cpp b/llvm/lib/Target/PISA/PISATargetMachine.cpp
index 79bb260240b3a..f4b3e163a3b8d 100644
--- a/llvm/lib/Target/PISA/PISATargetMachine.cpp
+++ b/llvm/lib/Target/PISA/PISATargetMachine.cpp
@@ -8,12 +8,24 @@
#include "PISATargetMachine.h"
#include "PISA.h"
+#include "PISAMachineFunctionInfo.h"
+#include "PISATargetObjectFile.h"
#include "TargetInfo/PISATargetInfo.h"
#include "llvm/ADT/SmallString.h"
-#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
+#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
+#include "llvm/CodeGen/GlobalISel/Legalizer.h"
+#include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
+#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/InitializePasses.h"
#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/PISAAddrSpace.h"
#include "llvm/Target/TargetOptions.h"
using namespace llvm;
@@ -22,6 +34,18 @@ using namespace llvm;
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePISATarget() {
// Register the target.
RegisterTargetMachine<PISATargetMachine> Y(getThePISATarget());
+
+ PassRegistry &PR = *PassRegistry::getPassRegistry();
+ initializeGlobalISel(PR);
+ initializePISAPreLegalizerCombinerPass(PR);
+ initializePISAPostLegalizerCombinerPass(PR);
+}
+
+MachineFunctionInfo *PISATargetMachine::createMachineFunctionInfo(
+ BumpPtrAllocator &Allocator, const Function &F,
+ const TargetSubtargetInfo *STI) const {
+ return PISAMachineFunctionInfo::create<PISAMachineFunctionInfo>(
+ Allocator, F, static_cast<const PISASubtarget *>(STI));
}
static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) {
@@ -30,6 +54,9 @@ static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) {
return *RM;
}
+// Pin PISATargetObjectFile's vtables to this file.
+PISATargetObjectFile::~PISATargetObjectFile() {}
+
PISATargetMachine::PISATargetMachine(const Target &T, const Triple &TT,
StringRef CPU, StringRef FS,
const TargetOptions &Options,
@@ -39,7 +66,7 @@ PISATargetMachine::PISATargetMachine(const Target &T, const Triple &TT,
: CodeGenTargetMachineImpl(T, TT.computeDataLayout(), TT, CPU, FS, Options,
getEffectiveRelocModel(RM),
getEffectiveCodeModel(CM, CodeModel::Small), OL),
- TLOF(std::make_unique<TargetLoweringObjectFileELF>()),
+ TLOF(std::make_unique<PISATargetObjectFile>()),
Subtarget(TT, CPU.str(), FS.str(), *this) {
initAsmInfo();
setGlobalISel(true);
@@ -62,21 +89,120 @@ PISATargetMachine::getSubtargetImpl(const Function &F) const {
Key.append(FS);
auto &I = SubtargetMap[Key];
- if (!I)
+ if (!I) {
I = std::make_unique<PISASubtarget>(TargetTriple, CPU.str(), FS.str(),
*this);
+ }
return I.get();
}
+unsigned PISATargetMachine::getAssumedAddrSpace(const Value *V) const {
+ const auto *Ld = dyn_cast<LoadInst>(V);
+ if (!Ld || Ld->getPointerOperand()->getType()->getPointerAddressSpace() !=
+ unsigned(PISAAS::AddressSpace::CONSTANT))
+ return ~0U;
+ return unsigned(PISAAS::AddressSpace::GLOBAL);
+}
+
+std::pair<const Value *, unsigned>
+PISATargetMachine::getPredicatedAddrSpace(const Value *V) const {
+ auto *II = dyn_cast<IntrinsicInst>(V);
+ if (!II)
+ return std::make_pair(nullptr, -1);
+
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::pisa_isaddr_private:
+ return std::make_pair(II->getArgOperand(0),
+ unsigned(PISAAS::AddressSpace::PRIVATE));
+ case Intrinsic::pisa_isaddr_global:
+ return std::make_pair(II->getArgOperand(0),
+ unsigned(PISAAS::AddressSpace::GLOBAL));
+ case Intrinsic::pisa_isaddr_shared:
+ return std::make_pair(II->getArgOperand(0),
+ unsigned(PISAAS::AddressSpace::SHARED));
+ default:
+ break;
+ }
+ return std::make_pair(nullptr, -1);
+}
+
+TargetTransformInfo
+PISATargetMachine::getTargetTransformInfo(const Function &F) const {
+ return TargetTransformInfo(F.getDataLayout());
+}
+
namespace {
+// PISA Code Generator Pass Configuration Options.
+//
+// PISA is a virtual-register-only target: it maintains virtual registers
+// throughout the pipeline and does not run register allocation. This
+// configuration wires up the GlobalISel selection stages and disables the
+// standard machine passes that assume physical registers exist.
class PISAPassConfig : public TargetPassConfig {
public:
PISAPassConfig(PISATargetMachine &TM, PassManagerBase &PM)
- : TargetPassConfig(TM, PM) {}
+ : TargetPassConfig(TM, PM) {
+ disablePass(&GCLoweringID);
+ disablePass(&ShadowStackGCLoweringID);
+ }
PISATargetMachine &getPISATargetMachine() const {
return getTM<PISATargetMachine>();
}
+
+ void addIRPasses() override {
+ TargetPassConfig::addIRPasses();
+
+ // Disable passes that assume physical registers exist.
+ disablePass(&PrologEpilogCodeInserterID);
+ disablePass(&MachineLateInstrsCleanupID);
+ disablePass(&MachineCopyPropagationID);
+ disablePass(&TailDuplicateLegacyID);
+ disablePass(&StackMapLivenessID);
+ disablePass(&LiveDebugValuesID);
+ disablePass(&PostRAMachineSinkingID);
+ disablePass(&PostRASchedulerID);
+ disablePass(&FuncletLayoutID);
+ disablePass(&PatchableFunctionID);
+ disablePass(&ShrinkWrapID);
+ disablePass(&RemoveLoadsIntoFakeUsesID);
+ disablePass(&GCMachineCodeAnalysisID);
+ }
+
+ bool addIRTranslator() override {
+ addPass(new IRTranslatorLegacy(getOptLevel()));
+ return false;
+ }
+
+ void addPreLegalizeMachineIR() override {
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(createPISAPreLegalizerCombiner());
+ }
+
+ bool addLegalizeMachineIR() override {
+ addPass(new LegalizerLegacy());
+ return false;
+ }
+
+ void addPreRegBankSelect() override {
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ addPass(&MachineCSELegacyID);
+ addPass(createPISAPostLegalizerCombiner());
+ }
+ }
+
+ bool addRegBankSelect() override {
+ addPass(new RegBankSelect());
+ return false;
+ }
+
+ // Instruction selection (addGlobalInstructionSelect) is added in a
+ // subsequent change together with the PISA instruction selector.
+
+ // PISA does not allocate physical registers.
+ FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
+ bool addRegAssignAndRewriteFast() override { return false; }
+ bool addRegAssignAndRewriteOptimized() override { return false; }
};
} // namespace
diff --git a/llvm/lib/Target/PISA/PISATargetMachine.h b/llvm/lib/Target/PISA/PISATargetMachine.h
index a378e4f20401a..5e98087d2272f 100644
--- a/llvm/lib/Target/PISA/PISATargetMachine.h
+++ b/llvm/lib/Target/PISA/PISATargetMachine.h
@@ -12,6 +12,7 @@
#include "PISASubtarget.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
+#include "llvm/Support/PISAAddrSpace.h"
#include <memory>
#include <optional>
@@ -33,13 +34,40 @@ class PISATargetMachine : public CodeGenTargetMachineImpl {
// target-specific attributes of each function.
const PISASubtarget *getSubtargetImpl() const = delete;
- TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
+ TargetTransformInfo getTargetTransformInfo(const Function &F) const override;
+ TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
bool usesPhysRegsForValues() const override { return false; }
TargetLoweringObjectFile *getObjFileLowering() const override {
return TLOF.get();
}
+
+ MachineFunctionInfo *
+ createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F,
+ const TargetSubtargetInfo *STI) const override;
+
+ static int64_t getNullPointerValue(unsigned AS) {
+ return (AS == (unsigned)PISAAS::AddressSpace::PRIVATE ||
+ AS == (unsigned)PISAAS::AddressSpace::SHARED)
+ ? -1
+ : 0;
+ }
+
+ bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DstAS) const override {
+ auto Sas = (SrcAS == (unsigned)PISAAS::AddressSpace::GLOBAL) ||
+ (SrcAS == (unsigned)PISAAS::AddressSpace::CONSTANT) ||
+ (SrcAS == (unsigned)PISAAS::AddressSpace::GENERIC);
+ auto Das = (DstAS == (unsigned)PISAAS::AddressSpace::GLOBAL) ||
+ (DstAS == (unsigned)PISAAS::AddressSpace::CONSTANT) ||
+ (DstAS == (unsigned)PISAAS::AddressSpace::GENERIC);
+ return Sas && Das;
+ }
+
+ unsigned getAssumedAddrSpace(const Value *V) const override;
+
+ std::pair<const Value *, unsigned>
+ getPredicatedAddrSpace(const Value *V) const override;
};
} // namespace llvm
diff --git a/llvm/lib/Target/PISA/PISATargetObjectFile.h b/llvm/lib/Target/PISA/PISATargetObjectFile.h
new file mode 100644
index 0000000000000..360103ed3f6ec
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISATargetObjectFile.h
@@ -0,0 +1,59 @@
+//===-- PISATargetObjectFile.h - PISA Object Info -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISATARGETOBJECTFILE_H
+#define LLVM_LIB_TARGET_PISA_PISATARGETOBJECTFILE_H
+
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSection.h"
+#include "llvm/MC/SectionKind.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+
+class PISATargetObjectFile : public TargetLoweringObjectFile {
+public:
+ ~PISATargetObjectFile() override;
+
+ void Initialize(MCContext &Ctx, const TargetMachine &TM) override {
+ TargetLoweringObjectFile::Initialize(Ctx, TM);
+ }
+ // All words in a PISA module (excepting the first 5 ones) are a linear
+ // sequence of instructions in a specific order. We put all the instructions
+ // in the single text section.
+ MCSection *getSectionForConstant(const DataLayout &DL, SectionKind Kind,
+ const Constant *C, Align &Alignment,
+ const Function *F) const override {
+ return TextSection;
+ }
+ MCSection *getExplicitSectionGlobal(const GlobalObject *GO, SectionKind Kind,
+ const TargetMachine &TM) const override {
+ return TextSection;
+ }
+ MCSection *SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind,
+ const TargetMachine &TM) const override {
+ return TextSection;
+ }
+ // PISA doesn't want '\01' suppression mangling to strip the leading '\01'.
+ // A named variable should just be passed along as is.
+ MCSymbol *getTargetSymbol(const GlobalValue *GV,
+ const TargetMachine &TM) const override {
+ SmallString<128> Name;
+ if (GV->hasName())
+ Name = GV->getName();
+ else
+ TM.getNameWithPrefix(Name, GV, getMangler());
+ return getContext().getOrCreateSymbol(Name);
+ }
+};
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISATARGETOBJECTFILE_H
diff --git a/llvm/lib/TargetParser/CMakeLists.txt b/llvm/lib/TargetParser/CMakeLists.txt
index cb45571583d23..29c058677dab6 100644
--- a/llvm/lib/TargetParser/CMakeLists.txt
+++ b/llvm/lib/TargetParser/CMakeLists.txt
@@ -23,6 +23,7 @@ add_llvm_component_library(LLVMTargetParser
Host.cpp
LoongArchTargetParser.cpp
NVPTXTargetParser.cpp
+ PISATargetParser.cpp
PPCTargetParser.cpp
RISCVISAInfo.cpp
RISCVTargetParser.cpp
diff --git a/llvm/lib/TargetParser/PISATargetParser.cpp b/llvm/lib/TargetParser/PISATargetParser.cpp
new file mode 100644
index 0000000000000..8b84d7eecd3ef
--- /dev/null
+++ b/llvm/lib/TargetParser/PISATargetParser.cpp
@@ -0,0 +1,24 @@
+//===-- PISATargetParser.cpp - PISA target parsing defines ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/TargetParser/PISATargetParser.h"
+#include "llvm/TargetParser/TargetParser.h"
+
+#define GET_SUBTARGETFEATURES_ENUM
+#define GET_SUBTARGETFEATURES_KV
+#include "llvm/TargetParser/PISAGenTargetFeatures.inc"
+
+void llvm::PISA::fillFeatureMap(StringRef CPU, StringMap<bool> &Features) {
+ PISATargetInfo Info = getPISATargetInfo(stripCPUPrefix(CPU));
+ if (Info.Name.empty())
+ return;
+ if (std::optional<StringMap<bool>> Default = getCPUDefaultTargetFeatures(
+ Info.Name, BasicPISASubTypeKV, BasicPISAFeatureKV))
+ for (const auto &KV : *Default)
+ Features[KV.first()] = KV.second;
+}
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir b/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
new file mode 100644
index 0000000000000..3c2e0376c6fda
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
@@ -0,0 +1,45 @@
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# Regression test to make sure truncating double into i8 doesn't try to produce an 8 bit float.
+--- |
+ target triple = "pisa"
+
+ define pisa_kernel void @test(i8 %id) {
+ %b = bitcast <1 x double> zeroinitializer to <8 x i8>
+ %elem = extractelement <8 x i8> %b, i8 %id
+ store i8 %elem, ptr addrspace(1) null, align 1
+ ret void
+ }
+...
+---
+name: test
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; CHECK-LABEL: {{^}}name: test
+ ; CHECK: [[CONST:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[CONST]](i64)
+ ; CHECK: G_EXTRACT_VECTOR_ELT [[BITCAST]](<2 x i32>)
+ %0:reg8b(i8) = loadParam_i8 0, 0
+ %2:_(i64) = G_FCONSTANT double 0.000000e+00
+ %6:_(p1) = G_CONSTANT i64 0
+ %20:_(i8) = G_TRUNC %2:_(i64)
+ %21:_(i8) = G_CONSTANT i8 0
+ %28:_(<4 x i8>) = G_BUILD_VECTOR %20:_(i8), %21:_(i8), %21:_(i8), %21:_(i8)
+ %29:_(<4 x i8>) = G_BUILD_VECTOR %21:_(i8), %21:_(i8), %21:_(i8), %21:_(i8)
+ %5:_(i32) = G_ZEXT %0:reg8b(i8)
+ %18:_(i32) = G_BITCAST %28:_(<4 x i8>)
+ %19:_(i32) = G_BITCAST %29:_(<4 x i8>)
+ %7:_(<2 x i32>) = G_BUILD_VECTOR %18:_(i32), %19:_(i32)
+ %8:_(i32) = G_CONSTANT i32 2
+ %9:_(i32) = G_LSHR %5:_, %8:_(i32)
+ %10:_(i32) = G_EXTRACT_VECTOR_ELT %7:_(<2 x i32>), %9:_(i32)
+ %11:_(i32) = G_CONSTANT i32 3
+ %12:_(i32) = G_AND %5:_, %11:_
+ %14:_(i32) = G_SHL %12:_, %11:_(i32)
+ %15:_(i32) = G_LSHR %10:_, %14:_(i32)
+ %4:_(i8) = G_TRUNC %15:_(i32)
+ G_STORE %4:_(i8), %6:_(p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir
new file mode 100644
index 0000000000000..b0dec696aeede
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir
@@ -0,0 +1,48 @@
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ define i32 @combine_abs_max_i32(i32 %data, i32 %mask, i32 %passthru) {
+ %a = call i32 @llvm.abs.i32(i32 %data, i1 false)
+ %val = call i32 @llvm.pisa.ired.i32(i8 4, i32 %a, i32 %mask, i32 %passthru)
+ ret i32 %val
+ }
+
+ define i16 @combine_abs_max_i16(i16 %data, i32 %mask, i16 %passthru) {
+ %a = call i16 @llvm.abs.i16(i16 %data, i1 false)
+ %val = call i16 @llvm.pisa.ired.i16(i8 4, i16 %a, i32 %mask, i16 %passthru)
+ ret i16 %val
+ }
+...
+---
+name: combine_abs_max_i32
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1 (%ir-block.0):
+ ; CHECK-LABEL: name: combine_abs_max_i32
+ ; CHECK: G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired), 8, %0(i32), %1(i32), %2(i32)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:reg32b(i32) = functionParameter_i32 2
+ %3:registers(i32) = G_ABS %0
+ %4:registers(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired), 4, %3(i32), %1(i32), %2(i32)
+ %5:reg32b(i32) = COPY %4(i32)
+ retValue_i32_r %5(i32)
+...
+---
+name: combine_abs_max_i16
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1 (%ir-block.0):
+ ; CHECK-LABEL: name: combine_abs_max_i16
+ ; CHECK: G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired), 8, %0(i16), %1(i32), %2(i16)
+ %0:reg16b(i16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:reg16b(i16) = functionParameter_i16 2
+ %3:registers(i16) = G_ABS %0
+ %4:registers(i16) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.pisa.ired), 4, %3(i16), %1(i32), %2(i16)
+ %5:reg16b(i16) = COPY %4(i16)
+ retValue_i16_r %5(i16)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir
new file mode 100644
index 0000000000000..534a6a345f4fd
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir
@@ -0,0 +1,89 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck -check-prefix=COMBINER %s
+--- |
+ target triple = "pisa"
+
+ define i32 @test_cmp_and_all_ones(i32 %dst_width, i32 %dst_height, i32 %conv, i32 %conv2) {
+ b0:
+ %cmp = icmp sle i32 %dst_width, %conv
+ %cmp4 = icmp sle i32 %dst_height, %conv2
+ %or.1 = or i1 %cmp, %cmp4
+ br i1 %or.1, label %b1, label %b2
+
+ b1: ; preds = %b0
+ ret i32 0
+
+ b2: ; preds = %b0
+ ret i32 1
+ }
+...
+---
+name: test_cmp_and_all_ones
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ ; COMBINER-LABEL: name: test_cmp_and_all_ones
+ ; COMBINER: bb.0.b0:
+ ; COMBINER-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; COMBINER-NEXT: {{ $}}
+ ; COMBINER-NEXT: %0:reg32b(i32) = functionParameter_i32 0
+ ; COMBINER-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; COMBINER-NEXT: %2:reg32b(i32) = functionParameter_i32 2
+ ; COMBINER-NEXT: %3:reg32b(i32) = functionParameter_i32 3
+ ; COMBINER-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; COMBINER-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; COMBINER-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(sgt), %0(i32), %2
+ ; COMBINER-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(sgt), %1(i32), %3
+ ; COMBINER-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; COMBINER-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; COMBINER-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C3]], [[C2]]
+ ; COMBINER-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; COMBINER-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; COMBINER-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[ICMP1]](i1), [[C5]], [[C4]]
+ ; COMBINER-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SELECT]], [[SELECT1]]
+ ; COMBINER-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; COMBINER-NEXT: [[ICMP2:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND]](i32), [[C6]]
+ ; COMBINER-NEXT: G_BRCOND [[ICMP2]](i1), %bb.2
+ ; COMBINER-NEXT: G_BR %bb.1
+ ; COMBINER-NEXT: {{ $}}
+ ; COMBINER-NEXT: bb.1.b1:
+ ; COMBINER-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = COPY [[C1]](i32)
+ ; COMBINER-NEXT: retValue_i32_r [[COPY]](i32)
+ ; COMBINER-NEXT: {{ $}}
+ ; COMBINER-NEXT: bb.2.b2:
+ ; COMBINER-NEXT: [[COPY1:%[0-9]+]]:reg32b(i32) = COPY [[C]](i32)
+ ; COMBINER-NEXT: retValue_i32_r [[COPY1]](i32)
+ bb.1.b0:
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+ %0:reg32b(i32) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:reg32b(i32) = functionParameter_i32 2
+ %3:reg32b(i32) = functionParameter_i32 3
+ %7:_(i32) = G_CONSTANT i32 1
+ %9:_(i32) = G_CONSTANT i32 0
+ %4:_(i1) = G_ICMP intpred(sgt), %0(i32), %2
+ %5:_(i1) = G_ICMP intpred(sgt), %1(i32), %3
+ %16:_(i32) = G_CONSTANT i32 0
+ %17:_(i32) = G_CONSTANT i32 -1
+ %13:_(i32) = G_SELECT %4(i1), %17, %16
+ %18:_(i32) = G_CONSTANT i32 0
+ %19:_(i32) = G_CONSTANT i32 -1
+ %14:_(i32) = G_SELECT %5(i1), %19, %18
+ %15:_(i32) = G_AND %13, %14
+ %20:_(i32) = G_CONSTANT i32 0
+ %21:_(i32) = G_CONSTANT i32 1
+ %22:_(i32) = G_AND %15, %21
+ %6:_(i1) = G_ICMP intpred(ne), %22(i32), %20
+ G_BRCOND %6(i1), %bb.3
+ G_BR %bb.2
+
+ bb.2.b1:
+ %10:reg32b(i32) = COPY %9(i32)
+ retValue_i32_r %10(i32)
+
+ bb.3.b2:
+ %8:reg32b(i32) = COPY %7(i32)
+ retValue_i32_r %8(i32)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir
new file mode 100644
index 0000000000000..08a8dd687113a
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir
@@ -0,0 +1,88 @@
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ define void @test_extract_build_vector(ptr %addr, ptr %addr2) {
+ %rv = load <4 x i32>, ptr %addr, align 32
+ store <4 x i32> %rv, ptr %addr2, align 32
+ ret void
+ }
+ ; Negative test: extract 2-element subvector from 8-element build_vector.
+ ; matchExtractSubvectorBuildVector requires NumDstElts == 4, so this must NOT fold.
+ define void @test_no_fold_2dst(ptr %addr) { ret void }
+ ; Negative test: source is G_LOAD, not G_BUILD_VECTOR; must NOT fold.
+ define void @test_no_fold_non_build_vector(ptr %addr, ptr %addr2) {
+ %rv = load <8 x i32>, ptr %addr, align 32
+ store <4 x i32> undef, ptr %addr2, align 16
+ ret void
+ }
+...
+---
+name: test_extract_build_vector
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: {{^}}name: test_extract_build_vector
+ ; CHECK: (<4 x i32>) = G_LOAD %0(p0)
+ ; CHECK-NEXT: G_STORE %{{[0-9]+}}(<4 x i32>)
+ %0:reg32b(p0) = functionParameter_i32 0
+ %1:reg32b(p0) = functionParameter_i32 1
+ %20:registers(<4 x i32>) = G_LOAD %0:reg32b(p0) :: (load (<4 x i32>) from %ir.addr, align 32, addrspace 0)
+ %37:registers(i32) = G_CONSTANT i32 0
+ %21:registers(i32) = G_EXTRACT_VECTOR_ELT %20:registers(<4 x i32>), %37:registers(i32)
+ %38:registers(i32) = G_CONSTANT i32 1
+ %22:registers(i32) = G_EXTRACT_VECTOR_ELT %20:registers(<4 x i32>), %38:registers(i32)
+ %39:registers(i32) = G_CONSTANT i32 2
+ %23:registers(i32) = G_EXTRACT_VECTOR_ELT %20:registers(<4 x i32>), %39:registers(i32)
+ %40:registers(i32) = G_CONSTANT i32 3
+ %24:registers(i32) = G_EXTRACT_VECTOR_ELT %20:registers(<4 x i32>), %40:registers(i32)
+ %3:registers(<4 x i32>) = G_BUILD_VECTOR %21:registers(i32), %22:registers(i32), %23:registers(i32), %24:registers(i32)
+ G_STORE %3:registers(<4 x i32>), %1:reg32b(p0) :: (store (<4 x i32>) into %ir.addr2, align 32, addrspace 0)
+ ret
+---
+# Negative test: NumDstElts(2) != 4, so matchExtractSubvectorBuildVector must
+# return false and the G_EXTRACT_SUBVECTOR must survive unchanged.
+name: test_no_fold_2dst
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: {{^}}name: test_no_fold_2dst
+ ; CHECK: G_BUILD_VECTOR
+ ; CHECK: G_EXTRACT_SUBVECTOR
+ %0:reg32b(p0) = functionParameter_i32 0
+ %10:registers(i32) = G_CONSTANT i32 1
+ %11:registers(i32) = G_CONSTANT i32 2
+ %12:registers(i32) = G_CONSTANT i32 3
+ %13:registers(i32) = G_CONSTANT i32 4
+ %14:registers(i32) = G_CONSTANT i32 5
+ %15:registers(i32) = G_CONSTANT i32 6
+ %16:registers(i32) = G_CONSTANT i32 7
+ %17:registers(i32) = G_CONSTANT i32 8
+ ; Build an 8-element vector.
+ %20:registers(<8 x i32>) = G_BUILD_VECTOR %10:registers(i32), %11:registers(i32), %12:registers(i32), %13:registers(i32), %14:registers(i32), %15:registers(i32), %16:registers(i32), %17:registers(i32)
+ ; Extract a 2-element subvector starting at offset 0. NumDstElts=2 != 4, no fold.
+ %30:registers(<2 x i32>) = G_EXTRACT_SUBVECTOR %20:registers(<8 x i32>), 0
+ G_STORE %30:registers(<2 x i32>), %0:reg32b(p0) :: (store (<2 x i32>) into %ir.addr, addrspace 0)
+ ret
+---
+# Negative test: source is a G_LOAD (not G_BUILD_VECTOR), so
+# matchExtractSubvectorBuildVector must return false (BV == null check).
+name: test_no_fold_non_build_vector
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: {{^}}name: test_no_fold_non_build_vector
+ ; CHECK-NOT: G_BUILD_VECTOR
+ ; CHECK: G_EXTRACT_SUBVECTOR
+ %0:reg32b(p0) = functionParameter_i32 0
+ %1:reg32b(p0) = functionParameter_i32 1
+ ; Load an 8-element vector from memory (not a G_BUILD_VECTOR).
+ %20:registers(<8 x i32>) = G_LOAD %0:reg32b(p0) :: (load (<8 x i32>) from %ir.addr, align 32, addrspace 0)
+ ; Extract 4 elements at offset 0. Source is not G_BUILD_VECTOR: no fold.
+ %30:registers(<4 x i32>) = G_EXTRACT_SUBVECTOR %20:registers(<8 x i32>), 0
+ G_STORE %30:registers(<4 x i32>), %1:reg32b(p0) :: (store (<4 x i32>) into %ir.addr2, addrspace 0)
+ ret
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir
new file mode 100644
index 0000000000000..b5434203547c2
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir
@@ -0,0 +1,18 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+---
+name: test_fadd_constant
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: test_fadd_constant
+ ; CHECK: %res:reg32b(f32) = G_FCONSTANT float 1.000000e+00
+ ; CHECK-NEXT: %ret:reg32b(i32) = G_BITCAST %res(f32)
+ ; CHECK-NEXT: retValue_i32_r %ret(i32)
+ %lhs:reg32b(f32) = G_FCONSTANT float 0.0
+ %rhs:reg32b(f32) = G_FCONSTANT float 1.0
+ %res:reg32b(f32) = G_FADD %lhs:reg32b, %rhs:reg32b
+ %ret:reg32b(i32) = G_BITCAST %res(f32)
+ retValue_i32_r %ret(i32)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
new file mode 100644
index 0000000000000..0658e26af5512
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
@@ -0,0 +1,74 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# This test exercises the `NoVectors` early-break inside
+# PISA::getDefIgnoringBitcasts(). The combine
+# unmerge_bitcast_buildvector_to_bitcast calls getDefIgnoringBitcasts with
+# NoVectors=true while walking each G_BUILD_VECTOR operand. When the walk
+# encounters a G_BITCAST whose source or destination is a vector type,
+# the helper must take the early `break` branch on the line:
+# if (NoVectors && (DstTy.isVector() || SrcTy.isVector())) break;
+# The walker then stops at that G_BITCAST and the matcher's G_UNMERGE_VALUES
+# check fails so the combine does not fire.
+#
+# The combine must NOT replace the G_BUILD_VECTOR. The body-level CHECK lines
+# below verify that the explicit G_BUILD_VECTOR remains.
+
+# The shifts_of_constants combine should fold the hinted shift into one constant.
+# The body-level CHECK lines below verify the folded G_CONSTANT result.
+
+--- |
+ define <2 x i16> @build_vector_with_vector_bitcast_in_chain(i16 %a, i16 %b) {
+ ret <2 x i16> zeroinitializer
+ }
+
+ define i16 @shift_of_constants_via_assert_sext_hints(i16 %unused) {
+ ret i16 0
+ }
+...
+---
+name: build_vector_with_vector_bitcast_in_chain
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: build_vector_with_vector_bitcast_in_chain
+ ; CHECK: %0:reg16b(i16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg16b(i16) = functionParameter_i16 1
+ ; CHECK-NEXT: %bv:_(<2 x i16>) = G_BUILD_VECTOR %0(i16), %1(i16)
+ ; CHECK-NEXT: %ret:regv2_16b(<2 x i16>) = COPY %bv(<2 x i16>)
+ ; CHECK-NEXT: retValue_v2i16_r %ret(<2 x i16>)
+ %0:reg16b(i16) = functionParameter_i16 0
+ %1:reg16b(i16) = functionParameter_i16 1
+ ; Two scalar-from-vector bitcasts whose source type is the vector
+ ; <2 x i8>. getDefIgnoringBitcasts(NoVectors=true) immediately hits
+ ; the `break` because SrcTy.isVector() is true.
+ %va:_(<2 x i8>) = G_BITCAST %0:reg16b(i16)
+ %vb:_(<2 x i8>) = G_BITCAST %1:reg16b(i16)
+ %a16:_(i16) = G_BITCAST %va:_(<2 x i8>)
+ %b16:_(i16) = G_BITCAST %vb:_(<2 x i8>)
+ %bv:_(<2 x i16>) = G_BUILD_VECTOR %a16:_(i16), %b16:_(i16)
+ %ret:regv2_16b(<2 x i16>) = COPY %bv:_(<2 x i16>)
+ retValue_v2i16_r %ret:regv2_16b(<2 x i16>)
+...
+
+---
+name: shift_of_constants_via_assert_sext_hints
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: shift_of_constants_via_assert_sext_hints
+ ; CHECK: %r:_(i16) = G_CONSTANT i16 4
+ ; CHECK-NEXT: %ret:reg16b(i16) = COPY %r(i16)
+ ; CHECK-NEXT: retValue_i16_r %ret(i16)
+ %c1:_(i16) = G_CONSTANT i16 1
+ %h1:_(i16) = G_ASSERT_SEXT %c1:_(i16), 8
+ %c2:_(i16) = G_CONSTANT i16 2
+ %h2:_(i16) = G_ASSERT_SEXT %c2:_(i16), 8
+ %r:_(i16) = G_SHL %h1:_(i16), %h2:_(i16)
+ %ret:reg16b(i16) = COPY %r:_(i16)
+ retValue_i16_r %ret:reg16b(i16)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir
new file mode 100644
index 0000000000000..959aaa318bf7f
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ ; Test that computeKnownBitsForTargetInstr correctly propagates range
+ ; attribute information from pisa.local.id intrinsics.
+ ; The range attribute [0, 65536) guarantees top 16 bits are zero,
+ ; which enables the elimination of G_TRUNC + G_ZEXT patterns.
+ ;
+ ; This test verifies that:
+ ; 1. computeKnownBitsForTargetInstr() in PISAISelLowering.cpp correctly
+ ; extracts and provides the range attribute information
+ ; 2. The known_bits_simplifications combiner group (which includes zext_trunc_fold)
+ ; uses this information to optimize away redundant TRUNC+ZEXT operations
+
+ declare i32 @llvm.pisa.local.id.x()
+
+ define i32 @test_local_id_trunc_zext() {
+ %id = call i32 @llvm.pisa.local.id.x()
+ %trunc = trunc i32 %id to i16
+ %zext = zext i16 %trunc to i32
+ ret i32 %zext
+ }
+
+ define i32 @test_local_id_y_trunc_zext() {
+ %id = call i32 @llvm.pisa.local.id.y()
+ %trunc = trunc i32 %id to i16
+ %zext = zext i16 %trunc to i32
+ ret i32 %zext
+ }
+
+ define i32 @test_local_id_z_trunc_zext() {
+ %id = call i32 @llvm.pisa.local.id.z()
+ %trunc = trunc i32 %id to i16
+ %zext = zext i16 %trunc to i32
+ ret i32 %zext
+ }
+
+...
+---
+name: test_local_id_trunc_zext
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; CHECK-LABEL: name: test_local_id_trunc_zext
+ ; CHECK: [[LOCAL_ID:%[0-9]+]]:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.id.x)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = COPY [[LOCAL_ID]](i32)
+ ; CHECK-NEXT: retValue_i32_r [[COPY]](i32)
+ %0:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.id.x)
+ %1:_(i16) = G_TRUNC %0(i32)
+ %2:_(i32) = G_ZEXT %1(i16)
+ %3:reg32b(i32) = COPY %2(i32)
+ retValue_i32_r %3(i32)
+
+...
+---
+name: test_local_id_y_trunc_zext
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; CHECK-LABEL: name: test_local_id_y_trunc_zext
+ ; CHECK: [[LOCAL_ID:%[0-9]+]]:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.id.y)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = COPY [[LOCAL_ID]](i32)
+ ; CHECK-NEXT: retValue_i32_r [[COPY]](i32)
+ %0:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.id.y)
+ %1:_(i16) = G_TRUNC %0(i32)
+ %2:_(i32) = G_ZEXT %1(i16)
+ %3:reg32b(i32) = COPY %2(i32)
+ retValue_i32_r %3(i32)
+
+...
+---
+name: test_local_id_z_trunc_zext
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; CHECK-LABEL: name: test_local_id_z_trunc_zext
+ ; CHECK: [[LOCAL_ID:%[0-9]+]]:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.id.z)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = COPY [[LOCAL_ID]](i32)
+ ; CHECK-NEXT: retValue_i32_r [[COPY]](i32)
+ %0:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.id.z)
+ %1:_(i16) = G_TRUNC %0(i32)
+ %2:_(i32) = G_ZEXT %1(i16)
+ %3:reg32b(i32) = COPY %2(i32)
+ retValue_i32_r %3(i32)
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir
new file mode 100644
index 0000000000000..572f6c3ab288d
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir
@@ -0,0 +1,52 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# Test that p2i_to_i2p_fixed combine rule correctly handles pointer address
+# spaces. A COPY between pointers of different address spaces is illegal.
+# The fold should only apply when source and destination pointer types match
+# exactly (same address space).
+
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
+
+ @gv = external addrspace(1) constant i8
+
+ define void @fold_same_addrspace() { ret void }
+ define void @no_fold_different_addrspace() { ret void }
+...
+
+# Same address space: p1 -> i64 -> p1. Should fold to COPY.
+---
+name: fold_same_addrspace
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: fold_same_addrspace
+ ; CHECK: [[GV:%[0-9]+]]:_(p1) = G_GLOBAL_VALUE @gv
+ ; CHECK-NEXT: G_STORE [[GV]](p1), [[GV]](p1) :: (store (i8), addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:_(p1) = G_GLOBAL_VALUE @gv
+ %1:_(i64) = G_PTRTOINT %0(p1)
+ %2:_(p1) = G_INTTOPTR %1(i64)
+ G_STORE %2(p1), %2(p1) :: (store (i8), addrspace 1)
+ ret
+...
+
+# Different non-generic address spaces: p1 -> i64 -> p2. Should NOT fold
+# to COPY because p1 != p2 even though both are 64-bit pointers.
+---
+name: no_fold_different_addrspace
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: no_fold_different_addrspace
+ ; CHECK: [[GV:%[0-9]+]]:_(p1) = G_GLOBAL_VALUE @gv
+ ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:_(i64) = G_PTRTOINT [[GV]](p1)
+ ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p2) = G_INTTOPTR [[PTRTOINT]](i64)
+ ; CHECK-NEXT: G_STORE [[INTTOPTR]](p2), [[GV]](p1) :: (store (i8), addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:_(p1) = G_GLOBAL_VALUE @gv
+ %1:_(i64) = G_PTRTOINT %0(p1)
+ %2:_(p2) = G_INTTOPTR %1(i64)
+ G_STORE %2(p2), %0(p1) :: (store (i8), addrspace 1)
+ ret
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
new file mode 100644
index 0000000000000..694e978edc5f5
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
@@ -0,0 +1,160 @@
+# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# Tests for PISAPreLegalizerCombinerImpl::matchSelectTruncOneZero.
+#
+# The combiner rewrites:
+# select(trunc(wide -> i1), N 1, N 0) => trunc(and(wide, 1), N)
+# to avoid introducing an illegal i1 compare when lowering bool-to-int.
+
+--- |
+ define i16 @test_select_trunc_one_zero_match(i32 %wide) { ret i16 0 }
+ define i16 @test_no_fold_cond_from_icmp(i32 %a) { ret i16 0 }
+ define i16 @test_no_fold_wrong_true_val(i32 %wide) { ret i16 0 }
+ define i16 @test_no_fold_wrong_false_val(i32 %wide) { ret i16 0 }
+ define i32 @test_no_fold_dst_geq_wide(i16 %wide) { ret i32 0 }
+ define i16 @test_no_fold_nonconstant_true(i32 %wide, i16 %t) { ret i16 0 }
+...
+
+# ============================================================================
+# Positive test: condition is i1 from G_TRUNC, true=1, false=0, DstTy < WideTy.
+# Expected transform: select(trunc(i32 to i1), i16 1, i16 0) => trunc(i32 & 1, i16)
+# ============================================================================
+---
+name: test_select_trunc_one_zero_match
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_select_trunc_one_zero_match
+ ; CHECK: %wide:reg32b(i32) = functionParameter_i32 0
+ ; CHECK: [[ONE32:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND %wide, [[ONE32]]
+ ; CHECK-NEXT: %sel:_(i16) = G_TRUNC [[AND]](i32)
+ ; CHECK-NEXT: %result:reg16b(i16) = COPY %sel(i16)
+ ; CHECK-NEXT: retValue_i16_r %result(i16)
+ %wide:reg32b(i32) = functionParameter_i32 0
+ %i1:_(i1) = G_TRUNC %wide(i32)
+ %one:_(i16) = G_CONSTANT i16 1
+ %zero:_(i16) = G_CONSTANT i16 0
+ %sel:_(i16) = G_SELECT %i1(i1), %one(i16), %zero(i16)
+ %result:reg16b(i16) = COPY %sel(i16)
+ retValue_i16_r %result(i16)
+
+# ============================================================================
+# Negative: condition is from G_ICMP (not G_TRUNC).
+# Covers matchSelectTruncOneZero line 1788: TruncMI->getOpcode() != G_TRUNC.
+# matchSelectTruncOneZero does NOT fire; no G_AND should appear.
+# ============================================================================
+---
+name: test_no_fold_cond_from_icmp
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_no_fold_cond_from_icmp
+ ; CHECK: %a:reg32b(i32) = functionParameter_i32 0
+ ; CHECK: G_ICMP
+ ; CHECK-NOT: G_AND
+ ; CHECK: retValue_i16_r
+ %a:reg32b(i32) = functionParameter_i32 0
+ %zero32:_(i32) = G_CONSTANT i32 0
+ %cond:_(i1) = G_ICMP intpred(ne), %a(i32), %zero32(i32)
+ %one:_(i16) = G_CONSTANT i16 1
+ %zero:_(i16) = G_CONSTANT i16 0
+ %sel:_(i16) = G_SELECT %cond(i1), %one(i16), %zero(i16)
+ %result:reg16b(i16) = COPY %sel(i16)
+ retValue_i16_r %result(i16)
+
+# ============================================================================
+# Negative: true value is 2, not 1.
+# Covers matchSelectTruncOneZero line 1806: !TrueOpt->Value.isOne().
+# matchSelectTruncOneZero does NOT fire; no G_AND should appear from it.
+# ============================================================================
+---
+name: test_no_fold_wrong_true_val
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_no_fold_wrong_true_val
+ ; CHECK: %wide:reg32b(i32) = functionParameter_i32 0
+ ; CHECK: G_TRUNC %wide(i32)
+ ; CHECK-NOT: G_AND %wide
+ ; CHECK: retValue_i16_r
+ %wide:reg32b(i32) = functionParameter_i32 0
+ %i1:_(i1) = G_TRUNC %wide(i32)
+ %two:_(i16) = G_CONSTANT i16 2
+ %zero:_(i16) = G_CONSTANT i16 0
+ %sel:_(i16) = G_SELECT %i1(i1), %two(i16), %zero(i16)
+ %result:reg16b(i16) = COPY %sel(i16)
+ retValue_i16_r %result(i16)
+
+# ============================================================================
+# Negative: false value is 3, not 0.
+# Covers matchSelectTruncOneZero line 1806: !FalseOpt->Value.isZero().
+# No fold; G_SELECT survives.
+# ============================================================================
+---
+name: test_no_fold_wrong_false_val
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_no_fold_wrong_false_val
+ ; CHECK: %wide:reg32b(i32) = functionParameter_i32 0
+ ; CHECK: G_TRUNC %wide(i32)
+ ; CHECK: G_SELECT %i1(i1), %one, %three
+ %wide:reg32b(i32) = functionParameter_i32 0
+ %i1:_(i1) = G_TRUNC %wide(i32)
+ %one:_(i16) = G_CONSTANT i16 1
+ %three:_(i16) = G_CONSTANT i16 3
+ %sel:_(i16) = G_SELECT %i1(i1), %one(i16), %three(i16)
+ %result:reg16b(i16) = COPY %sel(i16)
+ retValue_i16_r %result(i16)
+
+# ============================================================================
+# Negative: DstTy (i32) >= WideTy (i16).
+# Covers matchSelectTruncOneZero line 1798:
+# DstTy.getScalarSizeInBits() >= WideTy.getScalarSizeInBits()
+# No fold from matchSelectTruncOneZero; no G_AND %wide should appear.
+# ============================================================================
+---
+name: test_no_fold_dst_geq_wide
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_no_fold_dst_geq_wide
+ ; CHECK: %wide:reg16b(i16) = functionParameter_i16 0
+ ; CHECK: G_TRUNC %wide(i16)
+ ; CHECK-NOT: G_AND %wide
+ ; CHECK: retValue_i32_r
+ %wide:reg16b(i16) = functionParameter_i16 0
+ %i1:_(i1) = G_TRUNC %wide(i16)
+ %one:_(i32) = G_CONSTANT i32 1
+ %zero:_(i32) = G_CONSTANT i32 0
+ %sel:_(i32) = G_SELECT %i1(i1), %one(i32), %zero(i32)
+ %result:reg32b(i32) = COPY %sel(i32)
+ retValue_i32_r %result(i32)
+
+# ============================================================================
+# Negative: true operand is not a constant register.
+# Covers matchSelectTruncOneZero line 1804: !TrueOpt.
+# No fold; G_SELECT survives with the non-constant true operand.
+# ============================================================================
+---
+name: test_no_fold_nonconstant_true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_no_fold_nonconstant_true
+ ; CHECK: G_TRUNC
+ ; CHECK: G_SELECT {{.*}}, %true_val,
+ %wide:reg32b(i32) = functionParameter_i32 0
+ %true_val:reg16b(i16) = functionParameter_i16 1
+ %i1:_(i1) = G_TRUNC %wide(i32)
+ %zero:_(i16) = G_CONSTANT i16 0
+ %sel:_(i16) = G_SELECT %i1(i1), %true_val(i16), %zero(i16)
+ %result:reg16b(i16) = COPY %sel(i16)
+ retValue_i16_r %result(i16)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir
new file mode 100644
index 0000000000000..d492c5fd6c653
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck -check-prefix=COMBINER %s
+--- |
+
+ define i8 @test_trunc_and_trunc(i32 %a) {
+ %x = trunc i32 %a to i16
+ %y = and i16 %x, 7
+ %z = trunc i16 %y to i8
+ ret i8 %z
+ }
+
+ define i8 @test_trunc_or_trunc(i32 %a) {
+ %x = trunc i32 %a to i16
+ %y = or i16 %x, 7
+ %z = trunc i16 %y to i8
+ ret i8 %z
+ }
+
+ define i8 @test_trunc_xor_trunc(i32 %a) {
+ %x = trunc i32 %a to i16
+ %y = xor i16 %x, 7
+ %z = trunc i16 %y to i8
+ ret i8 %z
+ }
+...
+---
+name: test_trunc_and_trunc
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; COMBINER-LABEL: name: test_trunc_and_trunc
+ ; COMBINER: %0:reg32b(i32) = functionParameter_i32 0
+ ; COMBINER-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; COMBINER-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND %0, [[C]]
+ ; COMBINER-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[AND]](i32)
+ ; COMBINER-NEXT: [[COPY:%[0-9]+]]:reg8b(i8) = COPY [[TRUNC]](i8)
+ ; COMBINER-NEXT: retValue_i8_r [[COPY]](i8)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %2:_(i16) = G_CONSTANT i16 7
+ %1:_(i16) = G_TRUNC %0(i32)
+ %3:_(i16) = G_AND %1, %2
+ %4:_(i8) = G_TRUNC %3(i16)
+ %5:reg8b(i8) = COPY %4(i8)
+ retValue_i8_r %5(i8)
+...
+---
+name: test_trunc_or_trunc
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; COMBINER-LABEL: name: test_trunc_or_trunc
+ ; COMBINER: %0:reg32b(i32) = functionParameter_i32 0
+ ; COMBINER-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; COMBINER-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR %0, [[C]]
+ ; COMBINER-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[OR]](i32)
+ ; COMBINER-NEXT: [[COPY:%[0-9]+]]:reg8b(i8) = COPY [[TRUNC]](i8)
+ ; COMBINER-NEXT: retValue_i8_r [[COPY]](i8)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %2:_(i16) = G_CONSTANT i16 7
+ %1:_(i16) = G_TRUNC %0(i32)
+ %3:_(i16) = G_OR %1, %2
+ %4:_(i8) = G_TRUNC %3(i16)
+ %5:reg8b(i8) = COPY %4(i8)
+ retValue_i8_r %5(i8)
+...
+---
+name: test_trunc_xor_trunc
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ ; COMBINER-LABEL: name: test_trunc_xor_trunc
+ ; COMBINER: %0:reg32b(i32) = functionParameter_i32 0
+ ; COMBINER-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; COMBINER-NEXT: [[XOR:%[0-9]+]]:_(i32) = G_XOR %0, [[C]]
+ ; COMBINER-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[XOR]](i32)
+ ; COMBINER-NEXT: [[COPY:%[0-9]+]]:reg8b(i8) = COPY [[TRUNC]](i8)
+ ; COMBINER-NEXT: retValue_i8_r [[COPY]](i8)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %2:_(i16) = G_CONSTANT i16 7
+ %1:_(i16) = G_TRUNC %0(i32)
+ %3:_(i16) = G_XOR %1, %2
+ %4:_(i8) = G_TRUNC %3(i16)
+ %5:reg8b(i8) = COPY %4(i8)
+ retValue_i8_r %5(i8)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
new file mode 100644
index 0000000000000..b5abbdcfdb747
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
@@ -0,0 +1,56 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# Exercises PISAPreLegalizerCombinerImpl::applyTruncatedShift. The combine
+# rule trunc_shift_to_vector_extract turns:
+# %lo = trunc i32 %x to i16
+# %sh = lshr i32 %x, 16
+# %hi = trunc i32 %sh to i16
+# into:
+# %1 = bitcast i32 %x to <2 x i16>
+# %hi = extractelement <2 x i16> %1, 1
+# %lo = extractelement <2 x i16> %1, 0
+#
+# This test specifically covers the inner loop body that, after matching
+# the high-part trunc as the root, walks `LshMI.getOperand(1)`'s users to
+# find and rewrite the matching low-part G_TRUNC. The body of the
+# `if (DstSize == NarrowSize)` is the location flagged in the coverage
+# report for this file; both the lo and hi extracts must appear in the
+# output.
+
+--- |
+ define i16 @lo_and_hi_truncs(i32 %x) {
+ ret i16 0
+ }
+...
+---
+name: lo_and_hi_truncs
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: lo_and_hi_truncs
+ ; CHECK: %0:reg32b(i32) = functionParameter_i32 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST %0(i32)
+ ; CHECK-NEXT: %lo:_(i16) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<2 x i16>), [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %hi:_(i16) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<2 x i16>), [[C1]](i32)
+ ; CHECK-NEXT: %lo32:_(i32) = G_ANYEXT %lo(i16)
+ ; CHECK-NEXT: %hi32:_(i32) = G_ANYEXT %hi(i16)
+ ; CHECK-NEXT: %sum:_(i32) = G_ADD %lo32, %hi32
+ ; CHECK-NEXT: %ret16:_(i16) = G_TRUNC %sum(i32)
+ ; CHECK-NEXT: %r:reg16b(i16) = COPY %ret16(i16)
+ ; CHECK-NEXT: retValue_i16_r %r(i16)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %shamt:_(i32) = G_CONSTANT i32 16
+ %sh:_(i32) = G_LSHR %0:reg32b(i32), %shamt:_(i32)
+ %lo:_(i16) = G_TRUNC %0:reg32b(i32)
+ %hi:_(i16) = G_TRUNC %sh:_(i32)
+ %lo32:_(i32) = G_ANYEXT %lo:_(i16)
+ %hi32:_(i32) = G_ANYEXT %hi:_(i16)
+ %sum:_(i32) = G_ADD %lo32:_(i32), %hi32:_(i32)
+ %ret16:_(i16) = G_TRUNC %sum:_(i32)
+ %r:reg16b(i16) = COPY %ret16:_(i16)
+ retValue_i16_r %r:reg16b(i16)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir b/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
new file mode 100644
index 0000000000000..86d7ce2b0bbc9
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
@@ -0,0 +1,63 @@
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# Tests for PISAPostLegalizerCombinerImpl::matchFixIllegalShiftAmt /
+# applyFixIllegalShiftAmt.
+#
+# The post-legalizer combiner fires when a G_SHL/G_LSHR/G_ASHR has a shift
+# amount whose type is not i32. mul_to_shl and similar rules can introduce
+# such shifts. applyFixIllegalShiftAmt corrects the amount type by:
+# - inserting G_TRUNC if the amount type is wider than i32 (e.g. i64)
+# - inserting G_ZEXT if the amount type is narrower than i32 (e.g. i16)
+
+--- |
+ define i64 @test_shl_i64_i64_amt(i64 %val, i64 %amt) { ret i64 0 }
+ define i32 @test_ashr_i32_i16_amt(i32 %val, i16 %amt) { ret i32 0 }
+...
+
+# ============================================================================
+# Trunc path: shift amount is i64 (>32 bits) -> G_TRUNC inserted to i32.
+# Covers applyFixIllegalShiftAmt line 1673: MIB.buildTrunc(I32, ShAmtReg)
+# ============================================================================
+---
+name: test_shl_i64_i64_amt
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_shl_i64_i64_amt
+ ; CHECK: %val:reg64b(i64) = functionParameter_i64 0
+ ; CHECK-NEXT: %amt:reg64b(i64) = functionParameter_i64 1
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC %amt(i64)
+ ; CHECK-NEXT: %result:_(i64) = G_SHL %val, [[TRUNC]](i32)
+ ; CHECK-NEXT: %r:reg64b(i64) = COPY %result(i64)
+ ; CHECK-NEXT: retValue_i64_r %r(i64)
+ %val:reg64b(i64) = functionParameter_i64 0
+ %amt:reg64b(i64) = functionParameter_i64 1
+ %result:_(i64) = G_SHL %val(i64), %amt(i64)
+ %r:reg64b(i64) = COPY %result(i64)
+ retValue_i64_r %r:reg64b(i64)
+
+# ============================================================================
+# ZExt path: shift amount is i16 (<32 bits) -> G_ZEXT inserted to i32.
+# Covers applyFixIllegalShiftAmt line 1674: MIB.buildZExt(I32, ShAmtReg)
+# ============================================================================
+---
+name: test_ashr_i32_i16_amt
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_ashr_i32_i16_amt
+ ; CHECK: %val:reg32b(i32) = functionParameter_i32 0
+ ; CHECK-NEXT: %amt:reg16b(i16) = functionParameter_i16 1
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT %amt(i16)
+ ; CHECK-NEXT: %result:_(i32) = G_ASHR %val, [[ZEXT]](i32)
+ ; CHECK-NEXT: %r:reg32b(i32) = COPY %result(i32)
+ ; CHECK-NEXT: retValue_i32_r %r(i32)
+ %val:reg32b(i32) = functionParameter_i32 0
+ %amt:reg16b(i16) = functionParameter_i16 1
+ %result:_(i32) = G_ASHR %val(i32), %amt(i16)
+ %r:reg32b(i32) = COPY %result(i32)
+ retValue_i32_r %r:reg32b(i32)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir
new file mode 100644
index 0000000000000..adc32984beca6
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir
@@ -0,0 +1,76 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Sub-byte scalar G_STORE paths unreachable from .ll: the prelegalizer combiner
+# rewrites every sub-byte store value into a single G_TRUNC, so only hand-written
+# MIR keeps the original def and covers the remaining branches.
+
+---
+# G_CONSTANT source: no wider def to fold, so the value is widened via G_INSERT
+# into an undef byte rather than an illegal G_ANYEXT from i4.
+name: store_i4_no_chain_uses_insert
+body: |
+ bb.0:
+ liveins: $reg64b_0
+
+ ; CHECK-LABEL: name: store_i4_no_chain_uses_insert
+ ; CHECK: liveins: $reg64b_0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p1) = COPY $reg64b_0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 7
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 -16
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[DEF]], [[C1]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i16) = G_OR [[AND]], [[C]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[OR]](i16)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i8) = COPY [[TRUNC]](i8)
+ ; CHECK-NEXT: G_STORE [[COPY]](i8), %ptr(p1) :: (store (i8), addrspace 1)
+ %ptr:_(p1) = COPY $reg64b_0
+ %val:_(i4) = G_CONSTANT i4 7
+ G_STORE %val(i4), %ptr(p1) :: (store (i4), addrspace 1)
+...
+
+---
+# Multi-level G_TRUNC chain (i16 -> i6 -> i4): the walk skips the sub-byte
+# intermediate (i6) and truncates the byte-or-wider source (i16) directly.
+name: store_i4_multilevel_trunc_chain
+body: |
+ bb.0:
+ liveins: $reg64b_0, $reg16b_0
+
+ ; CHECK-LABEL: name: store_i4_multilevel_trunc_chain
+ ; CHECK: liveins: $reg64b_0, $reg16b_0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p1) = COPY $reg64b_0
+ ; CHECK-NEXT: %w:_(i16) = COPY $reg16b_0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC %w(i16)
+ ; CHECK-NEXT: G_STORE [[TRUNC]](i8), %ptr(p1) :: (store (i8), addrspace 1)
+ %ptr:_(p1) = COPY $reg64b_0
+ %w:_(i16) = COPY $reg16b_0
+ %t6:_(i6) = G_TRUNC %w(i16)
+ %val:_(i4) = G_TRUNC %t6(i6)
+ G_STORE %val(i4), %ptr(p1) :: (store (i4), addrspace 1)
+...
+
+---
+# Same chain walk reached through G_BITCAST links, exercising the G_BITCAST arm
+# while folding to the wider i16 source.
+name: store_i4_bitcast_chain
+body: |
+ bb.0:
+ liveins: $reg64b_0, $reg16b_0
+
+ ; CHECK-LABEL: name: store_i4_bitcast_chain
+ ; CHECK: liveins: $reg64b_0, $reg16b_0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %ptr:_(p1) = COPY $reg64b_0
+ ; CHECK-NEXT: %w:_(i16) = COPY $reg16b_0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC %w(i16)
+ ; CHECK-NEXT: G_STORE [[TRUNC]](i8), %ptr(p1) :: (store (i8), addrspace 1)
+ %ptr:_(p1) = COPY $reg64b_0
+ %w:_(i16) = COPY $reg16b_0
+ %t4:_(i4) = G_TRUNC %w(i16)
+ %v2i2:_(<2 x i2>) = G_BITCAST %t4(i4)
+ %val:_(i4) = G_BITCAST %v2i2(<2 x i2>)
+ G_STORE %val(i4), %ptr(p1) :: (store (i4), addrspace 1)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalize-threeway-cmp.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalize-threeway-cmp.mir
new file mode 100644
index 0000000000000..5b9b37652e6af
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalize-threeway-cmp.mir
@@ -0,0 +1,90 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=pisa -run-pass=legalizer -global-isel-abort=1 -verify-machineinstrs %s -o - | FileCheck %s
+---
+name: test_ucmp
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_ucmp
+ ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[C1]](i1), [[C3]], [[C4]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i16) = G_SELECT [[C2]](i1), [[C5]], [[SELECT]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[SELECT1]](i16)
+ ; CHECK-NEXT: G_STORE [[TRUNC]](i8), [[C]](p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:_(i64) = G_CONSTANT i64 0
+ %2:_(p1) = G_CONSTANT i64 0
+ %1:_(i8) = G_UCMP %0(i64), %0
+ G_STORE %1(i8), %2(p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+
+...
+---
+name: test_scmp
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_scmp
+ ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[C1]](i1), [[C3]], [[C4]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i16) = G_SELECT [[C2]](i1), [[C5]], [[SELECT]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[SELECT1]](i16)
+ ; CHECK-NEXT: G_STORE [[TRUNC]](i8), [[C]](p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:_(i64) = G_CONSTANT i64 0
+ %2:_(p1) = G_CONSTANT i64 0
+ %1:_(i8) = G_SCMP %0(i64), %0
+ G_STORE %1(i8), %2(p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+
+...
+---
+name: test_scmpv
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_scmpv
+ ; CHECK: %0:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 0
+ ; CHECK-NEXT: %1:regv4_32b(<4 x i32>) = loadParam_v4i32 1, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %0(<4 x i32>)
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(i32), [[UV5:%[0-9]+]]:_(i32), [[UV6:%[0-9]+]]:_(i32), [[UV7:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %1(<4 x i32>)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(sgt), [[UV]](i32), [[UV4]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(sgt), [[UV1]](i32), [[UV5]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(i1) = G_ICMP intpred(sgt), [[UV2]](i32), [[UV6]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(i1) = G_ICMP intpred(sgt), [[UV3]](i32), [[UV7]]
+ ; CHECK-NEXT: [[UV8:%[0-9]+]]:_(i32), [[UV9:%[0-9]+]]:_(i32), [[UV10:%[0-9]+]]:_(i32), [[UV11:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %0(<4 x i32>)
+ ; CHECK-NEXT: [[UV12:%[0-9]+]]:_(i32), [[UV13:%[0-9]+]]:_(i32), [[UV14:%[0-9]+]]:_(i32), [[UV15:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %1(<4 x i32>)
+ ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[UV8]](i32), [[UV12]]
+ ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[UV9]](i32), [[UV13]]
+ ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[UV10]](i32), [[UV14]]
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[UV11]](i32), [[UV15]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C1]]
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i32) = G_SELECT [[ICMP1]](i1), [[C2]], [[C1]]
+ ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(i32) = G_SELECT [[ICMP2]](i1), [[C2]], [[C1]]
+ ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[ICMP3]](i1), [[C2]], [[C1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(i32) = G_SELECT [[ICMP4]](i1), [[C3]], [[SELECT]]
+ ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(i32) = G_SELECT [[ICMP5]](i1), [[C3]], [[SELECT1]]
+ ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(i32) = G_SELECT [[ICMP6]](i1), [[C3]], [[SELECT2]]
+ ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(i32) = G_SELECT [[ICMP7]](i1), [[C3]], [[SELECT3]]
+ ; CHECK-NEXT: [[BUILDVEC:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[SELECT4]](i32), [[SELECT5]](i32), [[SELECT6]](i32), [[SELECT7]](i32)
+ ; CHECK-NEXT: G_STORE [[BUILDVEC]](<4 x i32>), [[C]](p1) :: (store (<4 x i32>) into `ptr addrspace(1) null`, align 4, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 0
+ %1:regv4_32b(<4 x i32>) = loadParam_v4i32 1, 0
+ %3:_(p1) = G_CONSTANT i64 0
+ %2:_(<4 x i32>) = G_SCMP %0(<4 x i32>), %1
+ G_STORE %2(<4 x i32>), %3(p1) :: (store (<4 x i32>) into `ptr addrspace(1) null`, align 4, addrspace 1)
+ ret
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir
new file mode 100644
index 0000000000000..8543f42709ca5
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir
@@ -0,0 +1,182 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
+---
+name: test_2xi16_or
+body: |
+ bb.1.entry:
+ ; LEGAL-LABEL: name: test_2xi16_or
+ ; LEGAL: %0:regv2_16b(<2 x i16>) = functionParameter_v2i16 0
+ ; LEGAL-NEXT: %1:regv2_16b(<2 x i16>) = functionParameter_v2i16 1
+ ; LEGAL-NEXT: %2:regv2_16b(<2 x i16>) = functionParameter_v2i16 2
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST %0(<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST %1(<2 x i16>)
+ ; LEGAL-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST]], [[BITCAST1]]
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
+ ; LEGAL-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST2]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST %2(<2 x i16>)
+ ; LEGAL-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[BITCAST4]]
+ ; LEGAL-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR1]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv2_16b(<2 x i16>) = COPY [[BITCAST5]](<2 x i16>)
+ ; LEGAL-NEXT: retValue_v2i16_r [[COPY]](<2 x i16>)
+ %0:regv2_16b(<2 x i16>) = functionParameter_v2i16 0
+ %1:regv2_16b(<2 x i16>) = functionParameter_v2i16 1
+ %2:regv2_16b(<2 x i16>) = functionParameter_v2i16 2
+ %3:_(<2 x i16>) = G_OR %0, %1
+ %4:_(<2 x i16>) = G_OR %3, %2
+ %5:regv2_16b(<2 x i16>) = COPY %4
+ retValue_v2i16_r %5
+
+...
+---
+name: test_3xi16_or
+body: |
+ bb.1.entry:
+ ; LEGAL-LABEL: name: test_3xi16_or
+ ; LEGAL: %0:regv3_16b(<3 x i16>) = functionParameter_v3i16 0
+ ; LEGAL-NEXT: %1:regv3_16b(<3 x i16>) = functionParameter_v3i16 1
+ ; LEGAL-NEXT: %2:regv3_16b(<3 x i16>) = functionParameter_v3i16 2
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i16), [[UV1:%[0-9]+]]:_(i16), [[UV2:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES %0(<3 x i16>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[UV]](i16), [[UV1]](i16)
+ ; LEGAL-NEXT: [[UV3:%[0-9]+]]:_(i16), [[UV4:%[0-9]+]]:_(i16), [[UV5:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES %1(<3 x i16>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[UV3]](i16), [[UV4]](i16)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[BUILD_VECTOR]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BUILD_VECTOR1]](<2 x i16>)
+ ; LEGAL-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST]], [[BITCAST1]]
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
+ ; LEGAL-NEXT: [[OR1:%[0-9]+]]:_(i16) = G_OR [[UV2]], [[UV5]]
+ ; LEGAL-NEXT: [[UV6:%[0-9]+]]:_(i16), [[UV7:%[0-9]+]]:_(i16), [[UV8:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES %2(<3 x i16>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[UV6]](i16), [[UV7]](i16)
+ ; LEGAL-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST2]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[BUILD_VECTOR2]](<2 x i16>)
+ ; LEGAL-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[BITCAST4]]
+ ; LEGAL-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR2]](i32)
+ ; LEGAL-NEXT: [[OR3:%[0-9]+]]:_(i16) = G_OR [[OR1]], [[UV8]]
+ ; LEGAL-NEXT: [[UV9:%[0-9]+]]:_(i16), [[UV10:%[0-9]+]]:_(i16) = G_UNMERGE_VALUES [[BITCAST5]](<2 x i16>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<3 x i16>) = G_BUILD_VECTOR [[UV9]](i16), [[UV10]](i16), [[OR3]](i16)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv3_16b(<3 x i16>) = COPY [[BUILD_VECTOR3]](<3 x i16>)
+ ; LEGAL-NEXT: retValue_v3i16_r [[COPY]](<3 x i16>)
+ %0:regv3_16b(<3 x i16>) = functionParameter_v3i16 0
+ %1:regv3_16b(<3 x i16>) = functionParameter_v3i16 1
+ %2:regv3_16b(<3 x i16>) = functionParameter_v3i16 2
+ %3:_(<3 x i16>) = G_OR %0, %1
+ %4:_(<3 x i16>) = G_OR %3, %2
+ %5:regv3_16b(<3 x i16>) = COPY %4
+ retValue_v3i16_r %5
+
+...
+---
+name: test_4xi16_or
+body: |
+ bb.1.entry:
+ ; LEGAL-LABEL: name: test_4xi16_or
+ ; LEGAL: %0:regv4_16b(<4 x i16>) = functionParameter_v4i16 0
+ ; LEGAL-NEXT: %1:regv4_16b(<4 x i16>) = functionParameter_v4i16 1
+ ; LEGAL-NEXT: %2:regv4_16b(<4 x i16>) = functionParameter_v4i16 2
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(<2 x i16>), [[UV1:%[0-9]+]]:_(<2 x i16>) = G_UNMERGE_VALUES %0(<4 x i16>)
+ ; LEGAL-NEXT: [[UV2:%[0-9]+]]:_(<2 x i16>), [[UV3:%[0-9]+]]:_(<2 x i16>) = G_UNMERGE_VALUES %1(<4 x i16>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x i16>)
+ ; LEGAL-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[BITCAST]], [[BITCAST1]]
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
+ ; LEGAL-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x i16>)
+ ; LEGAL-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[BITCAST3]], [[BITCAST4]]
+ ; LEGAL-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR1]](i32)
+ ; LEGAL-NEXT: [[UV4:%[0-9]+]]:_(<2 x i16>), [[UV5:%[0-9]+]]:_(<2 x i16>) = G_UNMERGE_VALUES %2(<4 x i16>)
+ ; LEGAL-NEXT: [[BITCAST6:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST2]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST7:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x i16>)
+ ; LEGAL-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[BITCAST6]], [[BITCAST7]]
+ ; LEGAL-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR2]](i32)
+ ; LEGAL-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST5]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x i16>)
+ ; LEGAL-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[BITCAST9]], [[BITCAST10]]
+ ; LEGAL-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR3]](i32)
+ ; LEGAL-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x i16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x i16>), [[BITCAST11]](<2 x i16>)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv4_16b(<4 x i16>) = COPY [[CONCAT_VECTORS]](<4 x i16>)
+ ; LEGAL-NEXT: retValue_v4i16_r [[COPY]](<4 x i16>)
+ %0:regv4_16b(<4 x i16>) = functionParameter_v4i16 0
+ %1:regv4_16b(<4 x i16>) = functionParameter_v4i16 1
+ %2:regv4_16b(<4 x i16>) = functionParameter_v4i16 2
+ %3:_(<4 x i16>) = G_OR %0, %1
+ %4:_(<4 x i16>) = G_OR %3, %2
+ %5:regv4_16b(<4 x i16>) = COPY %4
+ retValue_v4i16_r %5
+
+...
+---
+name: test_2xi16_and
+body: |
+ bb.1.entry:
+ ; LEGAL-LABEL: name: test_2xi16_and
+ ; LEGAL: %0:regv2_16b(<2 x i16>) = functionParameter_v2i16 0
+ ; LEGAL-NEXT: %1:regv2_16b(<2 x i16>) = functionParameter_v2i16 1
+ ; LEGAL-NEXT: %2:regv2_16b(<2 x i16>) = functionParameter_v2i16 2
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST %0(<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST %1(<2 x i16>)
+ ; LEGAL-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[BITCAST1]]
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[AND]](i32)
+ ; LEGAL-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST2]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST %2(<2 x i16>)
+ ; LEGAL-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST3]], [[BITCAST4]]
+ ; LEGAL-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[AND1]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv2_16b(<2 x i16>) = COPY [[BITCAST5]](<2 x i16>)
+ ; LEGAL-NEXT: retValue_v2i16_r [[COPY]](<2 x i16>)
+ %0:regv2_16b(<2 x i16>) = functionParameter_v2i16 0
+ %1:regv2_16b(<2 x i16>) = functionParameter_v2i16 1
+ %2:regv2_16b(<2 x i16>) = functionParameter_v2i16 2
+ %3:_(<2 x i16>) = G_AND %0, %1
+ %4:_(<2 x i16>) = G_AND %3, %2
+ %5:regv2_16b(<2 x i16>) = COPY %4
+ retValue_v2i16_r %5
+...
+---
+name: test_2xi16_xor
+body: |
+ bb.1.entry:
+ ; LEGAL-LABEL: name: test_2xi16_xor
+ ; LEGAL: %0:regv2_16b(<2 x i16>) = functionParameter_v2i16 0
+ ; LEGAL-NEXT: %1:regv2_16b(<2 x i16>) = functionParameter_v2i16 1
+ ; LEGAL-NEXT: %2:regv2_16b(<2 x i16>) = functionParameter_v2i16 2
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST %0(<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST %1(<2 x i16>)
+ ; LEGAL-NEXT: [[XOR:%[0-9]+]]:_(i32) = G_XOR [[BITCAST]], [[BITCAST1]]
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[XOR]](i32)
+ ; LEGAL-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST2]](<2 x i16>)
+ ; LEGAL-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST %2(<2 x i16>)
+ ; LEGAL-NEXT: [[XOR1:%[0-9]+]]:_(i32) = G_XOR [[BITCAST3]], [[BITCAST4]]
+ ; LEGAL-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[XOR1]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv2_16b(<2 x i16>) = COPY [[BITCAST5]](<2 x i16>)
+ ; LEGAL-NEXT: retValue_v2i16_r [[COPY]](<2 x i16>)
+ %0:regv2_16b(<2 x i16>) = functionParameter_v2i16 0
+ %1:regv2_16b(<2 x i16>) = functionParameter_v2i16 1
+ %2:regv2_16b(<2 x i16>) = functionParameter_v2i16 2
+ %3:_(<2 x i16>) = G_XOR %0, %1
+ %4:_(<2 x i16>) = G_XOR %3, %2
+ %5:regv2_16b(<2 x i16>) = COPY %4
+ retValue_v2i16_r %5
+...
+---
+name: test_2xi32_and
+body: |
+ bb.1.entry:
+ ; LEGAL-LABEL: name: test_2xi32_and
+ ; LEGAL: %0:regv2_32b(<2 x i32>) = functionParameter_v2i32 0
+ ; LEGAL-NEXT: %1:regv2_32b(<2 x i32>) = functionParameter_v2i32 1
+ ; LEGAL-NEXT: %2:regv2_32b(<2 x i32>) = functionParameter_v2i32 2
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %0(<2 x i32>)
+ ; LEGAL-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %1(<2 x i32>)
+ ; LEGAL-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[UV2]]
+ ; LEGAL-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[UV3]]
+ ; LEGAL-NEXT: [[UV4:%[0-9]+]]:_(i32), [[UV5:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %2(<2 x i32>)
+ ; LEGAL-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[AND]], [[UV4]]
+ ; LEGAL-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[AND1]], [[UV5]]
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[AND2]](i32), [[AND3]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv2_32b(<2 x i32>) = COPY [[BUILD_VECTOR]](<2 x i32>)
+ ; LEGAL-NEXT: retValue_v2i32_r [[COPY]](<2 x i32>)
+ %0:regv2_32b(<2 x i32>) = functionParameter_v2i32 0
+ %1:regv2_32b(<2 x i32>) = functionParameter_v2i32 1
+ %2:regv2_32b(<2 x i32>) = functionParameter_v2i32 2
+ %3:_(<2 x i32>) = G_AND %0, %1
+ %4:_(<2 x i32>) = G_AND %3, %2
+ %5:regv2_32b(<2 x i32>) = COPY %4
+ retValue_v2i32_r %5
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir
new file mode 100644
index 0000000000000..411243b1a87f6
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir
@@ -0,0 +1,40 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
+
+# Check that G_CONCAT_VECTORS of vectors larger than 4 elts are legalized.
+--- |
+ define pisa_kernel void @test_concat_clamp_elts(ptr addrspace(1) writeonly captures(none) initializes((0, 3)) %A) local_unnamed_addr {
+ %L6 = load <3 x i16>, ptr addrspace(1) null, align 4294967296
+ %bc = bitcast <3 x i16> %L6 to <2 x i24>
+ %1 = extractelement <2 x i24> %bc, i64 0
+ %2 = zext <3 x i16> %L6 to <3 x i32>
+ %3 = bitcast <3 x i32> %2 to <6 x i16>
+ %A9.sroa.0.sroa.0.0.vec.expand = shufflevector <6 x i16> %3, <6 x i16> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %A9.sroa.0.sroa.0.0.vecblend = shufflevector <12 x i16> %A9.sroa.0.sroa.0.0.vec.expand, <12 x i16> <i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+ store i24 %1, ptr addrspace(1) %A, align 4
+ store <12 x i16> %A9.sroa.0.sroa.0.0.vecblend, ptr addrspace(1) null, align 4294967296
+ ret void
+ }
+...
+---
+name: test_concat_clamp_elts
+tracksRegLiveness: true
+body: |
+ bb.1 (%ir-block.0):
+ %0:reg64b(p1) = loadParam_i64 0, 0
+ %2:_(p1) = G_CONSTANT i64 0
+ %5:_(i32) = G_CONSTANT i32 0
+ %10:_(<12 x i16>) = G_IMPLICIT_DEF
+ %1:_(<3 x i16>) = G_LOAD %2:_(p1) :: (load (<3 x i16>) from `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ %3:_(<2 x i24>) = G_BITCAST %1:_(<3 x i16>)
+ %4:_(i24) = G_EXTRACT_VECTOR_ELT %3:_(<2 x i24>), %5:_(i32)
+ %6:_(<3 x i32>) = G_ZEXT %1:_(<3 x i16>)
+ %7:_(<6 x i16>) = G_BITCAST %6:_(<3 x i32>)
+ %14:_(<6 x i16>) = G_IMPLICIT_DEF
+ ; LEGAL-NOT: %{{[0-9]+}}:_(<12 x i16>) = G_CONCAT_VECTORS
+ %15:_(<12 x i16>) = G_CONCAT_VECTORS %7:_(<6 x i16>), %14:_(<6 x i16>)
+ %13:_(<12 x i16>) = G_SHUFFLE_VECTOR %15:_(<12 x i16>), %10:_, shufflemask(0, 1, 2, 3, 4, 5, undef, undef, undef, undef, undef, undef)
+ %16:_(<3 x i8>) = G_BITCAST %4:_(i24)
+ G_STORE %16:_(<3 x i8>), %0:reg64b(p1) :: (store (<3 x i8>) into %ir.A, align 4, addrspace 1)
+ G_STORE %13:_(<12 x i16>), %2:_(p1) :: (store (<12 x i16>) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir
new file mode 100644
index 0000000000000..e8fa9c84f0adb
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir
@@ -0,0 +1,158 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_constant_s32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s32
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 5
+ ; CHECK-NEXT: $reg32b_0 = COPY [[C]](i32)
+ %0:_(i32) = G_CONSTANT i32 5
+ $reg32b_0 = COPY %0
+...
+---
+name: test_constant_s64
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s64
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 -6148914691236517206
+ ; CHECK-NEXT: $reg32bx2 = COPY [[C]](i64)
+ %0:_(i64) = G_CONSTANT i64 -6148914691236517206
+ $reg32bx2 = COPY %0
+
+...
+---
+name: test_constant_s96
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s96
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 -6148886058121296224
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 -33334
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i128) = G_MERGE_VALUES [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i96) = G_TRUNC [[MV]](i128)
+ ; CHECK-NEXT: $reg32bx3 = COPY [[TRUNC]](i96)
+ %0:_(i96) = G_CONSTANT i96 -614891469095018605312352
+ $reg32bx3 = COPY %0
+
+...
+
+---
+name: test_constant_s7
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s7
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: $reg32b_0 = COPY [[C]](i32)
+ %0:_(i7) = G_CONSTANT i7 2
+ %1:_(i32) = G_ANYEXT %0
+ $reg32b_0 = COPY %1
+...
+
+---
+name: test_constant_s8
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s8
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: $reg32b_0 = COPY [[C]](i32)
+ %0:_(i8) = G_CONSTANT i8 8
+ %1:_(i32) = G_ANYEXT %0
+ $reg32b_0 = COPY %1
+...
+
+---
+name: test_constant_s16
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s16
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: $reg32b_0 = COPY [[C]](i32)
+ %0:_(i16) = G_CONSTANT i16 15
+ %1:_(i32) = G_ANYEXT %0
+ $reg32b_0 = COPY %1
+...
+
+
+---
+name: test_constant_s112
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s112
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 100
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i128) = G_MERGE_VALUES [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: $reg32bx4 = COPY [[MV]](i128)
+ %0:_(i112) = G_CONSTANT i112 100
+ %1:_(i128) = G_ANYEXT %0
+ $reg32bx4 = COPY %1
+...
+
+---
+name: test_constant_s128
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_s128
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 5
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i128) = G_MERGE_VALUES [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: $reg32bx4 = COPY [[MV]](i128)
+ %0:_(i128) = G_CONSTANT i128 5
+ $reg32bx4 = COPY %0
+...
+
+---
+name: test_constant_p4
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_p4
+ ; CHECK: [[C:%[0-9]+]]:_(p4) = G_CONSTANT i32 0
+ ; CHECK-NEXT: $reg32b_0 = COPY [[C]](p4)
+ %0:_(p4) = G_CONSTANT i32 0
+ $reg32b_0 = COPY %0
+...
+
+---
+name: test_constant_p1
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_p1
+ ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: $reg64b_0 = COPY [[C]](p1)
+ %0:_(p1) = G_CONSTANT i64 0
+ $reg64b_0 = COPY %0
+...
+
+---
+name: test_constant_p3
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_p3
+ ; CHECK: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 0
+ ; CHECK-NEXT: $reg32b_0 = COPY [[C]](p3)
+ %0:_(p3) = G_CONSTANT i32 0
+ $reg32b_0 = COPY %0
+...
+
+---
+name: test_constant_p0
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_constant_p0
+ ; CHECK: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+ ; CHECK-NEXT: $reg64b_0 = COPY [[C]](p0)
+ %0:_(p0) = G_CONSTANT i64 0
+ $reg64b_0 = COPY %0
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir
new file mode 100644
index 0000000000000..71079bbb50b91
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir
@@ -0,0 +1,15 @@
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: extract_subvector_s64
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: extract_subvector_s64
+ ; CHECK: [[SUBVECTOR_S32:%[0-9]+]]:_(<4 x i32>) = G_EXTRACT_SUBVECTOR {{%[0-9]+}}(<32 x i32>)
+ ; CHECK: [[SUBVECTOR_S64:%[0-9]+]]:_(<2 x i64>) = G_BITCAST [[SUBVECTOR_S32]](<4 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(i64) = G_CONSTANT i64 1
+ %2:_(<16 x i64>) = G_BUILD_VECTOR %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64)
+ %3:_(<2 x i64>) = G_EXTRACT_SUBVECTOR %2:_(<16 x i64>), 8
+ G_STORE %3:_(<2 x i64>), %0:_(p1) :: (store (<2 x i64>))
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
new file mode 100644
index 0000000000000..81147be2a538c
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
@@ -0,0 +1,52 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# Check that i32 vectors with an illegal number of elements are extended to the nearest legal size before extracting the element using the G_EXTRACT_VECTOR_ELT instruction.
+
+--- |
+ define pisa_kernel i32 @test_v10_32b(<10 x i32> %vec) {
+ %elem = extractelement <10 x i32> %vec, i32 5
+ ret i32 %elem
+ }
+
+ define pisa_kernel i32 @test_v20_32b(<20 x i32> %vec) {
+ %elem = extractelement <20 x i32> %vec, i32 10
+ ret i32 %elem
+ }
+...
+---
+name: test_v10_32b
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: {{^}}name: test_v10_32b
+ %2:regv2_32b(<2 x i32>) = loadParam_v2i32 0, 0
+ %3:regv2_32b(<2 x i32>) = loadParam_v2i32 0, 8
+ %4:regv2_32b(<2 x i32>) = loadParam_v2i32 0, 16
+ %5:regv2_32b(<2 x i32>) = loadParam_v2i32 0, 24
+ %6:regv2_32b(<2 x i32>) = loadParam_v2i32 0, 32
+ %1:_(<10 x i32>) = G_CONCAT_VECTORS %2:regv2_32b(<2 x i32>), %3:regv2_32b(<2 x i32>), %4:regv2_32b(<2 x i32>), %5:regv2_32b(<2 x i32>), %6:regv2_32b(<2 x i32>)
+ %8:_(i32) = G_CONSTANT i32 5
+ ; CHECK: G_EXTRACT_VECTOR_ELT
+ ; CHECK-SAME: <16 x i32>
+ %7:_(i32) = G_EXTRACT_VECTOR_ELT %1:_(<10 x i32>), %8:_(i32)
+ %9:reg32b(i32) = COPY %7:_(i32)
+ retValue_i32_r %9:reg32b(i32)
+...
+---
+name: test_v20_32b
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: {{^}}name: test_v20_32b
+ %2:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 0
+ %3:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 16
+ %4:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 32
+ %5:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 48
+ %6:regv4_32b(<4 x i32>) = loadParam_v4i32 0, 64
+ %1:_(<20 x i32>) = G_CONCAT_VECTORS %2:regv4_32b(<4 x i32>), %3:regv4_32b(<4 x i32>), %4:regv4_32b(<4 x i32>), %5:regv4_32b(<4 x i32>), %6:regv4_32b(<4 x i32>)
+ %8:_(i32) = G_CONSTANT i32 10
+ ; CHECK: G_EXTRACT_VECTOR_ELT
+ ; CHECK-SAME: <32 x i32>
+ %7:_(i32) = G_EXTRACT_VECTOR_ELT %1:_(<20 x i32>), %8:_(i32)
+ %9:reg32b(i32) = COPY %7:_(i32)
+ retValue_i32_r %9:reg32b(i32)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir
new file mode 100644
index 0000000000000..b51cc3ccf27b0
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir
@@ -0,0 +1,411 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+---
+name: test_afn_bfloat
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_afn_bfloat
+ ; CHECK: %0:reg16b(bf16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(bf16) = G_SITOFP %1(i32)
+ ; CHECK-NEXT: [[FEXP2_:%[0-9]+]]:_(bf16) = afn G_FEXP2 [[SITOFP]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(bf16) = afn G_FMUL %0, [[FEXP2_]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL]](bf16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ %0:reg16b(bf16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(bf16) = afn G_FLDEXP %0:reg16b, %1:reg32b(i32)
+ %3:reg16b(i16) = G_BITCAST %2:_(bf16)
+ retValue_i16_r %3:reg16b(i16)
+...
+
+---
+name: test_afn_half
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_afn_half
+ ; CHECK: %0:reg16b(f16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(f16) = G_SITOFP %1(i32)
+ ; CHECK-NEXT: [[FEXP2_:%[0-9]+]]:_(f16) = afn G_FEXP2 [[SITOFP]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f16) = afn G_FMUL %0, [[FEXP2_]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL]](f16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ %0:reg16b(f16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f16) = afn G_FLDEXP %0:reg16b, %1:reg32b(i32)
+ %3:reg16b(i16) = G_BITCAST %2:_(f16)
+ retValue_i16_r %3:reg16b(i16)
+...
+
+---
+name: test_afn_double
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_afn_double
+ ; CHECK: %0:reg64b(f64) = functionParameter_i64 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -3066
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 3066
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 3069
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[LSHR]](i32)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i64) = G_ZEXT [[SUB1]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 52
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i64) = G_SHL [[ZEXT]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i64) = G_SHL [[ZEXT1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f64) = G_BITCAST [[SHL]](i64)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f64) = G_BITCAST [[SHL1]](i64)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 340
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(i32) = G_ABS [[SMIN]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[ABS]](i32), [[C6]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f64) = G_SELECT [[ICMP]](i1), [[BITCAST1]], %0
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f64) = G_SELECT [[ICMP]](i1), %0, [[BITCAST1]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f64) = afn G_FMUL [[SELECT]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f64) = afn G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f64) = afn G_FMUL [[FMUL1]], [[SELECT1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg64b(i64) = G_BITCAST [[FMUL2]](f64)
+ ; CHECK-NEXT: retValue_i64_r [[COPY]](i64)
+ %0:reg64b(f64) = functionParameter_i64 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f64) = afn G_FLDEXP %0:reg64b, %1:reg32b(i32)
+ %3:reg64b(i64) = G_BITCAST %2:_(f64)
+ retValue_i64_r %3:reg64b(i64)
+...
+
+---
+name: test_bfloat
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_bfloat
+ ; CHECK: %0:reg16b(bf16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -378
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 378
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 381
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[SUB1]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i16) = G_SHL [[TRUNC]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i16) = G_SHL [[TRUNC1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(bf16) = G_BITCAST [[SHL]](i16)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(bf16) = G_BITCAST [[SHL1]](i16)
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(bf16) = G_FMUL %0, [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(bf16) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(bf16) = G_FMUL [[FMUL1]], [[BITCAST1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL2]](bf16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ %0:reg16b(bf16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(bf16) = G_FLDEXP %0:reg16b, %1:reg32b(i32)
+ %3:reg16b(i16) = G_BITCAST %2:_(bf16)
+ retValue_i16_r %3:reg16b(i16)
+...
+
+---
+name: test_half
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_half
+ ; CHECK: %0:reg16b(f16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -42
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 42
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 45
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[ADD]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i16) = G_CONSTANT i16 86
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i16) = G_MUL [[TRUNC]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i16) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i16) = G_SUB [[TRUNC]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i16) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 10
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i16) = G_SHL [[LSHR]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i16) = G_SHL [[SUB1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[SHL]](i16)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[SHL1]](i16)
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f16) = G_FMUL %0, [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f16) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f16) = G_FMUL [[FMUL1]], [[BITCAST1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL2]](f16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ %0:reg16b(f16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f16) = G_FLDEXP %0:reg16b, %1:reg32b(i32)
+ %3:reg16b(i16) = G_BITCAST %2:_(f16)
+ retValue_i16_r %3:reg16b(i16)
+...
+
+---
+name: test_float
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_float
+ ; CHECK: %0:reg32b(f32) = functionParameter_i32 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -378
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 378
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 381
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 23
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[SUB1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 42
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(i32) = G_ABS [[SMIN]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[ABS]](i32), [[C6]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f32) = G_SELECT [[ICMP]](i1), [[BITCAST1]], %0
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f32) = G_SELECT [[ICMP]](i1), %0, [[BITCAST1]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[SELECT]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[FMUL1]], [[SELECT1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = G_BITCAST [[FMUL2]](f32)
+ ; CHECK-NEXT: retValue_i32_r [[COPY]](i32)
+ %0:reg32b(f32) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f32) = G_FLDEXP %0:reg32b, %1:reg32b(i32)
+ %3:reg32b(i32) = G_BITCAST %2:_(f32)
+ retValue_i32_r %3:reg32b(i32)
+...
+
+---
+name: test_double
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_double
+ ; CHECK: %0:reg64b(f64) = functionParameter_i64 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -3066
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 3066
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 3069
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[LSHR]](i32)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i64) = G_ZEXT [[SUB1]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 52
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i64) = G_SHL [[ZEXT]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i64) = G_SHL [[ZEXT1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f64) = G_BITCAST [[SHL]](i64)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f64) = G_BITCAST [[SHL1]](i64)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 340
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(i32) = G_ABS [[SMIN]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[ABS]](i32), [[C6]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f64) = G_SELECT [[ICMP]](i1), [[BITCAST1]], %0
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f64) = G_SELECT [[ICMP]](i1), %0, [[BITCAST1]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[SELECT]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f64) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f64) = G_FMUL [[FMUL1]], [[SELECT1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg64b(i64) = G_BITCAST [[FMUL2]](f64)
+ ; CHECK-NEXT: retValue_i64_r [[COPY]](i64)
+ %0:reg64b(f64) = functionParameter_i64 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f64) = G_FLDEXP %0:reg64b, %1:reg32b(i32)
+ %3:reg64b(i64) = G_BITCAST %2:_(f64)
+ retValue_i64_r %3:reg64b(i64)
+...
+
+---
+name: test_strict_bfloat
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_strict_bfloat
+ ; CHECK: %0:reg16b(bf16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -378
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 378
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 381
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[SUB1]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i16) = G_SHL [[TRUNC]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i16) = G_SHL [[TRUNC1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(bf16) = G_BITCAST [[SHL]](i16)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(bf16) = G_BITCAST [[SHL1]](i16)
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(bf16) = G_FMUL %0, [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(bf16) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(bf16) = G_FMUL [[FMUL1]], [[BITCAST1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL2]](bf16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ %0:reg16b(bf16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(bf16) = G_STRICT_FLDEXP %0:reg16b, %1:reg32b(i32)
+ %3:reg16b(i16) = G_BITCAST %2:_(bf16)
+ retValue_i16_r %3:reg16b(i16)
+...
+
+---
+name: test_strict_half
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_strict_half
+ ; CHECK: %0:reg16b(f16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -42
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 42
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 45
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[ADD]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i16) = G_CONSTANT i16 86
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i16) = G_MUL [[TRUNC]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i16) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i16) = G_SUB [[TRUNC]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i16) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 10
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i16) = G_SHL [[LSHR]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i16) = G_SHL [[SUB1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[SHL]](i16)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[SHL1]](i16)
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f16) = G_FMUL %0, [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f16) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f16) = G_FMUL [[FMUL1]], [[BITCAST1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL2]](f16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ %0:reg16b(f16) = functionParameter_i16 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f16) = G_STRICT_FLDEXP %0:reg16b, %1:reg32b(i32)
+ %3:reg16b(i16) = G_BITCAST %2:_(f16)
+ retValue_i16_r %3:reg16b(i16)
+...
+
+---
+name: test_strict_float
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_strict_float
+ ; CHECK: %0:reg32b(f32) = functionParameter_i32 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -378
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 378
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 381
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 23
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[SUB1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[SHL]](i32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f32) = G_BITCAST [[SHL1]](i32)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 42
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(i32) = G_ABS [[SMIN]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[ABS]](i32), [[C6]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f32) = G_SELECT [[ICMP]](i1), [[BITCAST1]], %0
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f32) = G_SELECT [[ICMP]](i1), %0, [[BITCAST1]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f32) = G_FMUL [[SELECT]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f32) = G_FMUL [[FMUL1]], [[SELECT1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = G_BITCAST [[FMUL2]](f32)
+ ; CHECK-NEXT: retValue_i32_r [[COPY]](i32)
+ %0:reg32b(f32) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f32) = G_STRICT_FLDEXP %0:reg32b, %1:reg32b(i32)
+ %3:reg32b(i32) = G_BITCAST %2:_(f32)
+ retValue_i32_r %3:reg32b(i32)
+...
+
+---
+name: test_strict_double
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_strict_double
+ ; CHECK: %0:reg64b(f64) = functionParameter_i64 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 -3066
+ ; CHECK-NEXT: [[SMAX:%[0-9]+]]:_(i32) = G_SMAX %1, [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 3066
+ ; CHECK-NEXT: [[SMIN:%[0-9]+]]:_(i32) = G_SMIN [[SMAX]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 3069
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[SMIN]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 21846
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i32) = G_MUL [[ADD]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[MUL]], [[C4]](i32)
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[ADD]], [[LSHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[SUB]], [[LSHR]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[LSHR]](i32)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i64) = G_ZEXT [[SUB1]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 52
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i64) = G_SHL [[ZEXT]], [[C5]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i64) = G_SHL [[ZEXT1]], [[C5]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(f64) = G_BITCAST [[SHL]](i64)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(f64) = G_BITCAST [[SHL1]](i64)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 340
+ ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(i32) = G_ABS [[SMIN]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(slt), [[ABS]](i32), [[C6]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f64) = G_SELECT [[ICMP]](i1), [[BITCAST1]], %0
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f64) = G_SELECT [[ICMP]](i1), %0, [[BITCAST1]]
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(f64) = G_FMUL [[SELECT]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f64) = G_FMUL [[FMUL]], [[BITCAST]]
+ ; CHECK-NEXT: [[FMUL2:%[0-9]+]]:_(f64) = G_FMUL [[FMUL1]], [[SELECT1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg64b(i64) = G_BITCAST [[FMUL2]](f64)
+ ; CHECK-NEXT: retValue_i64_r [[COPY]](i64)
+ %0:reg64b(f64) = functionParameter_i64 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:_(f64) = G_STRICT_FLDEXP %0:reg64b, %1:reg32b(i32)
+ %3:reg64b(i64) = G_BITCAST %2:_(f64)
+ retValue_i64_r %3:reg64b(i64)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir
new file mode 100644
index 0000000000000..eef864f65c4ca
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir
@@ -0,0 +1,48 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Verify that G_FREEZE is legalized successfully into two i64 parts.
+# NOTE: The intermediate widening to i128 is an internal step, and does not appear in the final output.
+
+--- |
+ define pisa_kernel void @test_freeze_s96() {
+ %a = load i96, ptr addrspace(1) null, align 16
+ %a.fr = freeze i96 %a
+ %lo = trunc i96 %a.fr to i32
+ %sh = lshr i96 %a.fr, 64
+ %hi = trunc i96 %sh to i32
+ store i32 %lo, ptr addrspace(1) null, align 4
+ store i32 %hi, ptr addrspace(1) null, align 4
+ ret void
+ }
+...
+---
+name: test_freeze_s96
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ %1:_(p1) = G_CONSTANT i64 0
+ %4:_(i96) = G_CONSTANT i96 64
+ %8:_(i64) = G_LOAD %1(p1) :: (load (i64) from `ptr addrspace(1) null`, align 16, addrspace 1)
+ %80:_(i96) = G_ZEXT %8
+ %11:_(p1) = G_CONSTANT i64 8
+ %14:_(i32) = G_LOAD %11(p1) :: (load (i32) from `ptr addrspace(1) null` + 8, align 8, basealign 16, addrspace 1)
+ %140:_(i96) = G_ZEXT %14
+ %16:_(i96) = G_CONSTANT i96 64
+ %15:_(i96) = G_SHL %140, %16(i96)
+ %17:_(i96) = G_OR %80, %15
+ %2:_(i96) = G_FREEZE %17
+ %3:_(i32) = G_TRUNC %2(i96)
+ %5:_(i96) = G_LSHR %2, %4(i96)
+ %6:_(i32) = G_TRUNC %5(i96)
+
+ ; CHECK-LABEL: {{^}}name: test_freeze_s96
+ ; CHECK-NOT: (i96) = G_FREEZE
+ ; CHECK: (i64) = G_FREEZE
+ ; CHECK: (i64) = G_FREEZE
+
+ G_STORE %3(i32), %1(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ G_STORE %6(i32), %1(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir
new file mode 100644
index 0000000000000..bcaf76d745111
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir
@@ -0,0 +1,116 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: test_frem_afn
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_frem_afn
+ ; CHECK: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[FDIV:%[0-9]+]]:_(f16) = arcp G_FDIV [[DEF]], [[DEF1]]
+ ; CHECK-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(f16) = afn G_INTRINSIC_TRUNC [[FDIV]]
+ ; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(f16) = afn G_FNEG [[INTRINSIC_TRUNC]]
+ ; CHECK-NEXT: [[FMA:%[0-9]+]]:_(f16) = afn G_FMA [[FNEG]], [[DEF1]], [[DEF]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[FMA]](f16)
+ ; CHECK-NEXT: G_STORE [[BITCAST]](i16), [[C]](p1) :: (store (i16) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ %0:_(f16) = G_IMPLICIT_DEF
+ %1:_(f16) = G_IMPLICIT_DEF
+ %2:_(f16) = afn G_FREM %0, %1
+ %3:_(p1) = G_CONSTANT i64 0
+ G_STORE %2:_(f16), %3:_(p1) :: (store (i16) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+...
+
+---
+name: test_frem_fp16
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_frem_fp16
+ ; CHECK: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[FPEXT:%[0-9]+]]:_(f32) = G_FPEXT [[DEF]](f16)
+ ; CHECK-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[DEF1]](f16)
+ ; CHECK-NEXT: [[FDIV:%[0-9]+]]:_(f32) = G_FDIV [[FPEXT]], [[FPEXT1]]
+ ; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[FDIV]](f32)
+ ; CHECK-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(f16) = G_INTRINSIC_TRUNC [[FPTRUNC]]
+ ; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(f16) = G_FNEG [[INTRINSIC_TRUNC]]
+ ; CHECK-NEXT: [[FMA:%[0-9]+]]:_(f16) = G_FMA [[FNEG]], [[DEF1]], [[DEF]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f16) = G_FCONSTANT half +inf
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(f16) = G_INTRINSIC intrinsic(@llvm.pisa.fabs), [[DEF]](f16)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(f16) = G_INTRINSIC intrinsic(@llvm.pisa.fabs), [[DEF1]](f16)
+ ; CHECK-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(oeq), [[INT]](f16), [[C]]
+ ; CHECK-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(oeq), [[INT1]](f16), [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f16) = G_SELECT [[FCMP1]](i1), [[DEF]], [[FMA]]
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f16) = G_SELECT [[FCMP]](i1), [[FMA]], [[SELECT]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i16) = G_BITCAST [[SELECT1]](f16)
+ ; CHECK-NEXT: G_STORE [[BITCAST]](i16), [[C1]](p1) :: (store (i16) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ %0:_(f16) = G_IMPLICIT_DEF
+ %1:_(f16) = G_IMPLICIT_DEF
+ %2:_(f16) = G_FREM %0, %1
+ %3:_(p1) = G_CONSTANT i64 0
+ G_STORE %2:_(f16), %3:_(p1) :: (store (i16) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+
+...
+
+---
+name: test_frem_fp32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_frem_fp32
+ ; CHECK: [[DEF:%[0-9]+]]:_(f32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(f32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[FDIV:%[0-9]+]]:_(f32) = G_FDIV [[DEF]], [[DEF1]]
+ ; CHECK-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(f32) = G_INTRINSIC_TRUNC [[FDIV]]
+ ; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(f32) = G_FNEG [[INTRINSIC_TRUNC]]
+ ; CHECK-NEXT: [[FMA:%[0-9]+]]:_(f32) = G_FMA [[FNEG]], [[DEF1]], [[DEF]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f32) = G_FCONSTANT float +inf
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.pisa.fabs), [[DEF]](f32)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.pisa.fabs), [[DEF1]](f32)
+ ; CHECK-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(oeq), [[INT]](f32), [[C]]
+ ; CHECK-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(oeq), [[INT1]](f32), [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f32) = G_SELECT [[FCMP1]](i1), [[DEF]], [[FMA]]
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f32) = G_SELECT [[FCMP]](i1), [[FMA]], [[SELECT]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SELECT1]](f32)
+ ; CHECK-NEXT: G_STORE [[BITCAST]](i32), [[C1]](p1) :: (store (i32) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ %0:_(f32) = G_IMPLICIT_DEF
+ %1:_(f32) = G_IMPLICIT_DEF
+ %2:_(f32) = G_FREM %0, %1
+ %3:_(p1) = G_CONSTANT i64 0
+ G_STORE %2:_(f32), %3:_(p1) :: (store (i32) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+
+...
+
+---
+name: test_frem_fp64
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_frem_fp64
+ ; CHECK: [[DEF:%[0-9]+]]:_(f64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(f64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[FDIV:%[0-9]+]]:_(f64) = G_FDIV [[DEF]], [[DEF1]]
+ ; CHECK-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(f64) = G_INTRINSIC_TRUNC [[FDIV]]
+ ; CHECK-NEXT: [[FNEG:%[0-9]+]]:_(f64) = G_FNEG [[INTRINSIC_TRUNC]]
+ ; CHECK-NEXT: [[FMA:%[0-9]+]]:_(f64) = G_FMA [[FNEG]], [[DEF1]], [[DEF]]
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double +inf
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(f64) = G_INTRINSIC intrinsic(@llvm.pisa.fabs), [[DEF]](f64)
+ ; CHECK-NEXT: [[INT1:%[0-9]+]]:_(f64) = G_INTRINSIC intrinsic(@llvm.pisa.fabs), [[DEF1]](f64)
+ ; CHECK-NEXT: [[FCMP:%[0-9]+]]:_(i1) = G_FCMP floatpred(oeq), [[INT]](f64), [[C]]
+ ; CHECK-NEXT: [[FCMP1:%[0-9]+]]:_(i1) = G_FCMP floatpred(oeq), [[INT1]](f64), [[C]]
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(f64) = G_SELECT [[FCMP1]](i1), [[DEF]], [[FMA]]
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(f64) = G_SELECT [[FCMP]](i1), [[FMA]], [[SELECT]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i64) = G_BITCAST [[SELECT1]](f64)
+ ; CHECK-NEXT: G_STORE [[BITCAST]](i64), [[C1]](p1) :: (store (i64) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ %0:_(f64) = G_IMPLICIT_DEF
+ %1:_(f64) = G_IMPLICIT_DEF
+ %2:_(f64) = G_FREM %0, %1
+ %3:_(p1) = G_CONSTANT i64 0
+ G_STORE %2:_(f64), %3:_(p1) :: (store (i64) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir
new file mode 100644
index 0000000000000..4f6217576f36d
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir
@@ -0,0 +1,35 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: test_nonpow2_uitofp
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_nonpow2_uitofp
+ ; CHECK: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16777215
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[DEF]], [[C]]
+ ; CHECK-NEXT: [[UITOFP:%[0-9]+]]:_(f32) = G_UITOFP [[AND]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UITOFP]](f32)
+ ; CHECK-NEXT: G_STORE [[BITCAST]](i32), %2:_(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ %1:_(i24) = G_IMPLICIT_DEF
+ %2:_(f32) = G_UITOFP %1:_(i24)
+ G_STORE %2:_(f32), %3:_(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+...
+
+---
+name: test_nonpow2_sitofp
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_nonpow2_sitofp
+ ; CHECK: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[DEF]], [[C]](i32)
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i32) = G_ASHR [[SHL]], [[C]](i32)
+ ; CHECK-NEXT: [[SITOFP:%[0-9]+]]:_(f32) = G_SITOFP [[ASHR]](i32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[SITOFP]](f32)
+ ; CHECK-NEXT: G_STORE [[BITCAST]](i32), %2:_(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ %1:_(i24) = G_IMPLICIT_DEF
+ %2:_(f32) = G_SITOFP %1:_(i24)
+ G_STORE %2:_(f32), %3:_(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir
new file mode 100644
index 0000000000000..6d310a0e5c7bb
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir
@@ -0,0 +1,443 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - -verify-machineinstrs | FileCheck %s
+
+---
+name: test_s1
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s1
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i16), [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C4]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i1) = G_CONSTANT i1 0
+ %1:_(i1) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i1), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s4
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s4
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 15
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[TRUNC]], [[C1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i16), [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C4]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i4) = G_CONSTANT i4 0
+ %1:_(i4) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i4), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s8
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s8
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 255
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i16), [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C4]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i8) = G_CONSTANT i8 0
+ %1:_(i8) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i8), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s14
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s14
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i16) = G_CONSTANT i16 16383
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[TRUNC]], [[C1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i16), [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C4]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i14) = G_CONSTANT i14 0
+ %1:_(i14) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i14), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s16
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s16
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[TRUNC]](i16), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C3]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i16) = G_CONSTANT i16 0
+ %1:_(i16) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i16), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s24
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s24
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16777215
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[TRUNC]], [[C1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i32), [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C4]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i24) = G_CONSTANT i24 0
+ %1:_(i24) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i24), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s32
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s32
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC %0(i64)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[TRUNC]](i32), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C3]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i32) = G_CONSTANT i32 0
+ %1:_(i32) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i32), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s48
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s48
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 281474976710655
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i64) = G_AND %0, [[C1]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i64), [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C2]], [[C3]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C4]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i48) = G_CONSTANT i48 0
+ %1:_(i48) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i48), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s62
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s62
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 4611686018427387903
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i64) = G_AND %0, [[C2]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i64), [[C]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[ICMP]](i1), [[C3]], [[C4]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i8) = G_TRUNC [[SELECT]](i16)
+ ; CHECK-NEXT: G_STORE [[TRUNC]](i8), [[C1]](p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i62) = G_CONSTANT i62 0
+ %5:_(p1) = G_CONSTANT i64 0
+ %1:_(i62) = G_TRUNC %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i62), %2
+ %4:_(i8) = G_ZEXT %3(i1)
+ G_STORE %4(i8), %5(p1) :: (store (i8) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+
+---
+name: test_s64
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s64
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), %0(i64), [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i32) = G_SELECT [[ICMP]](i1), [[C1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT]](i32), [[C3]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i64) = G_CONSTANT i64 0
+ %3:_(i1) = G_ICMP intpred(ugt), %0(i64), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s80
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s80
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR %0, [[C2]](i32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i64) = G_AND [[ASHR]], [[C3]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), %0(i64), [[C]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i64), [[C1]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[AND]](i64), [[C1]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[ICMP]](i1), [[C5]], [[C4]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i16) = G_SELECT [[ICMP1]](i1), [[C7]], [[C6]]
+ ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(i16) = G_SELECT [[ICMP2]](i1), [[SELECT]], [[SELECT1]]
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i16) = G_AND [[SELECT2]], [[C9]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND1]](i16), [[C8]]
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[ICMP3]](i1), [[C10]], [[C11]]
+ ; CHECK-NEXT: [[C12:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT3]](i32), [[C12]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i80) = G_CONSTANT i80 0
+ %1:_(i80) = G_SEXT %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i80), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s128
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s128
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR %0, [[C2]](i32)
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), %0(i64), [[C]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[ASHR]](i64), [[C1]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[ASHR]](i64), [[C1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[ICMP]](i1), [[C4]], [[C3]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i16) = G_SELECT [[ICMP1]](i1), [[C6]], [[C5]]
+ ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(i16) = G_SELECT [[ICMP2]](i1), [[SELECT]], [[SELECT1]]
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[SELECT2]], [[C8]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND]](i16), [[C7]]
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(i32) = G_SELECT [[ICMP3]](i1), [[C9]], [[C10]]
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT3]](i32), [[C11]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i128) = G_CONSTANT i128 0
+ %1:_(i128) = G_SEXT %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i128), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
+
+---
+name: test_s160
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_s160
+ ; CHECK: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR %0, [[C4]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 -1
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 4294967295
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i64) = G_AND [[ASHR]], [[C5]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i64) = G_AND [[ASHR]], [[C6]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[ASHR]], [[C7]]
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), %0(i64), [[C]]
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND]](i64), [[C1]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[AND]](i64), [[C1]]
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[ICMP]](i1), [[C9]], [[C8]]
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i16) = G_SELECT [[ICMP1]](i1), [[C11]], [[C10]]
+ ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(i16) = G_SELECT [[ICMP2]](i1), [[SELECT]], [[SELECT1]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND1]](i64), [[C2]]
+ ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[AND1]](i64), [[C2]]
+ ; CHECK-NEXT: [[C12:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C13:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(i16) = G_SELECT [[ICMP3]](i1), [[C13]], [[C12]]
+ ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(i16) = G_SELECT [[ICMP4]](i1), [[SELECT2]], [[SELECT3]]
+ ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[AND2]](i64), [[C3]]
+ ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[AND2]](i64), [[C3]]
+ ; CHECK-NEXT: [[C14:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C15:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(i16) = G_SELECT [[ICMP5]](i1), [[C15]], [[C14]]
+ ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(i16) = G_SELECT [[ICMP6]](i1), [[SELECT4]], [[SELECT5]]
+ ; CHECK-NEXT: [[C16:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C17:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i16) = G_AND [[SELECT6]], [[C17]]
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND3]](i16), [[C16]]
+ ; CHECK-NEXT: [[C18:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[C19:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(i32) = G_SELECT [[ICMP7]](i1), [[C18]], [[C19]]
+ ; CHECK-NEXT: [[C20:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: G_STORE [[SELECT7]](i32), [[C20]](p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ; CHECK-NEXT: ret
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %2:_(i160) = G_CONSTANT i160 0
+ %1:_(i160) = G_SEXT %0(i64)
+ %3:_(i1) = G_ICMP intpred(ugt), %1(i160), %2
+ %6:_(i32) = G_CONSTANT i32 13
+ %7:_(i32) = G_CONSTANT i32 17
+ %8:_(i32) = G_SELECT %3, %6, %7
+ %5:_(p1) = G_CONSTANT i64 0
+ G_STORE %8(i32), %5(p1) :: (store (i32) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir
new file mode 100644
index 0000000000000..5b7bb46ca3308
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir
@@ -0,0 +1,367 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_implicit_def_s1
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_implicit_def_s1
+ ; CHECK: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32b_0 = COPY [[DEF]](i32)
+ %0:_(i1) = G_IMPLICIT_DEF
+ %1:_(i32) = G_ANYEXT %0
+ $reg32b_0 = COPY %1
+...
+
+---
+name: test_implicit_def_s5
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s5
+ ; CHECK: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg16b_0 = COPY [[DEF]](i16)
+ %0:_(i5) = G_IMPLICIT_DEF
+ %1:_(i16) = G_ANYEXT %0
+ $reg16b_0 = COPY %1
+...
+
+---
+name: test_implicit_def_s8
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s8
+ ; CHECK: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg16b_0 = COPY [[DEF]](i16)
+ %0:_(i8) = G_IMPLICIT_DEF
+ %1:_(i16) = G_ANYEXT %0
+ $reg16b_0 = COPY %1
+...
+
+---
+name: test_implicit_def_s16
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s16
+ ; CHECK: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg16b_0 = COPY [[DEF]](i16)
+ %0:_(i16) = G_IMPLICIT_DEF
+ $reg16b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_s24
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s24
+ ; CHECK: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32b_0 = COPY [[DEF]](i32)
+ %0:_(i24) = G_IMPLICIT_DEF
+ %1:_(i32) = G_ANYEXT %0
+ $reg32b_0 = COPY %1
+...
+
+---
+name: test_implicit_def_s32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s32
+ ; CHECK: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32b_0 = COPY [[DEF]](i32)
+ %0:_(i32) = G_IMPLICIT_DEF
+ $reg32b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_s40
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s40
+ ; CHECK: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg64b_0 = COPY [[DEF]](i64)
+ %0:_(i40) = G_IMPLICIT_DEF
+ %1:_(i64) = G_ANYEXT %0
+ $reg64b_0 = COPY %1
+...
+
+---
+name: test_implicit_def_s64
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s64
+ ; CHECK: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg64b_0 = COPY [[DEF]](i64)
+ %0:_(i64) = G_IMPLICIT_DEF
+ $reg64b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_s72
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s72
+ ; CHECK: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i64) = COPY [[DEF]](i64)
+ ; CHECK-NEXT: $reg64b_0 = COPY [[COPY]](i64)
+ %0:_(i72) = G_IMPLICIT_DEF
+ %1:_(i64) = G_TRUNC %0
+ $reg64b_0 = COPY %1
+...
+
+---
+name: test_implicit_def_s96
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s96
+ ; CHECK: [[DEF:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i96) = G_TRUNC [[DEF]](i128)
+ ; CHECK-NEXT: $reg32bx3 = COPY [[TRUNC]](i96)
+ %0:_(i96) = G_IMPLICIT_DEF
+ $reg32bx3 = COPY %0
+...
+
+---
+name: test_implicit_def_s128
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s128
+ ; CHECK: [[DEF:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg64bx2 = COPY [[DEF]](i128)
+ %0:_(i128) = G_IMPLICIT_DEF
+ $reg64bx2 = COPY %0
+...
+
+---
+name: test_implicit_def_s160
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s160
+ ; CHECK: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i256) = G_MERGE_VALUES [[DEF]](i64), [[DEF1]](i64), [[DEF2]](i64), [[DEF3]](i64)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i160) = G_TRUNC [[MV]](i256)
+ ; CHECK-NEXT: $reg32bx5 = COPY [[TRUNC]](i160)
+ %0:_(i160) = G_IMPLICIT_DEF
+ $reg32bx5 = COPY %0
+...
+
+---
+name: test_implicit_def_s256
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_s256
+ ; CHECK: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF2:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF3:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i256) = G_MERGE_VALUES [[DEF]](i64), [[DEF1]](i64), [[DEF2]](i64), [[DEF3]](i64)
+ ; CHECK-NEXT: $reg64bx4 = COPY [[MV]](i256)
+ %0:_(i256) = G_IMPLICIT_DEF
+ $reg64bx4 = COPY %0
+...
+
+---
+name: test_implicit_def_v2i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v2i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx2 = COPY [[DEF]](<2 x i32>)
+ %0:_(<2 x i32>) = G_IMPLICIT_DEF
+ $reg32bx2 = COPY %0
+...
+
+
+---
+name: test_implicit_def_v3i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v3i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<3 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx3 = COPY [[DEF]](<3 x i32>)
+ %0:_(<3 x i32>) = G_IMPLICIT_DEF
+ $reg32bx3 = COPY %0
+...
+
+---
+name: test_implicit_def_v4i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v4i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<4 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx4 = COPY [[DEF]](<4 x i32>)
+ %0:_(<4 x i32>) = G_IMPLICIT_DEF
+ $reg32bx4 = COPY %0
+...
+
+---
+name: test_implicit_def_v5i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v5i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<5 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx5 = COPY [[DEF]](<5 x i32>)
+ %0:_(<5 x i32>) = G_IMPLICIT_DEF
+ $reg32bx5 = COPY %0
+...
+
+---
+name: test_implicit_def_v6i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v6i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<6 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx6 = COPY [[DEF]](<6 x i32>)
+ %0:_(<6 x i32>) = G_IMPLICIT_DEF
+ $reg32bx6 = COPY %0
+...
+
+---
+name: test_implicit_def_v7i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v7i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<7 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx7 = COPY [[DEF]](<7 x i32>)
+ %0:_(<7 x i32>) = G_IMPLICIT_DEF
+ $reg32bx7 = COPY %0
+...
+
+---
+name: test_implicit_def_v8i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v8i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<8 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx8 = COPY [[DEF]](<8 x i32>)
+ %0:_(<8 x i32>) = G_IMPLICIT_DEF
+ $reg32bx8 = COPY %0
+...
+
+---
+name: test_implicit_def_v16i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v16i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<16 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx16 = COPY [[DEF]](<16 x i32>)
+ %0:_(<16 x i32>) = G_IMPLICIT_DEF
+ $reg32bx16 = COPY %0
+...
+
+---
+name: test_implicit_def_v32i32
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v32i32
+ ; CHECK: [[DEF:%[0-9]+]]:_(<32 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx32 = COPY [[DEF]](<32 x i32>)
+ %0:_(<32 x i32>) = G_IMPLICIT_DEF
+ $reg32bx32 = COPY %0
+...
+
+---
+name: test_implicit_def_v2i1
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v2i1
+ ; CHECK: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[DEF]](i32), [[DEF1]](i32)
+ ; CHECK-NEXT: $reg32bx2 = COPY [[BUILD_VECTOR]](<2 x i32>)
+ %0:_(<2 x i1>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_ANYEXT %0
+ $reg32bx2 = COPY %1
+...
+
+---
+name: test_implicit_def_v2i8
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v2i8
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32bx2 = COPY [[DEF]](<2 x i32>)
+ %0:_(<2 x i8>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_ANYEXT %0
+ $reg32bx2 = COPY %1
+...
+
+---
+name: test_implicit_def_v2i16
+body: |
+ bb.0:
+
+ ; CHECK-LABEL: name: test_implicit_def_v2i16
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i16>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32b_0 = COPY [[DEF]](<2 x i16>)
+ %0:_(<2 x i16>) = G_IMPLICIT_DEF
+ $reg32b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_p4
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_implicit_def_p4
+ ; CHECK: [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32b_0 = COPY [[DEF]](p4)
+ %0:_(p4) = G_IMPLICIT_DEF
+ $reg32b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_p1
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_implicit_def_p1
+ ; CHECK: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg64b_0 = COPY [[DEF]](p1)
+ %0:_(p1) = G_IMPLICIT_DEF
+ $reg64b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_p3
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_implicit_def_p3
+ ; CHECK: [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg32b_0 = COPY [[DEF]](p3)
+ %0:_(p3) = G_IMPLICIT_DEF
+ $reg32b_0 = COPY %0
+...
+
+---
+name: test_implicit_def_p0
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_implicit_def_p0
+ ; CHECK: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: $reg64b_0 = COPY [[DEF]](p0)
+ %0:_(p0) = G_IMPLICIT_DEF
+ $reg64b_0 = COPY %0
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir
new file mode 100644
index 0000000000000..c25457854eda1
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir
@@ -0,0 +1,16 @@
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: insert_subvector_s64
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: insert_subvector_s64
+ ; CHECK: [[SUBVECTOR_S32:%[0-9]+]]:_(<4 x i32>) = G_BITCAST %3(<2 x i64>)
+ ; CHECK: [[VECTOR_S32:%[0-9]+]]:_(<32 x i32>) = G_INSERT_SUBVECTOR {{%[0-9]+}}, [[SUBVECTOR_S32]](<4 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(i64) = G_CONSTANT i64 1
+ %2:_(<16 x i64>) = G_BUILD_VECTOR %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64), %1:_(i64)
+ %3:_(<2 x i64>) = G_BUILD_VECTOR %1:_(i64), %1:_(i64)
+ %4:_(<16 x i64>) = G_INSERT_SUBVECTOR %2:_(<16 x i64>), %3:_(<2 x i64>), 8
+ G_STORE %4:_(<16 x i64>), %0:_(p1) :: (store (<16 x i64>))
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir
new file mode 100644
index 0000000000000..67fda24269841
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir
@@ -0,0 +1,388 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
+# Check that loads/stores of vectors of <n x i32> elements with n
+# being [5,7] elements is resolved to a load of <8 x i32> for smaller
+# alignments. However, when alignment is > 8 bytes for slm and constant
+# buffer, and > 32 bytes for global memory, then [5, 6] elements
+# is legalized to <3 x i64> to minimize overfetch. The original
+# <n x i32> result is then extracted from this new result.
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-G1"
+
+ define <5 x i32> @load_shared_5xi32(ptr addrspace(3) %arg) {
+ %1 = load <5 x i32>, ptr addrspace(3) %arg, align 4
+ ret <5 x i32> %1
+ }
+
+ define <5 x i32> @load_shared_5xi32_8bytes_alignment(ptr addrspace(3) %arg) {
+ %1 = load <5 x i32>, ptr addrspace(3) %arg, align 8
+ ret <5 x i32> %1
+ }
+
+ define <6 x i32> @load_shared_6xi32(ptr addrspace(3) %arg) {
+ %1 = load <6 x i32>, ptr addrspace(3) %arg, align 4
+ ret <6 x i32> %1
+ }
+
+ define <6 x i32> @load_shared_6xi32_8bytes_alignment(ptr addrspace(3) %arg) {
+ %1 = load <6 x i32>, ptr addrspace(3) %arg, align 8
+ ret <6 x i32> %1
+ }
+
+ define <7 x i32> @load_shared_7xi32(ptr addrspace(3) %arg) {
+ %1 = load <7 x i32>, ptr addrspace(3) %arg, align 8
+ ret <7 x i32> %1
+ }
+
+ define <5 x i32> @load_global_5xi32(ptr addrspace(1) %arg) {
+ %1 = load <5 x i32>, ptr addrspace(1) %arg, align 4
+ ret <5 x i32> %1
+ }
+
+ define <5 x i32> @load_global_5xi32_32bytes_alignment(ptr addrspace(1) %arg) {
+ %1 = load <5 x i32>, ptr addrspace(1) %arg, align 32
+ ret <5 x i32> %1
+ }
+
+ define <6 x i32> @load_global_6xi32(ptr addrspace(1) %arg) {
+ %1 = load <6 x i32>, ptr addrspace(1) %arg, align 4
+ ret <6 x i32> %1
+ }
+
+ define <6 x i32> @load_global_6xi32_32bytes_alignment(ptr addrspace(1) %arg) {
+ %1 = load <6 x i32>, ptr addrspace(1) %arg, align 32
+ ret <6 x i32> %1
+ }
+
+ define <7 x i32> @load_global_7xi32(ptr addrspace(1) %arg) {
+ %1 = load <7 x i32>, ptr addrspace(1) %arg, align 32
+ ret <7 x i32> %1
+ }
+
+ define <5 x i32> @load_constant_5xi32(ptr addrspace(2) %arg) {
+ %1 = load <5 x i32>, ptr addrspace(2) %arg, align 4
+ ret <5 x i32> %1
+ }
+
+ define <5 x i32> @load_constant_5xi32_8bytes_alignment(ptr addrspace(2) %arg) {
+ %1 = load <5 x i32>, ptr addrspace(2) %arg, align 8
+ ret <5 x i32> %1
+ }
+
+ define <6 x i32> @load_constant_6xi32(ptr addrspace(2) %arg) {
+ %1 = load <6 x i32>, ptr addrspace(2) %arg, align 4
+ ret <6 x i32> %1
+ }
+
+ define <6 x i32> @load_constant_6xi32_8bytes_alignment(ptr addrspace(2) %arg) {
+ %1 = load <6 x i32>, ptr addrspace(2) %arg, align 8
+ ret <6 x i32> %1
+ }
+
+ define <7 x i32> @load_constant_7xi32(ptr addrspace(2) %arg) {
+ %1 = load <7 x i32>, ptr addrspace(2) %arg, align 8
+ ret <7 x i32> %1
+ }
+...
+---
+name: load_shared_5xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_shared_5xi32
+ ; LEGAL: %0:reg32b(p3) = functionParameter_i32 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD %0(p3) :: (load (<8 x i32>) from %ir.arg, align 4, addrspace 3)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<5 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 0
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<5 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<5 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv5_32b(<5 x i32>) = COPY [[COPY]](<5 x i32>)
+ ; LEGAL-NEXT: retValue_v5i32_r [[COPY1]](<5 x i32>)
+ %0:reg32b(p3) = functionParameter_i32 0
+ %1:_(<5 x i32>) = G_LOAD %0(p3) :: (load (<5 x i32>) from %ir.arg, align 4, addrspace 3)
+ %2:regv5_32b(<5 x i32>) = COPY %1(<5 x i32>)
+ retValue_v5i32_r %2(<5 x i32>)
+...
+---
+name: load_shared_5xi32_8bytes_alignment
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_shared_5xi32_8bytes_alignment
+ ; LEGAL: %0:reg32b(p3) = functionParameter_i32 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<3 x i64>) = G_LOAD %0(p3) :: (load (<3 x i64>) from %ir.arg, align 8, addrspace 3)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i64), [[UV1:%[0-9]+]]:_(i64), [[UV2:%[0-9]+]]:_(i64) = G_UNMERGE_VALUES [[LOAD]](<3 x i64>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV]](i64)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV1]](i64)
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV2]](i64)
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(<6 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BITCAST]](<2 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<2 x i32>), 2
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR2:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR1]], [[BITCAST2]](<2 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[INSERT_SUBVECTOR2]](<6 x i32>)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<5 x i32>) = G_EXTRACT_SUBVECTOR [[COPY]](<6 x i32>), 0
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:_(<5 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<5 x i32>)
+ ; LEGAL-NEXT: [[COPY2:%[0-9]+]]:regv5_32b(<5 x i32>) = COPY [[COPY1]](<5 x i32>)
+ ; LEGAL-NEXT: retValue_v5i32_r [[COPY2]](<5 x i32>)
+ %0:reg32b(p3) = functionParameter_i32 0
+ %1:_(<5 x i32>) = G_LOAD %0(p3) :: (load (<5 x i32>) from %ir.arg, align 8, addrspace 3)
+ %2:regv5_32b(<5 x i32>) = COPY %1(<5 x i32>)
+ retValue_v5i32_r %2(<5 x i32>)
+...
+---
+name: load_shared_6xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_shared_6xi32
+ ; LEGAL: %0:reg32b(p3) = functionParameter_i32 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD %0(p3) :: (load (<8 x i32>) from %ir.arg, align 4, addrspace 3)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 0
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<6 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv6_32b(<6 x i32>) = COPY [[COPY]](<6 x i32>)
+ ; LEGAL-NEXT: retValue_v6i32_r [[COPY1]](<6 x i32>)
+ %0:reg32b(p3) = functionParameter_i32 0
+ %1:_(<6 x i32>) = G_LOAD %0(p3) :: (load (<6 x i32>) from %ir.arg, align 4, addrspace 3)
+ %2:regv6_32b(<6 x i32>) = COPY %1(<6 x i32>)
+ retValue_v6i32_r %2(<6 x i32>)
+...
+---
+name: load_shared_6xi32_8bytes_alignment
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_shared_6xi32_8bytes_alignment
+ ; LEGAL: %0:reg32b(p3) = functionParameter_i32 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<3 x i64>) = G_LOAD %0(p3) :: (load (<3 x i64>) from %ir.arg, align 8, addrspace 3)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i64), [[UV1:%[0-9]+]]:_(i64), [[UV2:%[0-9]+]]:_(i64) = G_UNMERGE_VALUES [[LOAD]](<3 x i64>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV]](i64)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV1]](i64)
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV2]](i64)
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(<6 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BITCAST]](<2 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<2 x i32>), 2
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR2:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR1]], [[BITCAST2]](<2 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[INSERT_SUBVECTOR2]](<6 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv6_32b(<6 x i32>) = COPY [[COPY]](<6 x i32>)
+ ; LEGAL-NEXT: retValue_v6i32_r [[COPY1]](<6 x i32>)
+ %0:reg32b(p3) = functionParameter_i32 0
+ %1:_(<6 x i32>) = G_LOAD %0(p3) :: (load (<6 x i32>) from %ir.arg, align 8, addrspace 3)
+ %2:regv6_32b(<6 x i32>) = COPY %1(<6 x i32>)
+ retValue_v6i32_r %2(<6 x i32>)
+...
+---
+name: load_shared_7xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_shared_7xi32
+ ; LEGAL: %0:reg32b(p3) = functionParameter_i32 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD %0(p3) :: (load (<8 x i32>) from %ir.arg, align 8, addrspace 3)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<7 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 0
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<7 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<7 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv7_32b(<7 x i32>) = COPY [[COPY]](<7 x i32>)
+ ; LEGAL-NEXT: retValue_v7i32_r [[COPY1]](<7 x i32>)
+ %0:reg32b(p3) = functionParameter_i32 0
+ %1:_(<7 x i32>) = G_LOAD %0(p3) :: (load (<7 x i32>) from %ir.arg, align 8, addrspace 3)
+ %2:regv7_32b(<7 x i32>) = COPY %1(<7 x i32>)
+ retValue_v7i32_r %2(<7 x i32>)
+...
+---
+name: load_global_5xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_global_5xi32
+ ; LEGAL: %0:reg64b(p1) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<4 x i32>) = G_LOAD %0(p1) :: (load (<4 x i32>) from %ir.arg, align 4, addrspace 1)
+ ; LEGAL-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; LEGAL-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD %0, [[C]](i64)
+ ; LEGAL-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p1) :: (load (i32) from %ir.arg + 16, addrspace 1)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](<4 x i32>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32), [[UV2]](i32), [[UV3]](i32), [[LOAD1]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv5_32b(<5 x i32>) = COPY [[BUILD_VECTOR]](<5 x i32>)
+ ; LEGAL-NEXT: retValue_v5i32_r [[COPY]](<5 x i32>)
+ %0:reg64b(p1) = functionParameter_i64 0
+ %1:_(<5 x i32>) = G_LOAD %0(p1) :: (load (<5 x i32>) from %ir.arg, align 4, addrspace 1)
+ %2:regv5_32b(<5 x i32>) = COPY %1(<5 x i32>)
+ retValue_v5i32_r %2(<5 x i32>)
+...
+---
+name: load_global_5xi32_32bytes_alignment
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_global_5xi32_32bytes_alignment
+ ; LEGAL: %0:reg64b(p1) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<3 x i64>) = G_LOAD %0(p1) :: (load (<3 x i64>) from %ir.arg, align 32, addrspace 1)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i64), [[UV1:%[0-9]+]]:_(i64), [[UV2:%[0-9]+]]:_(i64) = G_UNMERGE_VALUES [[LOAD]](<3 x i64>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV]](i64)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV1]](i64)
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV2]](i64)
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(<6 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BITCAST]](<2 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<2 x i32>), 2
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR2:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR1]], [[BITCAST2]](<2 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[INSERT_SUBVECTOR2]](<6 x i32>)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<5 x i32>) = G_EXTRACT_SUBVECTOR [[COPY]](<6 x i32>), 0
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:_(<5 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<5 x i32>)
+ ; LEGAL-NEXT: [[COPY2:%[0-9]+]]:regv5_32b(<5 x i32>) = COPY [[COPY1]](<5 x i32>)
+ ; LEGAL-NEXT: retValue_v5i32_r [[COPY2]](<5 x i32>)
+ %0:reg64b(p1) = functionParameter_i64 0
+ %1:_(<5 x i32>) = G_LOAD %0(p1) :: (load (<5 x i32>) from %ir.arg, align 32, addrspace 1)
+ %2:regv5_32b(<5 x i32>) = COPY %1(<5 x i32>)
+ retValue_v5i32_r %2(<5 x i32>)
+...
+---
+name: load_global_6xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_global_6xi32
+ ; LEGAL: %0:reg64b(p1) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<4 x i32>) = G_LOAD %0(p1) :: (load (<4 x i32>) from %ir.arg, align 4, addrspace 1)
+ ; LEGAL-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; LEGAL-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD %0, [[C]](i64)
+ ; LEGAL-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[PTR_ADD]](p1) :: (load (<2 x i32>) from %ir.arg + 16, align 4, addrspace 1)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](<4 x i32>)
+ ; LEGAL-NEXT: [[UV4:%[0-9]+]]:_(i32), [[UV5:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD1]](<2 x i32>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<6 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32), [[UV2]](i32), [[UV3]](i32), [[UV4]](i32), [[UV5]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv6_32b(<6 x i32>) = COPY [[BUILD_VECTOR]](<6 x i32>)
+ ; LEGAL-NEXT: retValue_v6i32_r [[COPY]](<6 x i32>)
+ %0:reg64b(p1) = functionParameter_i64 0
+ %1:_(<6 x i32>) = G_LOAD %0(p1) :: (load (<6 x i32>) from %ir.arg, align 4, addrspace 1)
+ %2:regv6_32b(<6 x i32>) = COPY %1(<6 x i32>)
+ retValue_v6i32_r %2(<6 x i32>)
+...
+---
+name: load_global_6xi32_32bytes_alignment
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_global_6xi32_32bytes_alignment
+ ; LEGAL: %0:reg64b(p1) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<3 x i64>) = G_LOAD %0(p1) :: (load (<3 x i64>) from %ir.arg, align 32, addrspace 1)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i64), [[UV1:%[0-9]+]]:_(i64), [[UV2:%[0-9]+]]:_(i64) = G_UNMERGE_VALUES [[LOAD]](<3 x i64>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV]](i64)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV1]](i64)
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV2]](i64)
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(<6 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BITCAST]](<2 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<2 x i32>), 2
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR2:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR1]], [[BITCAST2]](<2 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[INSERT_SUBVECTOR2]](<6 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv6_32b(<6 x i32>) = COPY [[COPY]](<6 x i32>)
+ ; LEGAL-NEXT: retValue_v6i32_r [[COPY1]](<6 x i32>)
+ %0:reg64b(p1) = functionParameter_i64 0
+ %1:_(<6 x i32>) = G_LOAD %0(p1) :: (load (<6 x i32>) from %ir.arg, align 32, addrspace 1)
+ %2:regv6_32b(<6 x i32>) = COPY %1(<6 x i32>)
+ retValue_v6i32_r %2(<6 x i32>)
+...
+---
+name: load_global_7xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_global_7xi32
+ ; LEGAL: %0:reg64b(p1) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD %0(p1) :: (load (<8 x i32>) from %ir.arg, addrspace 1)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<7 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 0
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<7 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<7 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv7_32b(<7 x i32>) = COPY [[COPY]](<7 x i32>)
+ ; LEGAL-NEXT: retValue_v7i32_r [[COPY1]](<7 x i32>)
+ %0:reg64b(p1) = functionParameter_i64 0
+ %1:_(<7 x i32>) = G_LOAD %0(p1) :: (load (<7 x i32>) from %ir.arg, align 32, addrspace 1)
+ %2:regv7_32b(<7 x i32>) = COPY %1(<7 x i32>)
+ retValue_v7i32_r %2(<7 x i32>)
+...
+---
+name: load_constant_5xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_constant_5xi32
+ ; LEGAL: %0:reg64b(p2) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<4 x i32>) = G_LOAD %0(p2) :: (load (<4 x i32>) from %ir.arg, align 4, addrspace 2)
+ ; LEGAL-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; LEGAL-NEXT: [[PTR_ADD:%[0-9]+]]:_(p2) = nuw inbounds G_PTR_ADD %0, [[C]](i64)
+ ; LEGAL-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p2) :: (load (i32) from %ir.arg + 16, addrspace 2)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](<4 x i32>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32), [[UV2]](i32), [[UV3]](i32), [[LOAD1]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv5_32b(<5 x i32>) = COPY [[BUILD_VECTOR]](<5 x i32>)
+ ; LEGAL-NEXT: retValue_v5i32_r [[COPY]](<5 x i32>)
+ %0:reg64b(p2) = functionParameter_i64 0
+ %1:_(<5 x i32>) = G_LOAD %0(p2) :: (load (<5 x i32>) from %ir.arg, align 4, addrspace 2)
+ %2:regv5_32b(<5 x i32>) = COPY %1(<5 x i32>)
+ retValue_v5i32_r %2(<5 x i32>)
+...
+---
+name: load_constant_5xi32_8bytes_alignment
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_constant_5xi32_8bytes_alignment
+ ; LEGAL: %0:reg64b(p2) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<3 x i64>) = G_LOAD %0(p2) :: (load (<3 x i64>) from %ir.arg, align 8, addrspace 2)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i64), [[UV1:%[0-9]+]]:_(i64), [[UV2:%[0-9]+]]:_(i64) = G_UNMERGE_VALUES [[LOAD]](<3 x i64>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV]](i64)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV1]](i64)
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV2]](i64)
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(<6 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BITCAST]](<2 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<2 x i32>), 2
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR2:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR1]], [[BITCAST2]](<2 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[INSERT_SUBVECTOR2]](<6 x i32>)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<5 x i32>) = G_EXTRACT_SUBVECTOR [[COPY]](<6 x i32>), 0
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:_(<5 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<5 x i32>)
+ ; LEGAL-NEXT: [[COPY2:%[0-9]+]]:regv5_32b(<5 x i32>) = COPY [[COPY1]](<5 x i32>)
+ ; LEGAL-NEXT: retValue_v5i32_r [[COPY2]](<5 x i32>)
+ %0:reg64b(p2) = functionParameter_i64 0
+ %1:_(<5 x i32>) = G_LOAD %0(p2) :: (load (<5 x i32>) from %ir.arg, align 8, addrspace 2)
+ %2:regv5_32b(<5 x i32>) = COPY %1(<5 x i32>)
+ retValue_v5i32_r %2(<5 x i32>)
+...
+---
+name: load_constant_6xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_constant_6xi32
+ ; LEGAL: %0:reg64b(p2) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<4 x i32>) = G_LOAD %0(p2) :: (load (<4 x i32>) from %ir.arg, align 4, addrspace 2)
+ ; LEGAL-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 16
+ ; LEGAL-NEXT: [[PTR_ADD:%[0-9]+]]:_(p2) = nuw inbounds G_PTR_ADD %0, [[C]](i64)
+ ; LEGAL-NEXT: [[LOAD1:%[0-9]+]]:_(<2 x i32>) = G_LOAD [[PTR_ADD]](p2) :: (load (<2 x i32>) from %ir.arg + 16, align 4, addrspace 2)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD]](<4 x i32>)
+ ; LEGAL-NEXT: [[UV4:%[0-9]+]]:_(i32), [[UV5:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[LOAD1]](<2 x i32>)
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<6 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32), [[UV2]](i32), [[UV3]](i32), [[UV4]](i32), [[UV5]](i32)
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:regv6_32b(<6 x i32>) = COPY [[BUILD_VECTOR]](<6 x i32>)
+ ; LEGAL-NEXT: retValue_v6i32_r [[COPY]](<6 x i32>)
+ %0:reg64b(p2) = functionParameter_i64 0
+ %1:_(<6 x i32>) = G_LOAD %0(p2) :: (load (<6 x i32>) from %ir.arg, align 4, addrspace 2)
+ %2:regv6_32b(<6 x i32>) = COPY %1(<6 x i32>)
+ retValue_v6i32_r %2(<6 x i32>)
+...
+---
+name: load_constant_6xi32_8bytes_alignment
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_constant_6xi32_8bytes_alignment
+ ; LEGAL: %0:reg64b(p2) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<3 x i64>) = G_LOAD %0(p2) :: (load (<3 x i64>) from %ir.arg, align 8, addrspace 2)
+ ; LEGAL-NEXT: [[UV:%[0-9]+]]:_(i64), [[UV1:%[0-9]+]]:_(i64), [[UV2:%[0-9]+]]:_(i64) = G_UNMERGE_VALUES [[LOAD]](<3 x i64>)
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV]](i64)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV1]](i64)
+ ; LEGAL-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[UV2]](i64)
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(<6 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BITCAST]](<2 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<2 x i32>), 2
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR2:%[0-9]+]]:_(<6 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR1]], [[BITCAST2]](<2 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<6 x i32>) = COPY [[INSERT_SUBVECTOR2]](<6 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv6_32b(<6 x i32>) = COPY [[COPY]](<6 x i32>)
+ ; LEGAL-NEXT: retValue_v6i32_r [[COPY1]](<6 x i32>)
+ %0:reg64b(p2) = functionParameter_i64 0
+ %1:_(<6 x i32>) = G_LOAD %0(p2) :: (load (<6 x i32>) from %ir.arg, align 8, addrspace 2)
+ %2:regv6_32b(<6 x i32>) = COPY %1(<6 x i32>)
+ retValue_v6i32_r %2(<6 x i32>)
+...
+---
+name: load_constant_7xi32
+body: |
+ bb.1 (%ir-block.0):
+ ; LEGAL-LABEL: name: load_constant_7xi32
+ ; LEGAL: %0:reg64b(p2) = functionParameter_i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD %0(p2) :: (load (<8 x i32>) from %ir.arg, align 8, addrspace 2)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<7 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 0
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<7 x i32>) = COPY [[EXTRACT_SUBVECTOR]](<7 x i32>)
+ ; LEGAL-NEXT: [[COPY1:%[0-9]+]]:regv7_32b(<7 x i32>) = COPY [[COPY]](<7 x i32>)
+ ; LEGAL-NEXT: retValue_v7i32_r [[COPY1]](<7 x i32>)
+ %0:reg64b(p2) = functionParameter_i64 0
+ %1:_(<7 x i32>) = G_LOAD %0(p2) :: (load (<7 x i32>) from %ir.arg, align 8, addrspace 2)
+ %2:regv7_32b(<7 x i32>) = COPY %1(<7 x i32>)
+ retValue_v7i32_r %2(<7 x i32>)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir
new file mode 100644
index 0000000000000..5a29c576890e8
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir
@@ -0,0 +1,98 @@
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_load_128
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_load_128
+ ; CHECK: {{%[0-9]+}}:_(<4 x i32>) = G_LOAD {{%[0-9]+}}(p1) :: (load (<4 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<128 x i1>) = G_LOAD %0:_(p1) :: (load (<128 x i1>))
+ %2:_(i32) = G_CONSTANT i32 0
+ %3:_(i1) = G_EXTRACT_VECTOR_ELT %1:_(<128 x i1>), %2:_(i32)
+ G_STORE %3:_(i1), %0:_(p1) :: (store (i1))
+...
+---
+name: test_load_192
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_load_192
+ ; CHECK: {{%[0-9]+}}:_(<3 x i64>) = G_LOAD {{%[0-9]+}}(p1) :: (load (<3 x i64>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<192 x i1>) = G_LOAD %0:_(p1) :: (load (<192 x i1>))
+ %2:_(i32) = G_CONSTANT i32 0
+ %3:_(i1) = G_EXTRACT_VECTOR_ELT %1:_(<192 x i1>), %2:_(i32)
+ G_STORE %3:_(i1), %0:_(p1) :: (store (i1))
+...
+---
+name: test_load_256
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_load_256
+ ; CHECK: {{%[0-9]+}}:_(<8 x i32>) = G_LOAD {{%[0-9]+}}(p1) :: (load (<8 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<256 x i1>) = G_LOAD %0:_(p1) :: (load (<256 x i1>))
+ %2:_(i32) = G_CONSTANT i32 0
+ %3:_(i1) = G_EXTRACT_VECTOR_ELT %1:_(<256 x i1>), %2:_(i32)
+ G_STORE %3:_(i1), %0:_(p1) :: (store (i1))
+...
+---
+name: test_load_512
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_load_512
+ ; CHECK: {{%[0-9]+}}:_(<8 x i32>) = G_LOAD {{%[0-9]+}}(p1) :: (load (<8 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<512 x i1>) = G_LOAD %0:_(p1) :: (load (<512 x i1>))
+ %2:_(i32) = G_CONSTANT i32 0
+ %3:_(i1) = G_EXTRACT_VECTOR_ELT %1:_(<512 x i1>), %2:_(i32)
+ G_STORE %3:_(i1), %0:_(p1) :: (store (i1))
+...
+---
+name: test_store_192
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_store_192
+ ; CHECK: G_STORE {{%[0-9]+}}(<3 x i64>), {{%[0-9]+}}(p1) :: (store (<3 x i64>)
+ %0:_(p1) = G_CONSTANT i64 16
+ %1:_(i1) = G_CONSTANT i1 1
+ %2:_(<64 x i1>) = G_BUILD_VECTOR %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1)
+ %3:_(<192 x i1>) = G_CONCAT_VECTORS %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>)
+ G_STORE %3:_(<192 x i1>), %0:_(p1) :: (store (<192 x i1>))
+...
+---
+name: test_store_128
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_store_128
+ ; CHECK: G_STORE {{%[0-9]+}}(<4 x i32>), {{%[0-9]+}}(p1) :: (store (<4 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(i1) = G_CONSTANT i1 1
+ %2:_(<64 x i1>) = G_BUILD_VECTOR %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1)
+ %3:_(<128 x i1>) = G_CONCAT_VECTORS %2:_(<64 x i1>), %2:_(<64 x i1>)
+ G_STORE %3:_(<128 x i1>), %0:_(p1) :: (store (<128 x i1>))
+...
+---
+name: test_store_256
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_store_256
+ ; CHECK: G_STORE {{%[0-9]+}}(<8 x i32>), {{%[0-9]+}}(p1) :: (store (<8 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(i1) = G_CONSTANT i1 1
+ %2:_(<64 x i1>) = G_BUILD_VECTOR %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1)
+ %3:_(<256 x i1>) = G_CONCAT_VECTORS %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>)
+ G_STORE %3:_(<256 x i1>), %0:_(p1) :: (store (<256 x i1>))
+...
+---
+name: test_store_512
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_store_512
+ ; CHECK: G_STORE {{%[0-9]+}}(<8 x i32>), {{%[0-9]+}}(p1) :: (store (<8 x i32>)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(i1) = G_CONSTANT i1 1
+ %2:_(<64 x i1>) = G_BUILD_VECTOR %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1), %1:_(i1)
+ %3:_(<512 x i1>) = G_CONCAT_VECTORS %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>), %2:_(<64 x i1>)
+ G_STORE %3:_(<512 x i1>), %0:_(p1) :: (store (<512 x i1>))
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir
new file mode 100644
index 0000000000000..dea8bb21f3cc5
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
+
+# Check that loads/stores of vectors of non-2^n elements are resolved to legal
+# instructions. As of creating this test, they are split apart into multiple
+# scalar loads/stores. Those scalar loads/stores are then conveniently bitcasted
+# to vectors of i8 which are more manageable.
+
+---
+name: test_store
+body: |
+ bb.0:
+
+ ; LEGAL-LABEL: name: test_store
+ ; LEGAL: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF1:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF2:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF3:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF4:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF5:%[0-9]+]]:_(i8) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i8>) = G_BUILD_VECTOR [[DEF]](i8), [[DEF1]](i8)
+ ; LEGAL-NEXT: G_STORE [[BUILD_VECTOR]](<2 x i8>), [[C]](p1) :: (store (<2 x i8>) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ ; LEGAL-NEXT: [[C1:%[0-9]+]]:_(p1) = G_CONSTANT i64 2
+ ; LEGAL-NEXT: G_STORE [[DEF2]](i8), [[C1]](p1) :: (store (i8) into `ptr addrspace(1) null` + 2, align 2, basealign 4294967296, addrspace 1)
+ ; LEGAL-NEXT: [[C2:%[0-9]+]]:_(p1) = G_CONSTANT i64 3
+ ; LEGAL-NEXT: G_STORE [[DEF3]](i8), [[C2]](p1) :: (store (i8) into `ptr addrspace(1) null` + 3, basealign 4294967296, addrspace 1)
+ ; LEGAL-NEXT: [[C3:%[0-9]+]]:_(p1) = G_CONSTANT i64 4
+ ; LEGAL-NEXT: G_STORE [[DEF4]](i8), [[C3]](p1) :: (store (i8) into `ptr addrspace(1) null` + 4, align 4, basealign 4294967296, addrspace 1)
+ ; LEGAL-NEXT: [[C4:%[0-9]+]]:_(p1) = G_CONSTANT i64 5
+ ; LEGAL-NEXT: G_STORE [[DEF5]](i8), [[C4]](p1) :: (store (i8) into `ptr addrspace(1) null` + 5, basealign 4294967296, addrspace 1)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<2 x i24>) = G_IMPLICIT_DEF
+ G_STORE %1:_(<2 x i24>), %0:_(p1) :: (store (<2 x i24>) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+...
+---
+name: test_load
+body: |
+ bb.0:
+
+ ; LEGAL-LABEL: name: test_load
+ ; LEGAL: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<2 x i8>) = G_LOAD [[C]](p1) :: (load (<2 x i8>) from `ptr addrspace(1) null`, align 32, addrspace 1)
+ ; LEGAL-NEXT: [[C1:%[0-9]+]]:_(p1) = G_CONSTANT i64 2
+ ; LEGAL-NEXT: [[LOAD1:%[0-9]+]]:_(i8) = G_LOAD [[C1]](p1) :: (load (i8) from `ptr addrspace(1) null` + 2, align 2, basealign 32, addrspace 1)
+ ; LEGAL-NEXT: G_STORE [[LOAD]](<2 x i8>), [[C]](p1) :: (store (<2 x i8>) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ ; LEGAL-NEXT: [[C2:%[0-9]+]]:_(p1) = G_CONSTANT i64 2
+ ; LEGAL-NEXT: G_STORE [[LOAD1]](i8), [[C2]](p1) :: (store (i8) into `ptr addrspace(1) null` + 2, align 2, basealign 4294967296, addrspace 1)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<2 x i24>) = G_LOAD %0:_(p1) :: (load (<2 x i24>) from `ptr addrspace(1) null`, align 32, addrspace 1)
+ %2:_(i24), %3:_(i24) = G_UNMERGE_VALUES %1:_(<2 x i24>)
+ G_STORE %2:_(i24), %0:_(p1) :: (store (i24) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir
new file mode 100644
index 0000000000000..168f7907827c8
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir
@@ -0,0 +1,84 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
+
+# Check that loads/stores of vectors of 32n-bit width elements wider than 64 bits are resolved to vectors of 32-bit elements.
+# Check that if (alignment < element bit width) scalarization is performed instead.
+
+---
+name: test_store
+body: |
+ bb.0:
+
+ ; LEGAL-LABEL: name: test_store
+ ; LEGAL: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF1:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[DEF]](i128)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[DEF1]](i128)
+ ; LEGAL-NEXT: [[DEF2:%[0-9]+]]:_(<8 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[DEF2]], [[BITCAST]](<4 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<4 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<8 x i32>) = COPY [[INSERT_SUBVECTOR1]](<8 x i32>)
+ ; LEGAL-NEXT: G_STORE [[COPY]](<8 x i32>), [[C]](p1) :: (store (<8 x i32>) into `ptr addrspace(1) null`, addrspace 1)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<2 x i128>) = G_IMPLICIT_DEF
+ G_STORE %1:_(<2 x i128>), %0:_(p1) :: (store (<2 x i128>) into `ptr addrspace(1) null`, align 32, addrspace 1)
+...
+---
+name: test_load
+body: |
+ bb.0:
+
+ ; LEGAL-LABEL: name: test_load
+ ; LEGAL: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; LEGAL-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD [[C]](p1) :: (load (<8 x i32>) from `ptr addrspace(1) null`, addrspace 1)
+ ; LEGAL-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<4 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 0
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(i128) = G_BITCAST [[EXTRACT_SUBVECTOR]](<4 x i32>)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[BITCAST]](i128)
+ ; LEGAL-NEXT: G_STORE [[BITCAST1]](<4 x i32>), [[C]](p1) :: (store (<4 x i32>) into `ptr addrspace(1) null`, align 32, addrspace 1)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<2 x i128>) = G_LOAD %0:_(p1) :: (load (<2 x i128>) from `ptr addrspace(1) null`, align 32, addrspace 1)
+ %2:_(i128), %3:_(i128) = G_UNMERGE_VALUES %1:_(<2 x i128>)
+ G_STORE %2:_(i128), %0:_(p1) :: (store (i128) into `ptr addrspace(1) null`, align 32, addrspace 1)
+
+...
+---
+name: test_store_align4
+body: |
+ bb.0:
+
+ ; LEGAL-LABEL: name: test_store_align4
+ ; LEGAL: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF1:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[DEF]](i128)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[DEF1]](i128)
+ ; LEGAL-NEXT: [[DEF2:%[0-9]+]]:_(<8 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[DEF2]], [[BITCAST]](<4 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<4 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<8 x i32>) = COPY [[INSERT_SUBVECTOR1]](<8 x i32>)
+ ; LEGAL-NEXT: G_STORE [[COPY]](<8 x i32>), [[C]](p1) :: (store (<8 x i32>) into `ptr addrspace(1) null`, align 4, addrspace 1)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<2 x i128>) = G_IMPLICIT_DEF
+ G_STORE %1:_(<2 x i128>), %0:_(p1) :: (store (<2 x i128>) into `ptr addrspace(1) null`, align 4, addrspace 1)
+...
+---
+name: test_store_align8
+body: |
+ bb.0:
+
+ ; LEGAL-LABEL: name: test_store_align8
+ ; LEGAL: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; LEGAL-NEXT: [[DEF:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[DEF1:%[0-9]+]]:_(i128) = G_IMPLICIT_DEF
+ ; LEGAL-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[DEF]](i128)
+ ; LEGAL-NEXT: [[BITCAST1:%[0-9]+]]:_(<4 x i32>) = G_BITCAST [[DEF1]](i128)
+ ; LEGAL-NEXT: [[DEF2:%[0-9]+]]:_(<8 x i32>) = IMPLICIT_DEF
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[DEF2]], [[BITCAST]](<4 x i32>), 0
+ ; LEGAL-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BITCAST1]](<4 x i32>), 4
+ ; LEGAL-NEXT: [[COPY:%[0-9]+]]:_(<8 x i32>) = COPY [[INSERT_SUBVECTOR1]](<8 x i32>)
+ ; LEGAL-NEXT: G_STORE [[COPY]](<8 x i32>), [[C]](p1) :: (store (<8 x i32>) into `ptr addrspace(1) null`, align 8, addrspace 1)
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(<2 x i128>) = G_IMPLICIT_DEF
+ G_STORE %1:_(<2 x i128>), %0:_(p1) :: (store (<2 x i128>) into `ptr addrspace(1) null`, align 8, addrspace 1)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
new file mode 100644
index 0000000000000..e9b14060413a3
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
@@ -0,0 +1,139 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Test that legalizeGMulh correctly handles carry propagation in the
+# schoolbook 64-bit multiply-high expansion.
+#
+# The key invariant: when summing the cross-term lower halves and the
+# carry from a_lo*b_lo, a carry can propagate out of bit 31 into the
+# upper 64 bits. A buggy implementation that shifts each cross-term
+# independently (P_hh + (P_lh>>32) + (P_hl>>32)) drops this carry.
+
+--- |
+ define i64 @test_umulh_i64(i64 %a, i64 %b) {
+ ret i64 0
+ }
+ define i64 @test_smulh_i64(i64 %a, i64 %b) {
+ ret i64 0
+ }
+...
+
+# --- G_UMULH i64 ---
+# Verify that the legalized sequence includes carry-aware cross-term
+# accumulation: the adds feeding the final result must include the
+# lower-half sums (AHibLoSum0/AHibLoSum1), not just independent shifts
+# of each cross-term product.
+---
+name: test_umulh_i64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; Split into 32-bit halves
+ ; Four schoolbook products
+ ; Carry-aware cross-term accumulation (the fix):
+ ; Final upper-64 computation uses SUM1 (not raw HALB):
+ ; CHECK-LABEL: name: test_umulh_i64
+ ; CHECK: %0:reg64b(i64) = functionParameter_i64 0
+ ; CHECK-NEXT: %1:reg64b(i64) = functionParameter_i64 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4294967295
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i64) = COPY %0(i64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i64) = COPY %1(i64)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i64) = G_LSHR [[COPY]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i64) = G_LSHR [[COPY1]], [[C2]](i32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i64) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i64) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i64) = G_MUL [[LSHR]], [[LSHR1]]
+ ; CHECK-NEXT: [[MUL1:%[0-9]+]]:_(i64) = G_MUL [[LSHR]], [[AND1]]
+ ; CHECK-NEXT: [[MUL2:%[0-9]+]]:_(i64) = G_MUL [[AND]], [[LSHR1]]
+ ; CHECK-NEXT: [[MUL3:%[0-9]+]]:_(i64) = G_MUL [[AND]], [[AND1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i64) = G_LSHR [[MUL3]], [[C3]](i32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[MUL2]], [[C]]
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i64) = G_ADD [[LSHR2]], [[AND2]]
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i64) = G_ADD [[MUL1]], [[ADD]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i64) = G_LSHR [[MUL2]], [[C4]](i32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i64) = G_LSHR [[ADD1]], [[C5]](i32)
+ ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(i64) = G_ADD [[LSHR3]], [[LSHR4]]
+ ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(i64) = G_ADD [[MUL]], [[ADD2]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(i64) = COPY [[ADD3]](i64)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:reg64b(i64) = COPY [[COPY2]](i64)
+ ; CHECK-NEXT: retValue_i64_r [[COPY3]](i64)
+ %0:reg64b(i64) = functionParameter_i64 0
+ %1:reg64b(i64) = functionParameter_i64 1
+ %2:_(i64) = G_UMULH %0, %1
+ %3:reg64b(i64) = COPY %2(i64)
+ retValue_i64_r %3(i64)
+...
+
+# --- G_SMULH i64 ---
+# Verify the signed path: operands are converted to absolute values,
+# then the same carry-aware multiply-high is performed, followed by
+# conditional negation.
+---
+name: test_smulh_i64
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; Sign extraction and abs conversion
+ ; Same schoolbook multiply with carries on absolute values
+ ; Carry-aware accumulation must be present
+ ; Upper-64 uses SUM1 (carry-aware)
+ ; Conditional negation for signed result
+ ; CHECK-LABEL: name: test_smulh_i64
+ ; CHECK: %0:reg64b(i64) = functionParameter_i64 0
+ ; CHECK-NEXT: %1:reg64b(i64) = functionParameter_i64 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 4294967295
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR %0, [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(i64) = G_ASHR %1, [[C2]](i32)
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i64) = G_XOR [[ASHR]], [[ASHR1]]
+ ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(i64) = G_XOR %0, [[ASHR]]
+ ; CHECK-NEXT: [[XOR2:%[0-9]+]]:_(i64) = G_XOR %1, [[ASHR1]]
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i64) = G_SUB [[XOR1]], [[ASHR]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i64) = G_SUB [[XOR2]], [[ASHR1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i64) = G_LSHR [[SUB]], [[C3]](i32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i64) = G_LSHR [[SUB1]], [[C4]](i32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i64) = G_AND [[SUB]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i64) = G_AND [[SUB1]], [[C]]
+ ; CHECK-NEXT: [[MUL:%[0-9]+]]:_(i64) = G_MUL [[LSHR]], [[LSHR1]]
+ ; CHECK-NEXT: [[MUL1:%[0-9]+]]:_(i64) = G_MUL [[LSHR]], [[AND1]]
+ ; CHECK-NEXT: [[MUL2:%[0-9]+]]:_(i64) = G_MUL [[AND]], [[LSHR1]]
+ ; CHECK-NEXT: [[MUL3:%[0-9]+]]:_(i64) = G_MUL [[AND]], [[AND1]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i64) = G_LSHR [[MUL3]], [[C5]](i32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[MUL2]], [[C]]
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i64) = G_ADD [[LSHR2]], [[AND2]]
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i64) = G_ADD [[MUL1]], [[ADD]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i64) = G_AND [[MUL3]], [[C]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i64) = G_SHL [[ADD1]], [[C6]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i64) = G_OR [[SHL]], [[AND3]]
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i64) = G_LSHR [[MUL2]], [[C7]](i32)
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i64) = G_LSHR [[ADD1]], [[C8]](i32)
+ ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(i64) = G_ADD [[LSHR3]], [[LSHR4]]
+ ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(i64) = G_ADD [[MUL]], [[ADD2]]
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(i64) = G_SUB [[C9]], [[XOR]]
+ ; CHECK-NEXT: [[XOR3:%[0-9]+]]:_(i64) = G_XOR [[ADD3]], [[SUB2]]
+ ; CHECK-NEXT: [[XOR4:%[0-9]+]]:_(i64) = G_XOR [[OR]], [[SUB2]]
+ ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(i64) = G_ADD [[XOR4]], [[XOR]]
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[XOR]](i64)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i64) = G_SHL [[ADD4]], [[TRUNC]](i32)
+ ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(i64) = G_ADD [[XOR3]], [[SHL1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg64b(i64) = COPY [[ADD5]](i64)
+ ; CHECK-NEXT: retValue_i64_r [[COPY]](i64)
+ %0:reg64b(i64) = functionParameter_i64 0
+ %1:reg64b(i64) = functionParameter_i64 1
+ %2:_(i64) = G_SMULH %0, %1
+ %3:reg64b(i64) = COPY %2(i64)
+ retValue_i64_r %3(i64)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir
new file mode 100644
index 0000000000000..0aaafc45bec2a
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir
@@ -0,0 +1,88 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Verify G_UMULH lowering on non-power-of-2 type.
+
+--- |
+ ; ModuleID = 'reduced.ll'
+ source_filename = "reduced.ll"
+ target datalayout = "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-G1-A4"
+
+ define pisa_kernel void @test_umulh_s24(<3 x i8> %call19) local_unnamed_addr {
+ %1 = bitcast <3 x i8> %call19 to i24
+ %2 = udiv i24 %1, 42
+ %B13 = zext nneg i24 %2 to i32
+ store i32 %B13, ptr addrspace(1) null, align 4294967296
+ ret void
+ }
+ define pisa_kernel void @test_umulh_i64(i64 %a, i64 %b) {
+ ret void
+ }
+ define pisa_kernel void @test_smulh_i64(i64 %a, i64 %b) {
+ ret void
+ }
+...
+---
+name: test_umulh_s24
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1 (%ir-block.0):
+ %1:reg8b(i8) = loadParam_i8 0, 0
+ %2:reg8b(i8) = loadParam_i8 0, 1
+ %3:reg8b(i8) = loadParam_i8 0, 2
+ %0:_(<3 x i8>) = G_BUILD_VECTOR %1(i8), %2(i8), %3(i8)
+ %8:_(p1) = G_CONSTANT i64 0
+ %4:_(i24) = G_BITCAST %0(<3 x i8>)
+ %10:_(i24) = G_CONSTANT i24 -3994575
+ %12:_(i24) = G_CONSTANT i24 5
+ %14:_(i24) = G_UMULH %4, %10
+ %15:_(i24) = G_LSHR %14, %12(i24)
+ %7:_(i32) = nneg G_ZEXT %15(i24)
+
+ ; CHECK-LABEL: {{^}}name: test_umulh_s24
+ ; CHECK-NOT: G_UMULH
+
+ G_STORE %7(i32), %8(p1) :: (store (i32) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ ret
+...
+---
+# G_UMULH i64: verifies that the legalizer expands G_UMULH on i64 via custom
+# legalization, replacing it with an arithmetic multiplication sequence.
+name: test_umulh_i64
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %1:reg64b(i64) = loadParam_i64 1, 0
+ %ptr:_(p1) = G_CONSTANT i64 0
+ ; CHECK-LABEL: {{^}}name: test_umulh_i64
+ ; CHECK-NOT: G_UMULH
+ ; CHECK: G_MUL
+ %2:_(i64) = G_UMULH %0(i64), %1(i64)
+ G_STORE %2(i64), %ptr(p1) :: (store (i64) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+
+...
+---
+# G_SMULH i64: verifies that the legalizer expands G_SMULH on i64 via custom
+# legalization. The signed path also emits G_NEG and XOR adjustments not
+# present in the unsigned path.
+name: test_smulh_i64
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %1:reg64b(i64) = loadParam_i64 1, 0
+ %ptr:_(p1) = G_CONSTANT i64 0
+ ; CHECK-LABEL: {{^}}name: test_smulh_i64
+ ; CHECK-NOT: G_SMULH
+ ; CHECK: G_MUL
+ %2:_(i64) = G_SMULH %0(i64), %1(i64)
+ G_STORE %2(i64), %ptr(p1) :: (store (i64) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir
new file mode 100644
index 0000000000000..fac84023771fa
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir
@@ -0,0 +1,66 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# Check that i5 phi operand is extended to i16.
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
+ target triple = "pisa"
+
+ %"class.sycl::_V1::vec" = type { <4 x float> }
+
+ define pisa_kernel void @_ZTSZZ13MatchSiftDataR8SiftDataS0_RN4sycl3_V15queueERfENKUlRNS2_7handlerEE4_clES7_EUlNS2_7nd_itemILi3EEEE_(ptr addrspace(3) %0, i1 %1) {
+ br i1 %1, label %.preheader, label %._crit_edge29
+
+ .preheader: ; preds = %2
+ br label %3
+
+ ._crit_edge29: ; preds = %2
+ ret void
+
+ 3: ; preds = %.preheader, %3
+ %lsr.iv = phi i5 [ %lsr.iv.next2, %3 ], [ 0, %.preheader ]
+ %4 = zext i5 %lsr.iv to i32
+ %5 = getelementptr %"class.sycl::_V1::vec", ptr addrspace(3) %0, i32 %4
+ %lsr.iv.next2 = add i5 %lsr.iv, 1
+ br label %3
+ }
+
+...
+---
+name: _ZTSZZ13MatchSiftDataR8SiftDataS0_RN4sycl3_V15queueERfENKUlRNS2_7handlerEE4_clES7_EUlNS2_7nd_itemILi3EEEE_
+tracksRegLiveness: true
+body: |
+ bb.1 (%ir-block.2):
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+ %0:reg32b(p3) = loadParam_i32 0, 0
+ %2:reg8b(i8) = loadParam_i8 1, 0
+ %1:reg8b(i1) = G_TRUNC %2(i8)
+ %9:_(i5) = G_CONSTANT i5 1
+ %11:_(i5) = G_CONSTANT i5 0
+ %12:_(i1) = G_CONSTANT i1 true
+ %13:_(i1) = G_XOR %1, %12
+ G_BRCOND %13(i1), %bb.3
+ G_BR %bb.2
+
+ bb.2..preheader:
+ successors: %bb.4(0x80000000)
+
+ ; CHECK-LABEL: bb.{{.*}}..preheader
+ ; CHECK: [[ZERO:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+
+ G_BR %bb.4
+
+ bb.3.._crit_edge29:
+ ret
+
+ bb.4 (%ir-block.3):
+ successors: %bb.4(0x80000000)
+
+ ; CHECK: [[PHI:%[0-9]+]]:_(i16) = G_PHI [[INC:%[0-9]+]](i16), %bb.3, [[ZERO]](i16), %bb.1
+ ; CHECK: [[ONE:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK: [[INC]]:_(i16) = G_ADD [[PHI]], [[ONE]]
+
+ %3:_(i5) = G_PHI %10(i5), %bb.4, %11(i5), %bb.2
+ %10:_(i5) = G_ADD %3, %9
+ G_BR %bb.4
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir
new file mode 100644
index 0000000000000..682d8b5428f92
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir
@@ -0,0 +1,64 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# Check that i96 phi operand doesn't exit with UnableToLegalize.
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
+ target triple = "pisa"
+
+ %"class.sycl::_V1::vec" = type { <4 x float> }
+
+ define pisa_kernel void @_ZTSZZ13MatchSiftDataR8SiftDataS0_RN4sycl3_V15queueERfENKUlRNS2_7handlerEE4_clES7_EUlNS2_7nd_itemILi3EEEE_(ptr addrspace(3) %0, i1 %1) {
+ br i1 %1, label %.preheader, label %._crit_edge29
+
+ .preheader: ; preds = %2
+ br label %3
+
+ ._crit_edge29: ; preds = %2
+ ret void
+
+ 3: ; preds = %.preheader, %3
+ %lsr.iv = phi i96 [ %lsr.iv.next2, %3 ], [ 0, %.preheader ]
+ %4 = trunc i96 %lsr.iv to i32
+ %5 = getelementptr %"class.sycl::_V1::vec", ptr addrspace(3) %0, i32 %4
+ %lsr.iv.next2 = add i96 %lsr.iv, 1
+ br label %3
+ }
+
+...
+---
+name: _ZTSZZ13MatchSiftDataR8SiftDataS0_RN4sycl3_V15queueERfENKUlRNS2_7handlerEE4_clES7_EUlNS2_7nd_itemILi3EEEE_
+tracksRegLiveness: true
+body: |
+ bb.1 (%ir-block.2):
+ successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+ %0:reg32b(p3) = loadParam_i32 0, 0
+ %2:reg8b(i8) = loadParam_i8 1, 0
+ %1:reg8b(i1) = G_TRUNC %2(i8)
+ %9:_(i96) = G_CONSTANT i96 1
+ %11:_(i96) = G_CONSTANT i96 0
+ %12:_(i1) = G_CONSTANT i1 true
+ %13:_(i1) = G_XOR %1, %12
+ G_BRCOND %13(i1), %bb.3
+ G_BR %bb.2
+
+ bb.2..preheader:
+ successors: %bb.4(0x80000000)
+
+
+ G_BR %bb.4
+
+ bb.3.._crit_edge29:
+ ret
+
+ bb.4 (%ir-block.3):
+ successors: %bb.4(0x80000000)
+ ; i96 phi inst ends up split into two i64 insts.
+ ; CHECK: %{{[0-9]+}}:_(i64) = G_PHI [[UNMERG_0:%[0-9]+]](i64), %bb.3, %{{[0-9]+}}(i64), %bb.1
+ ; CHECK: %{{[0-9]+}}:_(i64) = G_PHI [[UNMERG_1:%[0-9]+]](i64), %bb.3, %{{[0-9]+}}(i64), %bb.1
+ ; CHECK: [[UNMERG_0]]:_(i64), [[UNMERG_1]]:_(i64) = G_UNMERGE_VALUES %{{[0-9]+}}(i128)
+
+ %3:_(i96) = G_PHI %10(i96), %bb.4, %11(i96), %bb.2
+ %10:_(i96) = G_ADD %3, %9
+ G_BR %bb.4
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir
new file mode 100644
index 0000000000000..9c8dcbd2e9b2d
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir
@@ -0,0 +1,33 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
+ target triple = "pisa"
+
+ define pisa_kernel void @test(<4 x bfloat> %arg) {
+ entry:
+ br label %for.body
+
+ for.body: ; preds = %for.body, %entry
+ %y.05 = phi <4 x bfloat> [ %arg, %entry ], [ %y.05, %for.body ]
+ br label %for.body
+ }
+
+...
+---
+name: test
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK: [[VEC:%[0-9]+]]:_(<4 x i16>) = COPY
+ ; CHECK: {{%[0-9]+}}:_(i16), {{%[0-9]+}}:_(i16), {{%[0-9]+}}:_(i16), {{%[0-9]+}}:_(i16) = G_UNMERGE_VALUES [[VEC]]
+
+ %0:regv4_16b = functionParameter_v4i16 0
+ %56:_(<4 x i16>) = COPY %0:regv4_16b
+
+ bb.2.for.body:
+ ; CHECK-COUNT-4: {{%[0-9]+}}:_(i16) = G_PHI
+ %26:_(<4 x i16>) = G_PHI %26:_(<4 x i16>), %bb.2, %56:_(<4 x i16>), %bb.1
+ G_BR %bb.2
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir
new file mode 100644
index 0000000000000..11a6fa78ed52b
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir
@@ -0,0 +1,33 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# Check that i24 operands in phi inst are extended to i32.
+
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
+ target triple = "pisa"
+
+ define pisa_kernel void @_ZTSN4sycl3_V16detail9reduction7MainKrnIN39reduction_without_identity_param_common6kernelIiLb1ELb0EN16reduction_common19op_without_identityIiEENS0_5rangeILi1EEELNS6_14test_case_typeE3ELi2EEELNS2_8strategyE3EJEEE() {
+ entry:
+ br label %for.body
+
+ for.body: ; preds = %for.body, %entry
+ %LocalSum.sroa = phi i24 [ %LocalSum.sroa, %for.body ], [ 1, %entry ]
+ br label %for.body
+ }
+
+...
+---
+name: _ZTSN4sycl3_V16detail9reduction7MainKrnIN39reduction_without_identity_param_common6kernelIiLb1ELb0EN16reduction_common19op_without_identityIiEENS0_5rangeILi1EEELNS6_14test_case_typeE3ELi2EEELNS2_8strategyE3EJEEE
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK: [[ONE:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+
+ %1:_(i24) = G_CONSTANT i24 1
+
+ bb.2.for.body:
+ ; CHECK: [[V:%[0-9]+]]:_(i32) = G_PHI [[V]](i32), %bb.1, [[ONE]](i32), %bb.0
+
+ %0:_(i24) = G_PHI %0(i24), %bb.2, %1(i24), %bb.1
+ G_BR %bb.2
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir
new file mode 100644
index 0000000000000..66b742daeeb5d
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir
@@ -0,0 +1,38 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# Check that i40 operands in select inst are extended to i64.
+
+--- |
+ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
+ target triple = "pisa"
+
+ define pisa_kernel void @_ZTS22broadcast_group_kernelILi1EN4util11custom_typeEE(i1 %ov) {
+ entry:
+ %local_var.sroa.insert.shift = select i1 %ov, i40 0, i40 4294967296
+ %agg.tmp1.sroa.insert.insert = zext i40 %local_var.sroa.insert.shift to i64
+ %call = call i64 @_Z22__spirv_GroupBroadcastjmm(i64 %agg.tmp1.sroa.insert.insert)
+ ret void
+ }
+
+ declare i64 @_Z22__spirv_GroupBroadcastjmm(i64)
+
+...
+---
+name: _ZTS22broadcast_group_kernelILi1EN4util11custom_typeEE
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ %1:reg8b(i8) = loadParam_i8 0, 0
+ %0:reg8b(i1) = G_TRUNC %1(i8)
+
+ ; CHECK: [[Op0:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK: [[Op1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4294967296
+ ; CHECK: :_(i64) = G_SELECT %{{[0-9]+}}(i1), [[Op0]], [[Op1]]
+
+ %3:_(i40) = G_CONSTANT i40 0
+ %4:_(i40) = G_CONSTANT i40 4294967296
+ %2:_(i40) = G_SELECT %0(i1), %3, %4
+ %5:_(i64) = G_ZEXT %2(i40)
+ %6:reg64b(i64) = functionCall_i64_r @_Z22__spirv_GroupBroadcastjmm, %5(i64)
+ ret
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
new file mode 100644
index 0000000000000..ade58cf506a0a
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
@@ -0,0 +1,24 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
+
+# Check that alignment too large to store in 32-bit value isn't truncated.
+--- |
+ define void @test_concat_clamp_elts(ptr %addr) {
+ %vec.expand = shufflevector <6 x i16> undef, <6 x i16> undef, <12 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ store <12 x i16> %vec.expand, ptr addrspace(1) null, align 4294967296
+ ret void
+ }
+...
+---
+name: test_concat_clamp_elts
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; LEGAL-NOT: G_STORE %{{[0-9]+}}(i16)
+ ; LEGAL: G_STORE %{{[0-9]+}}(<4 x i32>)
+ %0:_(<6 x i16>) = G_IMPLICIT_DEF
+ %1:_(<6 x i16>) = G_IMPLICIT_DEF
+ %2:_(<12 x i16>) = G_CONCAT_VECTORS %0:_(<6 x i16>), %1:_(<6 x i16>)
+ %3:_(p1) = G_CONSTANT i64 0
+ G_STORE %2:_(<12 x i16>), %3:_(p1) :: (store (<12 x i16>) into `ptr addrspace(1) null`, align 4294967296, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir
new file mode 100644
index 0000000000000..79df8faa0f77e
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir
@@ -0,0 +1,515 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Checks that sequence produced by Expand IR instructions for udiv128 is legalized:
+--- |
+ define pisa_kernel void @test_udiv128(i64 %0, i64 %1) local_unnamed_addr {
+ .preheader3_udiv-special-cases:
+ %2 = tail call i32 @llvm.pisa.local.size.x()
+ %3 = zext i64 %0 to i128
+ %4 = add nuw nsw i128 %3, 1
+ %5 = zext i64 %1 to i128
+ %umax = tail call i128 @llvm.umax.i128(i128 %4, i128 %5)
+ %6 = icmp uge i64 %0, %1
+ %umin = zext i1 %6 to i128
+ %7 = add nuw nsw i128 %umin, %5
+ %8 = sub nsw i128 %umax, %7
+ %9 = zext nneg i32 %2 to i128
+ %10 = freeze i128 %9
+ %11 = freeze i128 %8
+ %12 = icmp eq i128 %10, 0
+ %13 = icmp eq i128 %11, 0
+ %14 = or i1 %12, %13
+ %15 = call i128 @llvm.ctlz.i128(i128 %10, i1 true)
+ %16 = call i128 @llvm.ctlz.i128(i128 %11, i1 true)
+ %17 = sub i128 %15, %16
+ %18 = icmp ugt i128 %17, 127
+ %19 = select i1 %14, i1 true, i1 %18
+ %20 = icmp eq i128 %17, 127
+ %21 = select i1 %19, i128 0, i128 %11
+ %22 = select i1 %19, i1 true, i1 %20
+ br i1 %22, label %udiv-end, label %udiv-bb1
+
+ udiv-bb1: ; preds = %.preheader3_udiv-special-cases
+ %23 = call { i128, i1 } @llvm.uadd.with.overflow.i128(i128 %17, i128 1)
+ %math = extractvalue { i128, i1 } %23, 0
+ %ov = extractvalue { i128, i1 } %23, 1
+ %24 = sub i128 127, %17
+ %25 = shl i128 %11, %24
+ br i1 %ov, label %udiv-loop-exit, label %udiv-preheader
+
+ udiv-preheader: ; preds = %udiv-bb1
+ %26 = lshr i128 %11, %math
+ %27 = add i128 %10, -1
+ br label %udiv-do-while
+
+ udiv-do-while: ; preds = %udiv-do-while, %udiv-preheader
+ %28 = phi i128 [ 0, %udiv-preheader ], [ %39, %udiv-do-while ]
+ %29 = phi i128 [ %math, %udiv-preheader ], [ %42, %udiv-do-while ]
+ %30 = phi i128 [ %26, %udiv-preheader ], [ %41, %udiv-do-while ]
+ %31 = phi i128 [ %25, %udiv-preheader ], [ %36, %udiv-do-while ]
+ %32 = shl i128 %30, 1
+ %33 = lshr i128 %31, 127
+ %34 = or i128 %32, %33
+ %35 = shl i128 %31, 1
+ %36 = or i128 %28, %35
+ %37 = sub i128 %27, %34
+ %38 = ashr i128 %37, 127
+ %39 = and i128 %38, 1
+ %40 = and i128 %38, %10
+ %41 = sub i128 %34, %40
+ %42 = add i128 %29, -1
+ %43 = icmp eq i128 %42, 0
+ br i1 %43, label %udiv-loop-exit, label %udiv-do-while
+
+ udiv-loop-exit: ; preds = %udiv-do-while, %udiv-bb1
+ %44 = phi i128 [ 0, %udiv-bb1 ], [ %39, %udiv-do-while ]
+ %45 = phi i128 [ %25, %udiv-bb1 ], [ %36, %udiv-do-while ]
+ %46 = shl i128 %45, 1
+ %47 = or i128 %44, %46
+ br label %udiv-end
+
+ udiv-end: ; preds = %udiv-loop-exit, %.preheader3_udiv-special-cases
+ %48 = phi i128 [ %47, %udiv-loop-exit ], [ %21, %.preheader3_udiv-special-cases ]
+ %49 = zext i1 %6 to i128
+ %50 = add i128 %48, %49
+ %51 = trunc i128 %50 to i64
+ store i64 %51, ptr addrspace(3) null, align 2147483648
+ ret void
+ }
+
+ ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+ declare range(i32 1, 65537) i32 @llvm.pisa.local.size.x() #0
+
+ ; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+ declare i128 @llvm.umax.i128(i128, i128) #1
+
+ ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+ declare i128 @llvm.ctlz.i128(i128, i1 immarg) #2
+
+ ; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+ declare { i128, i1 } @llvm.uadd.with.overflow.i128(i128, i128) #1
+
+ attributes #0 = { nocallback nofree nosync nounwind willreturn memory(none) }
+ attributes #1 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+ attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+...
+---
+name: test_udiv128
+alignment: 1
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_udiv128
+ ; CHECK: bb.0..preheader3_udiv-special-cases:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %0:reg64b(i64) = loadParam_i64 0, 0
+ ; CHECK-NEXT: %1:reg64b(i64) = loadParam_i64 1, 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 127
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 -1
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 -1
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(p3) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[INT:%[0-9]+]]:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.size.x)
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %0(i64)
+ ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C]](i64)
+ ; CHECK-NEXT: [[UADDO:%[0-9]+]]:_(i32), [[UADDO1:%[0-9]+]]:_(i1) = G_UADDO [[UV]], [[UV2]]
+ ; CHECK-NEXT: [[UADDE:%[0-9]+]]:_(i32), [[UADDE1:%[0-9]+]]:_(i1) = G_UADDE [[UV1]], [[UV3]], [[UADDO1]]
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDO]](i32), [[UADDE]](i32)
+ ; CHECK-NEXT: [[UV4:%[0-9]+]]:_(i32), [[UV5:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C9]](i64)
+ ; CHECK-NEXT: [[UV6:%[0-9]+]]:_(i32), [[UV7:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C1]](i64)
+ ; CHECK-NEXT: [[UADDE2:%[0-9]+]]:_(i32), [[UADDE3:%[0-9]+]]:_(i1) = G_UADDE [[UV4]], [[UV6]], [[UADDE1]]
+ ; CHECK-NEXT: [[UADDE4:%[0-9]+]]:_(i32), [[UADDE5:%[0-9]+]]:_(i1) = G_UADDE [[UV5]], [[UV7]], [[UADDE3]]
+ ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDE2]](i32), [[UADDE4]](i32)
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[MV]](i64), %1
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[MV1]](i64), [[C10]]
+ ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[MV1]](i64), [[C10]]
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C12:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(i16) = G_SELECT [[ICMP]](i1), [[C12]], [[C11]]
+ ; CHECK-NEXT: [[C13:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C14:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(i16) = G_SELECT [[ICMP1]](i1), [[C14]], [[C13]]
+ ; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(i16) = G_SELECT [[ICMP2]](i1), [[SELECT]], [[SELECT1]]
+ ; CHECK-NEXT: [[C15:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C16:%[0-9]+]]:_(i16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i16) = G_AND [[SELECT2]], [[C16]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND]](i16), [[C15]]
+ ; CHECK-NEXT: [[SELECT3:%[0-9]+]]:_(i64) = G_SELECT [[ICMP3]](i1), [[MV]], %1
+ ; CHECK-NEXT: [[SELECT4:%[0-9]+]]:_(i64) = G_SELECT [[ICMP3]](i1), [[MV1]], [[C10]]
+ ; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(i1) = G_ICMP intpred(uge), %0(i64), %1
+ ; CHECK-NEXT: [[C17:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C18:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C19:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; CHECK-NEXT: [[C20:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[SELECT5:%[0-9]+]]:_(i64) = G_SELECT [[ICMP4]](i1), [[C19]], [[C17]]
+ ; CHECK-NEXT: [[SELECT6:%[0-9]+]]:_(i64) = G_SELECT [[ICMP4]](i1), [[C20]], [[C18]]
+ ; CHECK-NEXT: [[UV8:%[0-9]+]]:_(i32), [[UV9:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SELECT5]](i64)
+ ; CHECK-NEXT: [[UV10:%[0-9]+]]:_(i32), [[UV11:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES %1(i64)
+ ; CHECK-NEXT: [[UADDO2:%[0-9]+]]:_(i32), [[UADDO3:%[0-9]+]]:_(i1) = G_UADDO [[UV8]], [[UV10]]
+ ; CHECK-NEXT: [[UADDE6:%[0-9]+]]:_(i32), [[UADDE7:%[0-9]+]]:_(i1) = G_UADDE [[UV9]], [[UV11]], [[UADDO3]]
+ ; CHECK-NEXT: [[UV12:%[0-9]+]]:_(i32), [[UV13:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SELECT6]](i64)
+ ; CHECK-NEXT: [[UV14:%[0-9]+]]:_(i32), [[UV15:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C10]](i64)
+ ; CHECK-NEXT: [[UADDE8:%[0-9]+]]:_(i32), [[UADDE9:%[0-9]+]]:_(i1) = G_UADDE [[UV12]], [[UV14]], [[UADDE7]]
+ ; CHECK-NEXT: [[UADDE10:%[0-9]+]]:_(i32), [[UADDE11:%[0-9]+]]:_(i1) = G_UADDE [[UV13]], [[UV15]], [[UADDE9]]
+ ; CHECK-NEXT: [[UV16:%[0-9]+]]:_(i32), [[UV17:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SELECT3]](i64)
+ ; CHECK-NEXT: [[USUBO:%[0-9]+]]:_(i32), [[USUBO1:%[0-9]+]]:_(i1) = G_USUBO [[UV16]], [[UADDO2]]
+ ; CHECK-NEXT: [[USUBE:%[0-9]+]]:_(i32), [[USUBE1:%[0-9]+]]:_(i1) = G_USUBE [[UV17]], [[UADDE6]], [[USUBO1]]
+ ; CHECK-NEXT: [[MV2:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBO]](i32), [[USUBE]](i32)
+ ; CHECK-NEXT: [[UV18:%[0-9]+]]:_(i32), [[UV19:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SELECT4]](i64)
+ ; CHECK-NEXT: [[USUBE2:%[0-9]+]]:_(i32), [[USUBE3:%[0-9]+]]:_(i1) = G_USUBE [[UV18]], [[UADDE8]], [[USUBE1]]
+ ; CHECK-NEXT: [[USUBE4:%[0-9]+]]:_(i32), [[USUBE5:%[0-9]+]]:_(i1) = G_USUBE [[UV19]], [[UADDE10]], [[USUBE3]]
+ ; CHECK-NEXT: [[MV3:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBE2]](i32), [[USUBE4]](i32)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(i32) = G_FREEZE [[INT]]
+ ; CHECK-NEXT: [[C21:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[MV4:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[FREEZE]](i32), [[C21]](i32)
+ ; CHECK-NEXT: [[C22:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[FREEZE1:%[0-9]+]]:_(i64) = G_FREEZE [[MV2]]
+ ; CHECK-NEXT: [[FREEZE2:%[0-9]+]]:_(i64) = G_FREEZE [[MV3]]
+ ; CHECK-NEXT: [[C23:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i64) = G_XOR [[MV4]], [[C2]]
+ ; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(i64) = G_XOR [[C22]], [[C3]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i64) = G_OR [[XOR]], [[XOR1]]
+ ; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[OR]](i64), [[C23]]
+ ; CHECK-NEXT: [[C24:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[XOR2:%[0-9]+]]:_(i64) = G_XOR [[FREEZE1]], [[C2]]
+ ; CHECK-NEXT: [[XOR3:%[0-9]+]]:_(i64) = G_XOR [[FREEZE2]], [[C3]]
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i64) = G_OR [[XOR2]], [[XOR3]]
+ ; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[OR1]](i64), [[C24]]
+ ; CHECK-NEXT: [[C25:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C26:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[SELECT7:%[0-9]+]]:_(i32) = G_SELECT [[ICMP5]](i1), [[C26]], [[C25]]
+ ; CHECK-NEXT: [[C27:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C28:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[SELECT8:%[0-9]+]]:_(i32) = G_SELECT [[ICMP6]](i1), [[C28]], [[C27]]
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[SELECT7]], [[SELECT8]]
+ ; CHECK-NEXT: [[C29:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[C22]](i64), [[C29]]
+ ; CHECK-NEXT: [[C30:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP8:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[C21]](i32), [[C30]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[FREEZE]](i32)
+ ; CHECK-NEXT: [[C31:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[CTLZ_ZERO_UNDEF]], [[C31]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF1:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[C21]](i32)
+ ; CHECK-NEXT: [[SELECT9:%[0-9]+]]:_(i32) = G_SELECT [[ICMP8]](i1), [[ADD]], [[CTLZ_ZERO_UNDEF1]]
+ ; CHECK-NEXT: [[C32:%[0-9]+]]:_(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[SELECT9]], [[C32]]
+ ; CHECK-NEXT: [[UV20:%[0-9]+]]:_(i32), [[UV21:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C22]](i64)
+ ; CHECK-NEXT: [[C33:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP9:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[UV21]](i32), [[C33]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF2:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[UV20]](i32)
+ ; CHECK-NEXT: [[C34:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(i32) = G_ADD [[CTLZ_ZERO_UNDEF2]], [[C34]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF3:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[UV21]](i32)
+ ; CHECK-NEXT: [[SELECT10:%[0-9]+]]:_(i32) = G_SELECT [[ICMP9]](i1), [[ADD2]], [[CTLZ_ZERO_UNDEF3]]
+ ; CHECK-NEXT: [[SELECT11:%[0-9]+]]:_(i32) = G_SELECT [[ICMP7]](i1), [[ADD1]], [[SELECT10]]
+ ; CHECK-NEXT: [[C35:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C36:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C37:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[ICMP10:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[FREEZE2]](i64), [[C37]]
+ ; CHECK-NEXT: [[UV22:%[0-9]+]]:_(i32), [[UV23:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[FREEZE1]](i64)
+ ; CHECK-NEXT: [[C38:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP11:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[UV23]](i32), [[C38]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF4:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[UV22]](i32)
+ ; CHECK-NEXT: [[C39:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(i32) = G_ADD [[CTLZ_ZERO_UNDEF4]], [[C39]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF5:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[UV23]](i32)
+ ; CHECK-NEXT: [[SELECT12:%[0-9]+]]:_(i32) = G_SELECT [[ICMP11]](i1), [[ADD3]], [[CTLZ_ZERO_UNDEF5]]
+ ; CHECK-NEXT: [[C40:%[0-9]+]]:_(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[ADD4:%[0-9]+]]:_(i32) = G_ADD [[SELECT12]], [[C40]]
+ ; CHECK-NEXT: [[UV24:%[0-9]+]]:_(i32), [[UV25:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[FREEZE2]](i64)
+ ; CHECK-NEXT: [[C41:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP12:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[UV25]](i32), [[C41]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF6:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[UV24]](i32)
+ ; CHECK-NEXT: [[C42:%[0-9]+]]:_(i32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[CTLZ_ZERO_UNDEF6]], [[C42]]
+ ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF7:%[0-9]+]]:_(i32) = G_CTLZ_ZERO_POISON [[UV25]](i32)
+ ; CHECK-NEXT: [[SELECT13:%[0-9]+]]:_(i32) = G_SELECT [[ICMP12]](i1), [[ADD5]], [[CTLZ_ZERO_UNDEF7]]
+ ; CHECK-NEXT: [[SELECT14:%[0-9]+]]:_(i32) = G_SELECT [[ICMP10]](i1), [[ADD4]], [[SELECT13]]
+ ; CHECK-NEXT: [[C43:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C44:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[USUBO2:%[0-9]+]]:_(i32), [[USUBO3:%[0-9]+]]:_(i1) = G_USUBO [[SELECT11]], [[SELECT14]]
+ ; CHECK-NEXT: [[USUBE6:%[0-9]+]]:_(i32), [[USUBE7:%[0-9]+]]:_(i1) = G_USUBE [[C35]], [[C43]], [[USUBO3]]
+ ; CHECK-NEXT: [[MV5:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBO2]](i32), [[USUBE6]](i32)
+ ; CHECK-NEXT: [[UV26:%[0-9]+]]:_(i32), [[UV27:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C36]](i64)
+ ; CHECK-NEXT: [[UV28:%[0-9]+]]:_(i32), [[UV29:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C44]](i64)
+ ; CHECK-NEXT: [[USUBE8:%[0-9]+]]:_(i32), [[USUBE9:%[0-9]+]]:_(i1) = G_USUBE [[UV26]], [[UV28]], [[USUBE7]]
+ ; CHECK-NEXT: [[USUBE10:%[0-9]+]]:_(i32), [[USUBE11:%[0-9]+]]:_(i1) = G_USUBE [[UV27]], [[UV29]], [[USUBE9]]
+ ; CHECK-NEXT: [[MV6:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBE8]](i32), [[USUBE10]](i32)
+ ; CHECK-NEXT: [[ICMP13:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[MV5]](i64), [[C4]]
+ ; CHECK-NEXT: [[ICMP14:%[0-9]+]]:_(i1) = G_ICMP intpred(ugt), [[MV6]](i64), [[C5]]
+ ; CHECK-NEXT: [[ICMP15:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[MV6]](i64), [[C5]]
+ ; CHECK-NEXT: [[C45:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C46:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT15:%[0-9]+]]:_(i16) = G_SELECT [[ICMP13]](i1), [[C46]], [[C45]]
+ ; CHECK-NEXT: [[C47:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[C48:%[0-9]+]]:_(i16) = G_CONSTANT i16 -1
+ ; CHECK-NEXT: [[SELECT16:%[0-9]+]]:_(i16) = G_SELECT [[ICMP14]](i1), [[C48]], [[C47]]
+ ; CHECK-NEXT: [[SELECT17:%[0-9]+]]:_(i16) = G_SELECT [[ICMP15]](i1), [[SELECT15]], [[SELECT16]]
+ ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[SELECT17]](i16)
+ ; CHECK-NEXT: [[FREEZE3:%[0-9]+]]:_(i32) = G_FREEZE [[ANYEXT]]
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[FREEZE3]]
+ ; CHECK-NEXT: [[C49:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C50:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C50]]
+ ; CHECK-NEXT: [[ICMP16:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND1]](i32), [[C49]]
+ ; CHECK-NEXT: [[C51:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[XOR4:%[0-9]+]]:_(i64) = G_XOR [[MV5]], [[C4]]
+ ; CHECK-NEXT: [[XOR5:%[0-9]+]]:_(i64) = G_XOR [[MV6]], [[C5]]
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i64) = G_OR [[XOR4]], [[XOR5]]
+ ; CHECK-NEXT: [[ICMP17:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[OR4]](i64), [[C51]]
+ ; CHECK-NEXT: [[SELECT18:%[0-9]+]]:_(i64) = G_SELECT [[ICMP16]](i1), [[C2]], [[FREEZE1]]
+ ; CHECK-NEXT: [[C52:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C53:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[SELECT19:%[0-9]+]]:_(i32) = G_SELECT [[ICMP17]](i1), [[C53]], [[C52]]
+ ; CHECK-NEXT: [[FREEZE4:%[0-9]+]]:_(i32) = G_FREEZE [[SELECT19]]
+ ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[FREEZE4]]
+ ; CHECK-NEXT: [[C54:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C55:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR5]], [[C55]]
+ ; CHECK-NEXT: [[ICMP18:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND2]](i32), [[C54]]
+ ; CHECK-NEXT: G_BRCOND [[ICMP18]](i1), %bb.5
+ ; CHECK-NEXT: G_BR %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1.udiv-bb1:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UV30:%[0-9]+]]:_(i32), [[UV31:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C]](i64)
+ ; CHECK-NEXT: [[UV32:%[0-9]+]]:_(i32), [[UV33:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C1]](i64)
+ ; CHECK-NEXT: [[UADDO4:%[0-9]+]]:_(i32), [[UADDO5:%[0-9]+]]:_(i1) = G_UADDO [[USUBO2]], [[UV30]]
+ ; CHECK-NEXT: [[UADDE12:%[0-9]+]]:_(i32), [[UADDE13:%[0-9]+]]:_(i1) = G_UADDE [[USUBE6]], [[UV31]], [[UADDO5]]
+ ; CHECK-NEXT: [[UADDE14:%[0-9]+]]:_(i32), [[UADDE15:%[0-9]+]]:_(i1) = G_UADDE [[USUBE8]], [[UV32]], [[UADDE13]]
+ ; CHECK-NEXT: [[UADDE16:%[0-9]+]]:_(i32), [[UADDE17:%[0-9]+]]:_(i1) = G_UADDE [[USUBE10]], [[UV33]], [[UADDE15]]
+ ; CHECK-NEXT: [[UV34:%[0-9]+]]:_(i32), [[UV35:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C4]](i64)
+ ; CHECK-NEXT: [[USUBO4:%[0-9]+]]:_(i32), [[USUBO5:%[0-9]+]]:_(i1) = G_USUBO [[UV34]], [[USUBO2]]
+ ; CHECK-NEXT: [[C56:%[0-9]+]]:_(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(i32) = G_SUB [[USUBO4]], [[C56]]
+ ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[C56]], [[USUBO4]]
+ ; CHECK-NEXT: [[C57:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP19:%[0-9]+]]:_(i1) = G_ICMP intpred(ult), [[USUBO4]](i32), [[C56]]
+ ; CHECK-NEXT: [[ICMP20:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[USUBO4]](i32), [[C57]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i64) = G_SHL [[FREEZE1]], [[USUBO4]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i64) = G_LSHR [[FREEZE1]], [[SUB1]](i32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i64) = G_SHL [[FREEZE2]], [[USUBO4]](i32)
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i64) = G_OR [[LSHR]], [[SHL1]]
+ ; CHECK-NEXT: [[C58:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i64) = G_SHL [[FREEZE1]], [[SUB]](i32)
+ ; CHECK-NEXT: [[SELECT20:%[0-9]+]]:_(i64) = G_SELECT [[ICMP19]](i1), [[SHL]], [[C58]]
+ ; CHECK-NEXT: [[SELECT21:%[0-9]+]]:_(i64) = G_SELECT [[ICMP19]](i1), [[OR6]], [[SHL2]]
+ ; CHECK-NEXT: [[SELECT22:%[0-9]+]]:_(i64) = G_SELECT [[ICMP20]](i1), [[FREEZE2]], [[SELECT21]]
+ ; CHECK-NEXT: G_BRCOND [[UADDE17]](i1), %bb.4
+ ; CHECK-NEXT: G_BR %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2.udiv-preheader:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C59:%[0-9]+]]:_(i32) = G_CONSTANT i32 64
+ ; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(i32) = G_SUB [[UADDO4]], [[C59]]
+ ; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[C59]], [[UADDO4]]
+ ; CHECK-NEXT: [[C60:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP21:%[0-9]+]]:_(i1) = G_ICMP intpred(ult), [[UADDO4]](i32), [[C59]]
+ ; CHECK-NEXT: [[ICMP22:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[UADDO4]](i32), [[C60]]
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i64) = G_LSHR [[FREEZE2]], [[UADDO4]](i32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i64) = G_LSHR [[FREEZE1]], [[UADDO4]](i32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i64) = G_SHL [[FREEZE2]], [[SUB3]](i32)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i64) = G_OR [[LSHR2]], [[SHL3]]
+ ; CHECK-NEXT: [[C61:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i64) = G_LSHR [[FREEZE2]], [[SUB2]](i32)
+ ; CHECK-NEXT: [[SELECT23:%[0-9]+]]:_(i64) = G_SELECT [[ICMP21]](i1), [[OR7]], [[LSHR3]]
+ ; CHECK-NEXT: [[SELECT24:%[0-9]+]]:_(i64) = G_SELECT [[ICMP22]](i1), [[FREEZE1]], [[SELECT23]]
+ ; CHECK-NEXT: [[SELECT25:%[0-9]+]]:_(i64) = G_SELECT [[ICMP21]](i1), [[LSHR1]], [[C61]]
+ ; CHECK-NEXT: [[UV36:%[0-9]+]]:_(i32), [[UV37:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C6]](i64)
+ ; CHECK-NEXT: [[UADDO6:%[0-9]+]]:_(i32), [[UADDO7:%[0-9]+]]:_(i1) = G_UADDO [[FREEZE]], [[UV36]]
+ ; CHECK-NEXT: [[UADDE18:%[0-9]+]]:_(i32), [[UADDE19:%[0-9]+]]:_(i1) = G_UADDE [[C21]], [[UV37]], [[UADDO7]]
+ ; CHECK-NEXT: [[UV38:%[0-9]+]]:_(i32), [[UV39:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C22]](i64)
+ ; CHECK-NEXT: [[UV40:%[0-9]+]]:_(i32), [[UV41:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C7]](i64)
+ ; CHECK-NEXT: [[UADDE20:%[0-9]+]]:_(i32), [[UADDE21:%[0-9]+]]:_(i1) = G_UADDE [[UV38]], [[UV40]], [[UADDE19]]
+ ; CHECK-NEXT: [[UADDE22:%[0-9]+]]:_(i32), [[UADDE23:%[0-9]+]]:_(i1) = G_UADDE [[UV39]], [[UV41]], [[UADDE21]]
+ ; CHECK-NEXT: [[MV7:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDO4]](i32), [[UADDE12]](i32)
+ ; CHECK-NEXT: [[MV8:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDE14]](i32), [[UADDE16]](i32)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3.udiv-do-while:
+ ; CHECK-NEXT: successors: %bb.4(0x04000000), %bb.3(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:_(i64) = G_PHI [[C2]](i64), %bb.2, %180(i64), %bb.3
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:_(i64) = G_PHI [[C3]](i64), %bb.2, %181(i64), %bb.3
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:_(i64) = G_PHI [[MV7]](i64), %bb.2, %130(i64), %bb.3
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:_(i64) = G_PHI [[MV8]](i64), %bb.2, %132(i64), %bb.3
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:_(i64) = G_PHI [[SELECT24]](i64), %bb.2, %152(i64), %bb.3
+ ; CHECK-NEXT: [[PHI5:%[0-9]+]]:_(i64) = G_PHI [[SELECT25]](i64), %bb.2, %154(i64), %bb.3
+ ; CHECK-NEXT: [[PHI6:%[0-9]+]]:_(i64) = G_PHI [[SELECT20]](i64), %bb.2, %215(i64), %bb.3
+ ; CHECK-NEXT: [[PHI7:%[0-9]+]]:_(i64) = G_PHI [[SELECT22]](i64), %bb.2, %216(i64), %bb.3
+ ; CHECK-NEXT: [[C62:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i64) = G_SHL [[PHI4]], [[C62]](i32)
+ ; CHECK-NEXT: [[C63:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i64) = G_SHL [[PHI5]], [[C63]](i32)
+ ; CHECK-NEXT: [[C64:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[LSHR4:%[0-9]+]]:_(i64) = G_LSHR [[PHI4]], [[C64]](i32)
+ ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i64) = G_OR [[SHL5]], [[LSHR4]]
+ ; CHECK-NEXT: [[C65:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[LSHR5:%[0-9]+]]:_(i64) = G_LSHR [[PHI7]], [[C65]](i32)
+ ; CHECK-NEXT: [[C66:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i64) = G_OR [[SHL4]], [[LSHR5]]
+ ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i64) = G_OR [[OR8]], [[C66]]
+ ; CHECK-NEXT: [[C67:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i64) = G_SHL [[PHI6]], [[C67]](i32)
+ ; CHECK-NEXT: [[C68:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i64) = G_SHL [[PHI7]], [[C68]](i32)
+ ; CHECK-NEXT: [[C69:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(i64) = G_LSHR [[PHI6]], [[C69]](i32)
+ ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i64) = G_OR [[SHL7]], [[LSHR6]]
+ ; CHECK-NEXT: [[OR12:%[0-9]+]]:_(i64) = G_OR [[PHI]], [[SHL6]]
+ ; CHECK-NEXT: [[OR13:%[0-9]+]]:_(i64) = G_OR [[PHI1]], [[OR11]]
+ ; CHECK-NEXT: [[UV42:%[0-9]+]]:_(i32), [[UV43:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[OR9]](i64)
+ ; CHECK-NEXT: [[USUBO6:%[0-9]+]]:_(i32), [[USUBO7:%[0-9]+]]:_(i1) = G_USUBO [[UADDO6]], [[UV42]]
+ ; CHECK-NEXT: [[USUBE12:%[0-9]+]]:_(i32), [[USUBE13:%[0-9]+]]:_(i1) = G_USUBE [[UADDE18]], [[UV43]], [[USUBO7]]
+ ; CHECK-NEXT: [[UV44:%[0-9]+]]:_(i32), [[UV45:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[OR10]](i64)
+ ; CHECK-NEXT: [[USUBE14:%[0-9]+]]:_(i32), [[USUBE15:%[0-9]+]]:_(i1) = G_USUBE [[UADDE20]], [[UV44]], [[USUBE13]]
+ ; CHECK-NEXT: [[USUBE16:%[0-9]+]]:_(i32), [[USUBE17:%[0-9]+]]:_(i1) = G_USUBE [[UADDE22]], [[UV45]], [[USUBE15]]
+ ; CHECK-NEXT: [[MV9:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBE14]](i32), [[USUBE16]](i32)
+ ; CHECK-NEXT: [[C70:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(i64) = G_ASHR [[MV9]], [[C70]](i32)
+ ; CHECK-NEXT: [[C71:%[0-9]+]]:_(i32) = G_CONSTANT i32 63
+ ; CHECK-NEXT: [[ASHR1:%[0-9]+]]:_(i64) = G_ASHR [[MV9]], [[C71]](i32)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i64) = G_AND [[ASHR]], [[C]]
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i64) = G_AND [[ASHR1]], [[C1]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i64) = G_AND [[ASHR]], [[MV4]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i64) = G_AND [[ASHR1]], [[C22]]
+ ; CHECK-NEXT: [[UV46:%[0-9]+]]:_(i32), [[UV47:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[OR9]](i64)
+ ; CHECK-NEXT: [[UV48:%[0-9]+]]:_(i32), [[UV49:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AND5]](i64)
+ ; CHECK-NEXT: [[USUBO8:%[0-9]+]]:_(i32), [[USUBO9:%[0-9]+]]:_(i1) = G_USUBO [[UV46]], [[UV48]]
+ ; CHECK-NEXT: [[USUBE18:%[0-9]+]]:_(i32), [[USUBE19:%[0-9]+]]:_(i1) = G_USUBE [[UV47]], [[UV49]], [[USUBO9]]
+ ; CHECK-NEXT: [[MV10:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBO8]](i32), [[USUBE18]](i32)
+ ; CHECK-NEXT: [[UV50:%[0-9]+]]:_(i32), [[UV51:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[OR10]](i64)
+ ; CHECK-NEXT: [[UV52:%[0-9]+]]:_(i32), [[UV53:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AND6]](i64)
+ ; CHECK-NEXT: [[USUBE20:%[0-9]+]]:_(i32), [[USUBE21:%[0-9]+]]:_(i1) = G_USUBE [[UV50]], [[UV52]], [[USUBE19]]
+ ; CHECK-NEXT: [[USUBE22:%[0-9]+]]:_(i32), [[USUBE23:%[0-9]+]]:_(i1) = G_USUBE [[UV51]], [[UV53]], [[USUBE21]]
+ ; CHECK-NEXT: [[MV11:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[USUBE20]](i32), [[USUBE22]](i32)
+ ; CHECK-NEXT: [[UV54:%[0-9]+]]:_(i32), [[UV55:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[PHI2]](i64)
+ ; CHECK-NEXT: [[UV56:%[0-9]+]]:_(i32), [[UV57:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C6]](i64)
+ ; CHECK-NEXT: [[UADDO8:%[0-9]+]]:_(i32), [[UADDO9:%[0-9]+]]:_(i1) = G_UADDO [[UV54]], [[UV56]]
+ ; CHECK-NEXT: [[UADDE24:%[0-9]+]]:_(i32), [[UADDE25:%[0-9]+]]:_(i1) = G_UADDE [[UV55]], [[UV57]], [[UADDO9]]
+ ; CHECK-NEXT: [[MV12:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDO8]](i32), [[UADDE24]](i32)
+ ; CHECK-NEXT: [[UV58:%[0-9]+]]:_(i32), [[UV59:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[PHI3]](i64)
+ ; CHECK-NEXT: [[UV60:%[0-9]+]]:_(i32), [[UV61:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C7]](i64)
+ ; CHECK-NEXT: [[UADDE26:%[0-9]+]]:_(i32), [[UADDE27:%[0-9]+]]:_(i1) = G_UADDE [[UV58]], [[UV60]], [[UADDE25]]
+ ; CHECK-NEXT: [[UADDE28:%[0-9]+]]:_(i32), [[UADDE29:%[0-9]+]]:_(i1) = G_UADDE [[UV59]], [[UV61]], [[UADDE27]]
+ ; CHECK-NEXT: [[MV13:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDE26]](i32), [[UADDE28]](i32)
+ ; CHECK-NEXT: [[C72:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[XOR6:%[0-9]+]]:_(i64) = G_XOR [[MV12]], [[C2]]
+ ; CHECK-NEXT: [[XOR7:%[0-9]+]]:_(i64) = G_XOR [[MV13]], [[C3]]
+ ; CHECK-NEXT: [[OR14:%[0-9]+]]:_(i64) = G_OR [[XOR6]], [[XOR7]]
+ ; CHECK-NEXT: [[ICMP23:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[OR14]](i64), [[C72]]
+ ; CHECK-NEXT: G_BRCOND [[ICMP23]](i1), %bb.4
+ ; CHECK-NEXT: G_BR %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4.udiv-loop-exit:
+ ; CHECK-NEXT: successors: %bb.5(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI8:%[0-9]+]]:_(i64) = G_PHI [[C2]](i64), %bb.1, [[AND3]](i64), %bb.3
+ ; CHECK-NEXT: [[PHI9:%[0-9]+]]:_(i64) = G_PHI [[SELECT20]](i64), %bb.1, [[OR12]](i64), %bb.3
+ ; CHECK-NEXT: [[C73:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i64) = G_SHL [[PHI9]], [[C73]](i32)
+ ; CHECK-NEXT: [[OR15:%[0-9]+]]:_(i64) = G_OR [[PHI8]], [[SHL8]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5.udiv-end:
+ ; CHECK-NEXT: [[PHI10:%[0-9]+]]:_(i64) = G_PHI [[OR15]](i64), %bb.4, [[SELECT18]](i64), %bb.0
+ ; CHECK-NEXT: [[C74:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C75:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; CHECK-NEXT: [[SELECT26:%[0-9]+]]:_(i64) = G_SELECT [[ICMP4]](i1), [[C75]], [[C74]]
+ ; CHECK-NEXT: [[UV62:%[0-9]+]]:_(i32), [[UV63:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[PHI10]](i64)
+ ; CHECK-NEXT: [[UV64:%[0-9]+]]:_(i32), [[UV65:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[SELECT26]](i64)
+ ; CHECK-NEXT: [[UADDO10:%[0-9]+]]:_(i32), [[UADDO11:%[0-9]+]]:_(i1) = G_UADDO [[UV62]], [[UV64]]
+ ; CHECK-NEXT: [[UADDE30:%[0-9]+]]:_(i32), [[UADDE31:%[0-9]+]]:_(i1) = G_UADDE [[UV63]], [[UV65]], [[UADDO11]]
+ ; CHECK-NEXT: [[MV14:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[UADDO10]](i32), [[UADDE30]](i32)
+ ; CHECK-NEXT: G_STORE [[MV14]](i64), [[C8]](p3) :: (store (i64) into `ptr addrspace(3) null`, align 2147483648, addrspace 3)
+ ; CHECK-NEXT: ret
+ bb.1..preheader3_udiv-special-cases:
+ %0:reg64b(i64) = loadParam_i64 0, 0
+ %1:reg64b(i64) = loadParam_i64 1, 0
+ %4:_(i128) = G_CONSTANT i128 1
+ %15:_(i128) = G_CONSTANT i128 0
+ %22:_(i128) = G_CONSTANT i128 127
+ %34:_(i128) = G_CONSTANT i128 -1
+ %60:_(p3) = G_CONSTANT i32 0
+ %2:_(i32) = G_INTRINSIC intrinsic(@llvm.pisa.local.size.x)
+ %3:_(i128) = G_ZEXT %0(i64)
+ %5:_(i128) = nuw nsw G_ADD %3, %4
+ %6:_(i128) = G_ZEXT %1(i64)
+ %7:_(i128) = G_UMAX %5, %6
+ %8:_(i1) = G_ICMP intpred(uge), %0(i64), %1
+ %9:_(i128) = G_ZEXT %8(i1)
+ %10:_(i128) = nuw nsw G_ADD %9, %6
+ %11:_(i128) = nsw G_SUB %7, %10
+ %61:_(i32) = G_FREEZE %2
+ %12:_(i128) = G_ZEXT %61(i32)
+ %14:_(i128) = G_FREEZE %11
+ %16:_(i1) = G_ICMP intpred(eq), %12(i128), %15
+ %17:_(i1) = G_ICMP intpred(eq), %14(i128), %15
+ %18:_(i1) = G_OR %16, %17
+ %19:_(i128) = G_CTLZ_ZERO_POISON %12(i128)
+ %20:_(i128) = G_CTLZ_ZERO_POISON %14(i128)
+ %21:_(i128) = G_SUB %19, %20
+ %23:_(i1) = G_ICMP intpred(ugt), %21(i128), %22
+ %63:_(i1) = G_FREEZE %23
+ %24:_(i1) = G_OR %18, %63
+ %26:_(i1) = G_ICMP intpred(eq), %21(i128), %22
+ %27:_(i128) = G_SELECT %24(i1), %15, %14
+ %65:_(i1) = G_FREEZE %26
+ %28:_(i1) = G_OR %24, %65
+ G_BRCOND %28(i1), %bb.6
+ G_BR %bb.2
+
+ bb.2.udiv-bb1:
+ %29:_(i128), %30:_(i1) = G_UADDO %21, %4
+ %31:_(i128) = G_SUB %22, %21
+ %32:_(i128) = G_SHL %14, %31(i128)
+ G_BRCOND %30(i1), %bb.5
+ G_BR %bb.3
+
+ bb.3.udiv-preheader:
+ %33:_(i128) = G_LSHR %14, %29(i128)
+ %35:_(i128) = G_ADD %12, %34
+
+ bb.4.udiv-do-while:
+ successors: %bb.5(0x04000000), %bb.4(0x7c000000)
+
+ %36:_(i128) = G_PHI %15(i128), %bb.3, %47(i128), %bb.4
+ %37:_(i128) = G_PHI %29(i128), %bb.3, %50(i128), %bb.4
+ %38:_(i128) = G_PHI %33(i128), %bb.3, %49(i128), %bb.4
+ %39:_(i128) = G_PHI %32(i128), %bb.3, %44(i128), %bb.4
+ %40:_(i128) = G_SHL %38, %4(i128)
+ %41:_(i128) = G_LSHR %39, %22(i128)
+ %42:_(i128) = G_OR %40, %41
+ %43:_(i128) = G_SHL %39, %4(i128)
+ %44:_(i128) = G_OR %36, %43
+ %45:_(i128) = G_SUB %35, %42
+ %46:_(i128) = G_ASHR %45, %22(i128)
+ %47:_(i128) = G_AND %46, %4
+ %48:_(i128) = G_AND %46, %12
+ %49:_(i128) = G_SUB %42, %48
+ %50:_(i128) = G_ADD %37, %34
+ %51:_(i1) = G_ICMP intpred(eq), %50(i128), %15
+ G_BRCOND %51(i1), %bb.5
+ G_BR %bb.4
+
+ bb.5.udiv-loop-exit:
+ %52:_(i128) = G_PHI %15(i128), %bb.2, %47(i128), %bb.4
+ %53:_(i128) = G_PHI %32(i128), %bb.2, %44(i128), %bb.4
+ %54:_(i128) = G_SHL %53, %4(i128)
+ %55:_(i128) = G_OR %52, %54
+
+ bb.6.udiv-end:
+ %56:_(i128) = G_PHI %55(i128), %bb.5, %27(i128), %bb.1
+ %57:_(i128) = G_ZEXT %8(i1)
+ %58:_(i128) = G_ADD %56, %57
+ %59:_(i64) = G_TRUNC %58(i128)
+ G_STORE %59(i64), %60(p3) :: (store (i64) into `ptr addrspace(3) null`, align 2147483648, addrspace 3)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir
new file mode 100644
index 0000000000000..e95a5b0a44db9
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir
@@ -0,0 +1,53 @@
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Verify UNMERGE lowering case where both SrcTy and DstTy are vectors with over 4 elements.
+
+---
+name: test_unmerge_vector
+legalized: false
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: {{^}}name: test_unmerge_vector
+ ; CHECK: [[REG0:%[0-9]+]]:regv2_32b(<2 x s32>) = loadParam_v2i32 0, 0
+ ; CHECK: [[REG1:%[0-9]+]]:regv2_32b(<2 x s32>) = loadParam_v2i32 0, 8
+ ; CHECK: [[REG2:%[0-9]+]]:regv2_32b(<2 x s32>) = loadParam_v2i32 0, 16
+ %2:regv2_32b(<2 x s32>) = loadParam_v2i32 0, 0
+ %3:regv2_32b(<2 x s32>) = loadParam_v2i32 0, 8
+ %4:regv2_32b(<2 x s32>) = loadParam_v2i32 0, 16
+
+ ; CHECK: [[LO0:%[0-9]+]]:_(s32), [[HI0:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[REG0]]
+ ; CHECK: [[LO1:%[0-9]+]]:_(s32), [[HI1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[REG1]]
+ ; CHECK: [[LO2:%[0-9]+]]:_(s32), [[HI2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[REG2]]
+ %1:_(<6 x s32>) = G_CONCAT_VECTORS %2(<2 x s32>), %3(<2 x s32>), %4(<2 x s32>)
+
+ ; CHECK: [[BC0:%[0-9]+]]:_(<4 x i8>) = G_BITCAST [[LO0]]
+ ; CHECK: [[BC1:%[0-9]+]]:_(<4 x i8>) = G_BITCAST [[HI0]]
+ ; CHECK: [[BC2:%[0-9]+]]:_(<4 x i8>) = G_BITCAST [[LO1]]
+ ; CHECK: [[BC3:%[0-9]+]]:_(<4 x i8>) = G_BITCAST [[HI1]]
+ ; CHECK: [[BC4:%[0-9]+]]:_(<4 x i8>) = G_BITCAST [[LO2]]
+ ; CHECK: [[BC5:%[0-9]+]]:_(<4 x i8>) = G_BITCAST [[HI2]]
+ %0:_(<24 x i8>) = G_BITCAST %1(<6 x s32>)
+
+ ; CHECK: [[B0_0:%[0-9]+]]:_(i8), [[B0_1:%[0-9]+]]:_(i8), [[B0_2:%[0-9]+]]:_(i8), [[B0_3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BC0]]
+ ; CHECK: [[B1_0:%[0-9]+]]:_(i8), [[B1_1:%[0-9]+]]:_(i8), [[B1_2:%[0-9]+]]:_(i8), [[B1_3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BC1]]
+ ; CHECK: [[B2_0:%[0-9]+]]:_(i8), [[B2_1:%[0-9]+]]:_(i8), [[B2_2:%[0-9]+]]:_(i8), [[B2_3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BC2]]
+ ; CHECK: [[B3_0:%[0-9]+]]:_(i8), [[B3_1:%[0-9]+]]:_(i8), [[B3_2:%[0-9]+]]:_(i8), [[B3_3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BC3]]
+ ; CHECK: [[B4_0:%[0-9]+]]:_(i8), [[B4_1:%[0-9]+]]:_(i8), [[B4_2:%[0-9]+]]:_(i8), [[B4_3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BC4]]
+ ; CHECK: [[B5_0:%[0-9]+]]:_(i8), [[B5_1:%[0-9]+]]:_(i8), [[B5_2:%[0-9]+]]:_(i8), [[B5_3:%[0-9]+]]:_(i8) = G_UNMERGE_VALUES [[BC5]]
+ %5:_(<6 x i8>), %6:_(<6 x i8>), %7:_(<6 x i8>), %8:_(<6 x i8>) = G_UNMERGE_VALUES %0(<24 x i8>)
+ %9:_(i48) = G_BITCAST %5(<6 x i8>)
+ %10:_(i48) = G_BITCAST %6(<6 x i8>)
+ %11:_(i48) = G_BITCAST %7(<6 x i8>)
+ %12:_(i48) = G_BITCAST %8(<6 x i8>)
+ %13:_(<4 x i48>) = G_BUILD_VECTOR %9(i48), %10(i48), %11(i48), %12(i48)
+ %14:_(<3 x i64>) = G_BITCAST %13(<4 x i48>)
+
+ ; CHECK-NOT: {{.*}} <24 x i8>
+ ; CHECK-NOT: {{.*}} <6 x i8>
+
+ %15:_(p1) = G_CONSTANT i64 0
+ G_STORE %14(<3 x i64>), %15(p1) :: (store (<3 x i64>) into `ptr addrspace(1) null`, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
new file mode 100644
index 0000000000000..b491bd641184e
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
@@ -0,0 +1,30 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# Ensure that G_ZEXT with large destination types is handled without causing a crash.
+
+--- |
+ define i128 @test_zext_big(i16 %arg) {
+ %ret = zext i16 %arg to i128
+ ret i128 %ret
+ }
+...
+---
+name: test_zext_big
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: test_zext_big
+ ; CHECK: %0:reg16b(i16) = functionParameter_i16 0
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i16) = G_CONSTANT i16 0
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES %0(i16), [[C]](i16), [[C]](i16), [[C]](i16)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(i128) = G_MERGE_VALUES [[MV]](i64), [[C1]](i64)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i64>) = G_BITCAST [[MV1]](i128)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:regv2_64b(<2 x i64>) = COPY [[BITCAST]](<2 x i64>)
+ ; CHECK-NEXT: retValue_v2i64_r [[COPY]](<2 x i64>)
+ %0:reg16b(i16) = functionParameter_i16 0
+ %1:_(i128) = G_ZEXT %0:reg16b(i16)
+ %2:_(<2 x i64>) = G_BITCAST %1:_(i128)
+ %3:regv2_64b(<2 x i64>) = COPY %2:_(<2 x i64>)
+ retValue_v2i64_r %3:regv2_64b(<2 x i64>)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir b/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir
new file mode 100644
index 0000000000000..6ffd5feac885d
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir
@@ -0,0 +1,21 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+---
+name: test_bfloat
+tracksRegLiveness: true
+body: |
+ bb.1.entry:
+ ; CHECK-LABEL: name: test_bfloat
+ ; CHECK: %0:reg16b(bf16) = functionParameter_i16 0
+ ; CHECK-NEXT: %1:reg16b(bf16) = functionParameter_i16 1
+ ; ------------vvvvvvvvvvvvvvvv- This handle was generated thanks to updating update_mir_test_checks.py
+ ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(bf16) = G_FMUL %0, %1
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg16b(i16) = G_BITCAST [[FMUL]](bf16)
+ ; CHECK-NEXT: retValue_i16_r [[COPY]](i16)
+ ; CHECK-NOT: error: {{.*}} expected a machine instruction
+ %0:reg16b(bf16) = functionParameter_i16 0
+ %1:reg16b(bf16) = functionParameter_i16 1
+ %2:_(bf16) = G_FMUL %0:reg16b, %1:reg16b(bf16)
+ %3:reg16b(i16) = G_BITCAST %2:_(bf16)
+ retValue_i16_r %3:reg16b(i16)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
new file mode 100644
index 0000000000000..3f43b136219ca
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
@@ -0,0 +1,67 @@
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# Tests the compare_select PostLegalizerCombiner rule:
+# %cond = G_ICMP intpred(ne), %x(s32), 0
+# %res = G_SELECT %cond(s1), %a(s32), %b(s32)
+# =>
+# %res = G_PISA_SELECT %x(s32), %a(s32), %b(s32)
+
+--- |
+ define i32 @compare_select_ne(i32 %x, i32 %a, i32 %b) {
+ %cond = icmp ne i32 %x, 0
+ %res = select i1 %cond, i32 %a, i32 %b
+ ret i32 %res
+ }
+
+ define i32 @compare_select_eq(i32 %x, i32 %a, i32 %b) {
+ %cond = icmp eq i32 %x, 0
+ %res = select i1 %cond, i32 %a, i32 %b
+ ret i32 %res
+ }
+...
+---
+name: compare_select_ne
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: compare_select_ne
+ ; CHECK: %0:reg32b(i32) = functionParameter_i32 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: %2:reg32b(i32) = functionParameter_i32 2
+ ; CHECK-NEXT: %5:_(i32) = G_PISA_SELECT %0, %1, %2
+ ; CHECK-NEXT: %6:reg32b(i32) = COPY %5(i32)
+ ; CHECK-NEXT: retValue_i32_r %6(i32)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:reg32b(i32) = functionParameter_i32 2
+ %3:_(i32) = G_CONSTANT i32 0
+ %4:_(i1) = G_ICMP intpred(ne), %0(i32), %3(i32)
+ %5:_(i32) = G_SELECT %4(i1), %1(i32), %2(i32)
+ %6:reg32b(i32) = COPY %5(i32)
+ retValue_i32_r %6:reg32b(i32)
+...
+---
+name: compare_select_eq
+legalized: true
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: compare_select_eq
+ ; CHECK: %0:reg32b(i32) = functionParameter_i32 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: %2:reg32b(i32) = functionParameter_i32 2
+ ; CHECK-NEXT: %5:_(i32) = G_PISA_SELECT %0, %2, %1
+ ; CHECK-NEXT: %6:reg32b(i32) = COPY %5(i32)
+ ; CHECK-NEXT: retValue_i32_r %6(i32)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %2:reg32b(i32) = functionParameter_i32 2
+ %3:_(i32) = G_CONSTANT i32 0
+ %4:_(i1) = G_ICMP intpred(eq), %0(i32), %3(i32)
+ %5:_(i32) = G_SELECT %4(i1), %1(i32), %2(i32)
+ %6:reg32b(i32) = COPY %5(i32)
+ retValue_i32_r %6:reg32b(i32)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir
new file mode 100644
index 0000000000000..2c8f60aee0b89
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir
@@ -0,0 +1,38 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# G_EXTRACT_SUBVECTOR fed by a chain of G_INSERT_SUBVECTOR into IMPLICIT_DEF,
+# where the extracted window fully contains the inserts. The postlegalizer
+# combiner rewrites the extract into a smaller insert chain with rebased
+# indices (applyExtractSubvectorPartial). Unlike the shufflevector-lowered
+# shape in combine-extract-subvector-partial.ll (whose chain base is the
+# generic G_IMPLICIT_DEF, which the matcher rejects), this hand-written chain
+# feeds a non-generic IMPLICIT_DEF, so the apply path fires.
+#
+# Here the extract of the upper 8 elements (index 8) sees two <4 x i32> inserts
+# at indices 8 and 12; both are fully inside [8, 16), so the combine fires and
+# rebases them to 0 and 4 in a fresh <8 x i32> chain.
+
+---
+name: extract_subvector_partial_upper
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: extract_subvector_partial_upper
+ ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[C1]](i32), [[C1]](i32), [[C1]](i32), [[C1]](i32)
+ ; CHECK-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[C1]](i32), [[C1]](i32), [[C1]](i32), [[C1]](i32)
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<8 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[INSERT_SUBVECTOR:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[DEF]], [[BUILD_VECTOR]](<4 x i32>), 0
+ ; CHECK-NEXT: [[INSERT_SUBVECTOR1:%[0-9]+]]:_(<8 x i32>) = G_INSERT_SUBVECTOR [[INSERT_SUBVECTOR]], [[BUILD_VECTOR1]](<4 x i32>), 4
+ ; CHECK-NEXT: G_STORE [[INSERT_SUBVECTOR1]](<8 x i32>), [[C]](p1) :: (store (<8 x i32>))
+ %0:_(p1) = G_CONSTANT i64 0
+ %1:_(i32) = G_CONSTANT i32 7
+ %2:_(<16 x i32>) = IMPLICIT_DEF
+ %3:_(<4 x i32>) = G_BUILD_VECTOR %1:_(i32), %1:_(i32), %1:_(i32), %1:_(i32)
+ %4:_(<4 x i32>) = G_BUILD_VECTOR %1:_(i32), %1:_(i32), %1:_(i32), %1:_(i32)
+ %5:_(<16 x i32>) = G_INSERT_SUBVECTOR %2:_(<16 x i32>), %3:_(<4 x i32>), 8
+ %6:_(<16 x i32>) = G_INSERT_SUBVECTOR %5:_(<16 x i32>), %4:_(<4 x i32>), 12
+ %7:_(<8 x i32>) = G_EXTRACT_SUBVECTOR %6:_(<16 x i32>), 8
+ G_STORE %7:_(<8 x i32>), %0:_(p1) :: (store (<8 x i32>))
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
new file mode 100644
index 0000000000000..e0ab946e993ca
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
@@ -0,0 +1,52 @@
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# Test that the PostLegalizer does not infinite-loop on shl(or x, const), const.
+#
+# When commute_shift is enabled together with hoist_logic_op_with_same_opcode_hands
+# in the PostLegalizer, the two rules form an infinite cycle on the OR variant:
+#
+# commute_shift:
+# shl (or x, c1), c2 -> or (shl x, c2), (shl c1, c2)
+#
+# hoist_logic_op_with_same_opcode_hands:
+# or (shl x, c2), (shl c1, c2) -> shl (or x, c1), c2
+#
+# Both created SHLs share the same shift-amount register, so
+# hoist_logic_op_with_same_opcode_hands recognises them as "same opcode,
+# same extra operand" and re-folds the OR through the SHL - recreating
+# the original input to commute_shift.
+#
+# commute_shift is therefore excluded from the PostLegalizer
+# (pisa_post_removed_combines).
+
+--- |
+ define i32 @shl_or_const(i32 %x) {
+ %or = or i32 %x, 4
+ %shl = shl i32 %or, 2
+ ret i32 %shl
+ }
+...
+---
+name: shl_or_const
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; The pattern must survive the PostLegalizer without hanging.
+ ; With commute_shift excluded, the shl(or) stays as-is.
+ ;
+ ; CHECK-LABEL: name: shl_or_const
+ ; CHECK: %0:reg32b(i32) = functionParameter_i32 0
+ ; CHECK: %c1:reg32b(i32) = G_CONSTANT i32 4
+ ; CHECK: %c2:reg32b(i32) = G_CONSTANT i32 2
+ ; CHECK: %or:reg32b(i32) = G_OR %0, %c1
+ ; CHECK: %shl:reg32b(i32) = G_SHL %or, %c2(i32)
+ ; CHECK: retValue_i32_r %shl(i32)
+ %0:reg32b(i32) = functionParameter_i32 0
+ %c1:reg32b(i32) = G_CONSTANT i32 4
+ %c2:reg32b(i32) = G_CONSTANT i32 2
+ %or:reg32b(i32) = G_OR %0:reg32b, %c1:reg32b
+ %shl:reg32b(i32) = G_SHL %or:reg32b, %c2:reg32b(i32)
+ retValue_i32_r %shl(i32)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir b/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir
new file mode 100644
index 0000000000000..e194f452990c4
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir
@@ -0,0 +1,29 @@
+# RUN: llc -march=pisa -run-pass=postrapseudos -verify-machineinstrs %s -o - | FileCheck -check-prefix=PSEUDO %s
+# RUN-CRASHES: llc -march=pisa -start-before=postrapseudos -verify-machineinstrs %s -o - | FileCheck -check-prefix=ASM %s
+--- |
+ define i32 @test_copy_pred(i32 %arg) {
+ ret i32 %arg
+ }
+...
+---
+name: test_copy_pred
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+body: |
+ bb.1:
+ ; PSEUDO-LABEL: {{^}}name: test_copy_pred
+ ; ASM-LABEL: @test_copy_pred
+ ; PSEUDO: %pred:pred(i1) = G_TRUNC %0(i32)
+ ; PSEUDO: %4:reg16b(i16) = sel_16_iip %pred(i1), 1, 0
+ ; PSEUDO: %copy:pred(i1) = ucmp_ne_16b_pri %4(i16), 0
+ ; PSEUDO: %rv:reg32b(i32) = G_ANYEXT %copy(i1)
+ ; ASM: mov.16b %r{{[0-9]+}}, 0
+ ; ASM: and.16b %r{{[0-9]+}}, %r{{[0-9]+}}, 1
+ %0:reg32b(i32) = functionParameter_i32 0
+ %pred:pred(i1) = G_TRUNC %0(i32)
+ %copy:pred(i1) = COPY %pred(i1)
+ %rv:reg32b(i32) = G_ANYEXT %copy(i1)
+ retValue_i32_r %rv(i32)
+
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
new file mode 100644
index 0000000000000..456a006bded3b
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
@@ -0,0 +1,181 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --validate-debuginfo False --version 6
+# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# The reduce_predicates PreLegalizerCombiner rule rewrites an AND/OR/XOR
+# reduction over the per-lane results of a vector compare into an explicit
+# s32 reduction. Reducing extract_vector_elt lanes of a <N x i1> G_ICMP with
+# G_AND / G_OR / G_XOR exercises each reduction-opcode arm of the rewrite.
+#
+# When the reduction result is wider than i1 and its single use is a G_ICMP
+# against 0 or -1, the rewrite folds that compare into the s32 reduction,
+# inverting the predicate for the -1 case (eq -> ne) and leaving it unchanged
+# for the 0 case.
+
+--- |
+ define i32 @reduce_and(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
+ define i32 @reduce_or(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
+ define i32 @reduce_xor(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
+ define i32 @reduce_and_use_icmp_neg1(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
+ define i32 @reduce_and_use_icmp_zero(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
+...
+---
+name: reduce_and
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: reduce_and
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x i1>) = G_ICMP intpred(eq), [[DEF]](<2 x i32>), [[DEF1]]
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x i32>) = G_SEXT [[ICMP]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C1]](i32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[EVEC]], [[EVEC1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND]](i32), [[C2]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:reg32b(i32) = G_ZEXT [[ICMP1]](i1)
+ ; CHECK-NEXT: retValue_i32_r [[ZEXT]](i32)
+ %0:_(<2 x i32>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_IMPLICIT_DEF
+ %2:_(<2 x i1>) = G_ICMP intpred(eq), %0(<2 x i32>), %1
+ %3:_(i32) = G_CONSTANT i32 0
+ %4:_(i32) = G_CONSTANT i32 1
+ %5:_(i1) = G_EXTRACT_VECTOR_ELT %2(<2 x i1>), %3(i32)
+ %6:_(i1) = G_EXTRACT_VECTOR_ELT %2(<2 x i1>), %4(i32)
+ %7:_(i1) = G_AND %5, %6
+ %8:reg32b(i32) = G_ZEXT %7(i1)
+ retValue_i32_r %8:reg32b(i32)
+...
+---
+name: reduce_or
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: reduce_or
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x i1>) = G_ICMP intpred(eq), [[DEF]](<2 x i32>), [[DEF1]]
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x i32>) = G_SEXT [[ICMP]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[EVEC]], [[EVEC1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[OR]](i32), [[C2]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:reg32b(i32) = G_ZEXT [[ICMP1]](i1)
+ ; CHECK-NEXT: retValue_i32_r [[ZEXT]](i32)
+ %0:_(<2 x i32>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_IMPLICIT_DEF
+ %2:_(<2 x i1>) = G_ICMP intpred(eq), %0(<2 x i32>), %1
+ %3:_(i32) = G_CONSTANT i32 0
+ %4:_(i32) = G_CONSTANT i32 1
+ %5:_(i1) = G_EXTRACT_VECTOR_ELT %2(<2 x i1>), %3(i32)
+ %6:_(i1) = G_EXTRACT_VECTOR_ELT %2(<2 x i1>), %4(i32)
+ %7:_(i1) = G_OR %5, %6
+ %8:reg32b(i32) = G_ZEXT %7(i1)
+ retValue_i32_r %8:reg32b(i32)
+...
+---
+name: reduce_xor
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: reduce_xor
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x i1>) = G_ICMP intpred(eq), [[DEF]](<2 x i32>), [[DEF1]]
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x i32>) = G_SEXT [[ICMP]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C1]](i32)
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i32) = G_XOR [[EVEC]], [[EVEC1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[XOR]](i32), [[C2]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:reg32b(i32) = G_ZEXT [[ICMP1]](i1)
+ ; CHECK-NEXT: retValue_i32_r [[ZEXT]](i32)
+ %0:_(<2 x i32>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_IMPLICIT_DEF
+ %2:_(<2 x i1>) = G_ICMP intpred(eq), %0(<2 x i32>), %1
+ %3:_(i32) = G_CONSTANT i32 0
+ %4:_(i32) = G_CONSTANT i32 1
+ %5:_(i1) = G_EXTRACT_VECTOR_ELT %2(<2 x i1>), %3(i32)
+ %6:_(i1) = G_EXTRACT_VECTOR_ELT %2(<2 x i1>), %4(i32)
+ %7:_(i1) = G_XOR %5, %6
+ %8:reg32b(i32) = G_ZEXT %7(i1)
+ retValue_i32_r %8:reg32b(i32)
+...
+---
+name: reduce_and_use_icmp_neg1
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: reduce_and_use_icmp_neg1
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x i1>) = G_ICMP intpred(eq), [[DEF]](<2 x i32>), [[DEF1]]
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x i32>) = G_SEXT [[ICMP]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C1]](i32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[EVEC]], [[EVEC1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(ne), [[AND]](i32), [[C2]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:reg32b(i32) = G_ZEXT [[ICMP1]](i1)
+ ; CHECK-NEXT: retValue_i32_r [[ZEXT]](i32)
+ %0:_(<2 x i32>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_IMPLICIT_DEF
+ %2:_(<2 x i1>) = G_ICMP intpred(eq), %0(<2 x i32>), %1
+ %3:_(<2 x i16>) = G_SEXT %2(<2 x i1>)
+ %4:_(i32) = G_CONSTANT i32 0
+ %5:_(i32) = G_CONSTANT i32 1
+ %6:_(i16) = G_EXTRACT_VECTOR_ELT %3(<2 x i16>), %4(i32)
+ %7:_(i16) = G_EXTRACT_VECTOR_ELT %3(<2 x i16>), %5(i32)
+ %8:_(i16) = G_AND %6, %7
+ %9:_(i16) = G_CONSTANT i16 -1
+ %10:_(i1) = G_ICMP intpred(eq), %8(i16), %9(i16)
+ %11:reg32b(i32) = G_ZEXT %10(i1)
+ retValue_i32_r %11(i32)
+...
+---
+name: reduce_and_use_icmp_zero
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: reduce_and_use_icmp_zero
+ ; CHECK: [[DEF:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(<2 x i32>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(<2 x i1>) = G_ICMP intpred(eq), [[DEF]](<2 x i32>), [[DEF1]]
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x i32>) = G_SEXT [[ICMP]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[SEXT]](<2 x i32>), [[C1]](i32)
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[EVEC]], [[EVEC1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(i1) = G_ICMP intpred(eq), [[AND]](i32), [[C2]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:reg32b(i32) = G_ZEXT [[ICMP1]](i1)
+ ; CHECK-NEXT: retValue_i32_r [[ZEXT]](i32)
+ %0:_(<2 x i32>) = G_IMPLICIT_DEF
+ %1:_(<2 x i32>) = G_IMPLICIT_DEF
+ %2:_(<2 x i1>) = G_ICMP intpred(eq), %0(<2 x i32>), %1
+ %3:_(<2 x i16>) = G_SEXT %2(<2 x i1>)
+ %4:_(i32) = G_CONSTANT i32 0
+ %5:_(i32) = G_CONSTANT i32 1
+ %6:_(i16) = G_EXTRACT_VECTOR_ELT %3(<2 x i16>), %4(i32)
+ %7:_(i16) = G_EXTRACT_VECTOR_ELT %3(<2 x i16>), %5(i32)
+ %8:_(i16) = G_AND %6, %7
+ %9:_(i16) = G_CONSTANT i16 0
+ %10:_(i1) = G_ICMP intpred(eq), %8(i16), %9(i16)
+ %11:reg32b(i32) = G_ZEXT %10(i1)
+ retValue_i32_r %11(i32)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
new file mode 100644
index 0000000000000..a8d3e9bd863b7
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
@@ -0,0 +1,49 @@
+# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+ define i64 @zext_and_mask_rhs(i32 %x) {
+ ret i64 0
+ }
+ define i64 @zext_and_mask_lhs(i32 %x) {
+ ret i64 0
+ }
+...
+
+# --- canonical: G_AND (G_ZEXT x), C ---
+---
+name: zext_and_mask_rhs
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: zext_and_mask_rhs
+ ; The combiner rewrites G_AND(G_ZEXT(x),C) to G_ZEXT(G_AND(x,trunc(C))).
+ ; CHECK: [[AND:%[0-9]+]]:_(i32) = G_AND
+ ; CHECK: [[ZEXT:%[0-9]+]]:_(i64) = G_ZEXT [[AND]](i32)
+ ; CHECK: retValue_i64_r
+ %0:reg32b(i32) = functionParameter_i32 0
+ %1:_(i64) = G_ZEXT %0:reg32b(i32)
+ %mask:_(i64) = G_CONSTANT i64 255
+ %2:_(i64) = G_AND %1:_(i64), %mask:_(i64)
+ %3:reg64b(i64) = COPY %2:_(i64)
+ retValue_i64_r %3:reg64b(i64)
+...
+
+# --- swapped: G_AND C, (G_ZEXT x) -- exercises operand-swap branch ---
+---
+name: zext_and_mask_lhs
+tracksRegLiveness: true
+isSSA: true
+body: |
+ bb.1:
+ ; CHECK-LABEL: name: zext_and_mask_lhs
+ ; CHECK: [[AND2:%[0-9]+]]:_(i32) = G_AND
+ ; CHECK: [[ZEXT2:%[0-9]+]]:_(i64) = G_ZEXT [[AND2]](i32)
+ ; CHECK: retValue_i64_r
+ %0:reg32b(i32) = functionParameter_i32 0
+ %1:_(i64) = G_ZEXT %0:reg32b(i32)
+ %mask:_(i64) = G_CONSTANT i64 65535
+ %2:_(i64) = G_AND %mask:_(i64), %1:_(i64)
+ %3:reg64b(i64) = COPY %2:_(i64)
+ retValue_i64_r %3:reg64b(i64)
+...
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir b/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir
new file mode 100644
index 0000000000000..c383a57135cfa
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir
@@ -0,0 +1,37 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: retain_extract_subvector
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: retain_extract_subvector
+ ; CHECK: %0:reg32b(p3) = functionParameter_i32 0
+ ; CHECK-NEXT: %1:reg32b(i32) = functionParameter_i32 1
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD %1, [[C]]
+ ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<8 x i32>) = G_LOAD %0(p3) :: (load (<8 x i32>))
+ ; CHECK-NEXT: [[EXTRACT_SUBVECTOR:%[0-9]+]]:_(<2 x i32>) = G_EXTRACT_SUBVECTOR [[LOAD]](<8 x i32>), 2
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[EXTRACT_SUBVECTOR]](<2 x i32>), %1(i32)
+ ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[EXTRACT_SUBVECTOR]](<2 x i32>), [[ADD]](i32)
+ ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[EVEC]], [[EVEC1]]
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:reg32b(i32) = COPY [[ADD1]](i32)
+ ; CHECK-NEXT: retValue_i32_r [[COPY]](i32)
+ %0:reg32b(p3) = functionParameter_i32 0
+ %1:reg32b(i32) = functionParameter_i32 1
+ %4:_(i32) = G_CONSTANT i32 1
+ %6:_(i32) = G_CONSTANT i32 2
+ %8:_(i32) = G_CONSTANT i32 3
+ %10:_(i32) = G_CONSTANT i32 4
+ %5:_(i32) = G_ADD %1, %4
+ %7:_(i32) = G_ADD %1, %6
+ %9:_(i32) = G_ADD %1, %8
+ %11:_(i32) = G_ADD %1, %10
+ %23:_(<8 x i32>) = G_LOAD %0(p3) :: (load (<8 x i32>))
+ %24:_(<2 x i32>) = G_EXTRACT_SUBVECTOR %23(<8 x i32>), 2
+ %13:_(i32) = G_EXTRACT_VECTOR_ELT %24(<2 x i32>), %1(i32)
+ %14:_(i32) = G_EXTRACT_VECTOR_ELT %24(<2 x i32>), %5(i32)
+ %18:_(i32) = G_ADD %13, %14
+ %22:reg32b(i32) = COPY %18(i32)
+ retValue_i32_r %22(i32)
+...
>From 472ca27ec11a474faa97fe9fe4f1b3a5402f18f9 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Wed, 5 Aug 2026 17:39:33 -0700
Subject: [PATCH 02/13] Fix buildbot issues
---
llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 1 +
llvm/lib/Target/RISCV/RISCVFrameLowering.cpp | 1 +
2 files changed, 2 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index fad04e0c12dde..88dd2585381ab 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1091,6 +1091,7 @@ bool SIFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
case TargetStackID::ScalableVector:
case TargetStackID::ScalablePredicateVector:
case TargetStackID::WasmLocal:
+ case TargetStackID::PISAShared:
return false;
}
llvm_unreachable("Invalid TargetStackID::Value");
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 35a36497d4f65..fa1240da879ca 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -2725,6 +2725,7 @@ bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
case TargetStackID::SGPRSpill:
case TargetStackID::WasmLocal:
case TargetStackID::ScalablePredicateVector:
+ case TargetStackID::PISAShared:
return false;
}
llvm_unreachable("Invalid TargetStackID::Value");
>From e40e9c5a808b485886ec992487d4142b9e654dd3 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 05:42:47 +0000
Subject: [PATCH 03/13] Do not use auto
---
.../llvm/TargetParser/PISATargetParser.h | 3 +-
llvm/lib/Target/PISA/PISACallLowering.cpp | 143 +--
llvm/lib/Target/PISA/PISAISelLowering.cpp | 41 +-
llvm/lib/Target/PISA/PISALegalizerInfo.cpp | 1012 +++++++++--------
.../lib/Target/PISA/PISAMachineFunctionInfo.h | 2 +-
.../Target/PISA/PISAPostLegalizerCombiner.cpp | 364 +++---
.../Target/PISA/PISAPreLegalizerCombiner.cpp | 405 +++----
llvm/lib/Target/PISA/PISARegisterBankInfo.cpp | 2 +-
llvm/lib/Target/PISA/PISASubtarget.cpp | 2 +-
llvm/lib/Target/PISA/PISATargetMachine.cpp | 6 +-
llvm/lib/Target/PISA/PISATargetMachine.h | 4 +-
llvm/lib/TargetParser/PISATargetParser.cpp | 2 +-
12 files changed, 1032 insertions(+), 954 deletions(-)
diff --git a/llvm/include/llvm/TargetParser/PISATargetParser.h b/llvm/include/llvm/TargetParser/PISATargetParser.h
index e9d5cd5bb1d4a..d78319f801914 100644
--- a/llvm/include/llvm/TargetParser/PISATargetParser.h
+++ b/llvm/include/llvm/TargetParser/PISATargetParser.h
@@ -42,7 +42,8 @@ inline PISATargetInfo getPISATargetInfo(StringRef Name) {
#include "PISATargetParser.def"
#undef PISA_TARGET
};
- auto *It = llvm::find_if(Info, [&Name](const PISATargetInfo &Entry) {
+ const PISATargetInfo *It =
+ llvm::find_if(Info, [&Name](const PISATargetInfo &Entry) {
return Entry.Name == Name;
});
return It == std::end(Info) ? DefaultInfo : *It;
diff --git a/llvm/lib/Target/PISA/PISACallLowering.cpp b/llvm/lib/Target/PISA/PISACallLowering.cpp
index 6928ed63d3aab..1868290306b12 100644
--- a/llvm/lib/Target/PISA/PISACallLowering.cpp
+++ b/llvm/lib/Target/PISA/PISACallLowering.cpp
@@ -34,13 +34,13 @@ bool PISACallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
if (VRegs.size() > 1)
return false;
if (Val) {
- auto &DL = MIRBuilder.getDataLayout();
- const auto &STI = MIRBuilder.getMF().getSubtarget();
+ const DataLayout &DL = MIRBuilder.getDataLayout();
+ const TargetSubtargetInfo &STI = MIRBuilder.getMF().getSubtarget();
unsigned Op = 0;
- auto *Ty = Val->getType();
- auto VReg = VRegs[0];
+ Type *Ty = Val->getType();
+ Register VReg = VRegs[0];
if (Ty->isVectorTy()) {
- auto *VTy = cast<FixedVectorType>(Ty);
+ FixedVectorType *VTy = cast<FixedVectorType>(Ty);
unsigned NumElts = VTy->getNumElements();
unsigned EltSize = DL.getTypeSizeInBits(Ty->getScalarType());
switch (EltSize) {
@@ -139,16 +139,21 @@ bool PISACallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
case 1: // change i1 to i16 (see lowerCall())
{
const LLT I16 = LLT::integer(16);
- auto Dst = MIRBuilder.getMRI()->createGenericVirtualRegister(I16);
- auto &MF = MIRBuilder.getMF();
- auto &F = MF.getFunction();
+ Register Dst =
+ MIRBuilder.getMRI()->createGenericVirtualRegister(I16);
+ MachineFunction &MF = MIRBuilder.getMF();
+ Function &F = MF.getFunction();
const DataLayout &DL = MF.getDataLayout();
ArgInfo RetInfo(VReg, *Val, 0);
setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);
- auto Sext = llvm::any_of(
- RetInfo.Flags, [](const auto &Flag) { return Flag.isSExt(); });
- auto Zext = llvm::any_of(
- RetInfo.Flags, [](const auto &Flag) { return Flag.isZExt(); });
+ bool Sext = llvm::any_of(RetInfo.Flags,
+ [](const ISD::ArgFlagsTy &Flag) {
+ return Flag.isSExt();
+ });
+ bool Zext = llvm::any_of(RetInfo.Flags,
+ [](const ISD::ArgFlagsTy &Flag) {
+ return Flag.isZExt();
+ });
if (Sext) {
MIRBuilder.buildSExt(Dst, VReg);
} else if (Zext) {
@@ -185,8 +190,8 @@ bool PISACallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
// there is an expectation of no register class being assigned.
// Having an extra copy here eliminates the problem; copy itself
// will be removed during instruction selection.
- auto *MRI = MIRBuilder.getMRI();
- auto Tmp = MRI->createGenericVirtualRegister(MRI->getType(VReg));
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ Register Tmp = MRI->createGenericVirtualRegister(MRI->getType(VReg));
MIRBuilder.buildCopy(Tmp, VReg);
MIRBuilder.buildInstr(Op).addUse(Tmp).constrainAllUses(
MIRBuilder.getTII(), *STI.getRegisterInfo(), *STI.getRegBankInfo());
@@ -200,18 +205,19 @@ bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
const Function &F,
ArrayRef<ArrayRef<Register>> VRegs,
FunctionLoweringInfo &FLI) const {
- auto *MRI = MIRBuilder.getMRI();
- auto &MF = MIRBuilder.getMF();
- auto &Ctx = F.getContext();
- auto *MFInfo = MF.getInfo<PISAMachineFunctionInfo>();
- auto &DL = F.getParent()->getDataLayout();
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ MachineFunction &MF = MIRBuilder.getMF();
+ LLVMContext &Ctx = F.getContext();
+ PISAMachineFunctionInfo *MFInfo = MF.getInfo<PISAMachineFunctionInfo>();
+ const DataLayout &DL = F.getParent()->getDataLayout();
bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
- for (const auto [i, Arg] : llvm::enumerate(F.args())) {
+ for (const Argument &Arg : F.args()) {
+ unsigned i = Arg.getArgNo();
assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs");
ArgInfo OrigArg{VRegs[i], Arg, static_cast<unsigned>(i)};
setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
- auto *ArgType = OrigArg.OrigValue->getType();
+ Type *ArgType = OrigArg.OrigValue->getType();
const bool IsByRef = ArgType->isPointerTy() && OrigArg.Flags[0].isByRef();
const unsigned ArgSize = IsByRef
? OrigArg.Flags[0].getByRefSize()
@@ -228,11 +234,11 @@ bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
loadParamWithOpcode(MIRBuilder, F, VRegs[i][0], ArgType, Op,
Arg.getArgNo(), 0);
} else if (IsKernel && ArgType->isVectorTy()) {
- auto *VectorTy = cast<FixedVectorType>(ArgType);
- auto NumElts = VectorTy->getNumElements();
- auto *EltTy = VectorTy->getElementType();
- auto EltSize = DL.getTypeSizeInBits(EltTy);
- auto Split = (NumElts > 4) || ((NumElts == 3) && (EltSize != 32)) ||
+ FixedVectorType *VectorTy = cast<FixedVectorType>(ArgType);
+ unsigned NumElts = VectorTy->getNumElements();
+ Type *EltTy = VectorTy->getElementType();
+ unsigned EltSize = DL.getTypeSizeInBits(EltTy);
+ bool Split = (NumElts > 4) || ((NumElts == 3) && (EltSize != 32)) ||
((NumElts == 4) && (EltSize == 64)) || (NumElts == 1);
if (Split) {
// handle odd-sized and large kernel args, e.g.
@@ -241,17 +247,17 @@ bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
// loadParam_8b @[arg+1] loadParam_v4_32b @[arg+16]
// loadParam_8b @[arg+2] buildVector(<8 x i32>)
// buildVector(<3 x i8>) bitcast(<16 x i16)
- auto TargetReg = VRegs[i][0];
+ Register TargetReg = VRegs[i][0];
- const auto *EltRegClass =
+ const TargetRegisterClass *EltRegClass =
(EltSize == 8
? &PISA::Reg8bRegClass
: (EltSize == 16 ? &PISA::Reg16bRegClass
: (EltSize == 32 ? &PISA::Reg32bRegClass
: &PISA::Reg64bRegClass)));
- auto TotalSize = NumElts * EltSize;
- auto EltLLT = LLT::integer(EltSize);
- auto I32 = LLT::integer(32);
+ unsigned TotalSize = NumElts * EltSize;
+ LLT EltLLT = LLT::integer(EltSize);
+ LLT I32 = LLT::integer(32);
if (NumElts <= 4) {
// do not group
} else if (TotalSize % 128 == 0) { // 4 x i32
@@ -279,16 +285,16 @@ bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
SmallVector<Register, 4> Regs;
for (unsigned I = 0; I < NumElts; I++) {
- auto Reg = MRI->createGenericVirtualRegister(EltLLT);
+ Register Reg = MRI->createGenericVirtualRegister(EltLLT);
MRI->setRegClass(Reg, EltRegClass);
Op = getLoadParamOpcode(MIRBuilder, F, Reg, EltTy);
loadParamWithOpcode(MIRBuilder, F, Reg, EltTy, Op, Arg.getArgNo(),
I * EltLLT.getSizeInBytes());
if (EltTy->isPointerTy()) {
// ld.param loads a scalar value, so convert to ptr here
- auto AS = cast<PointerType>(EltTy)->getAddressSpace();
- auto PtrLLT = LLT::pointer(AS, EltSize);
- auto CastReg = MRI->createGenericVirtualRegister(PtrLLT);
+ unsigned AS = cast<PointerType>(EltTy)->getAddressSpace();
+ LLT PtrLLT = LLT::pointer(AS, EltSize);
+ Register CastReg = MRI->createGenericVirtualRegister(PtrLLT);
MRI->setRegClass(CastReg, EltRegClass);
MIRBuilder.buildIntToPtr(CastReg, Reg);
Regs.push_back(CastReg);
@@ -296,11 +302,11 @@ bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
// ld.param loads an integer value; bitcast to float so that
// G_BUILD_VECTOR element types match the result vector element
// type.
- auto FloatLLT = EltTy->isBFloatTy() ? LLT::bfloat16()
- : EltSize == 16 ? LLT::float16()
- : EltSize == 32 ? LLT::float32()
- : LLT::float64();
- auto CastReg = MRI->createGenericVirtualRegister(FloatLLT);
+ LLT FloatLLT = EltTy->isBFloatTy() ? LLT::bfloat16()
+ : EltSize == 16 ? LLT::float16()
+ : EltSize == 32 ? LLT::float32()
+ : LLT::float64();
+ Register CastReg = MRI->createGenericVirtualRegister(FloatLLT);
MRI->setRegClass(CastReg, EltRegClass);
MIRBuilder.buildBitcast(CastReg, Reg);
Regs.push_back(CastReg);
@@ -339,18 +345,19 @@ void PISACallLowering::loadParamWithOpcode(MachineIRBuilder &MIRBuilder,
const Register &VReg, Type *ArgType,
unsigned Opcode, unsigned ArgNo,
unsigned Offset) const {
- auto *MRI = MIRBuilder.getMRI();
- auto &DL = F.getParent()->getDataLayout();
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ const DataLayout &DL = F.getParent()->getDataLayout();
bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
- const auto BitSize = DL.getTypeSizeInBits(ArgType->getScalarType());
+ const unsigned BitSize = DL.getTypeSizeInBits(ArgType->getScalarType());
- auto VReg16 = VReg;
+ Register VReg16 = VReg;
if (BitSize == 1) { // load arg into i16
VReg16 = MRI->createGenericVirtualRegister(LLT::integer(16));
MRI->setRegClass(VReg16, &PISA::Reg16bRegClass);
}
- auto MIB = MIRBuilder.buildInstr(Opcode).addDef(VReg16).addImm(ArgNo);
+ MachineInstrBuilder MIB =
+ MIRBuilder.buildInstr(Opcode).addDef(VReg16).addImm(ArgNo);
if (IsKernel)
MIB.addImm(Offset);
@@ -361,7 +368,7 @@ void PISACallLowering::loadParamWithOpcode(MachineIRBuilder &MIRBuilder,
if (IsKernel)
if (MDNode *MD = F.getMetadata("kernel_arg_name"))
if (ArgNo < MD->getNumOperands())
- if (auto *S = dyn_cast<MDString>(MD->getOperand(ArgNo)))
+ if (MDString *S = dyn_cast<MDString>(MD->getOperand(ArgNo)))
if (!S->getString().empty())
MIB.addExternalSymbol(
MIRBuilder.getMF().createExternalSymbolName(S->getString()));
@@ -375,14 +382,14 @@ unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
const Function &F,
const Register &VReg,
Type *ArgType) const {
- auto *MRI = MIRBuilder.getMRI();
- auto &MF = MIRBuilder.getMF();
- const auto *TRI = static_cast<const PISARegisterInfo *>(
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
+ MachineFunction &MF = MIRBuilder.getMF();
+ const PISARegisterInfo *TRI = static_cast<const PISARegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
unsigned Op = 0;
- auto &DL = F.getParent()->getDataLayout();
+ const DataLayout &DL = F.getParent()->getDataLayout();
const unsigned ParamScalar[2][4] = {
// [isKernel][8/16/32/64]
@@ -407,7 +414,7 @@ unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
{PISA::loadParam_v2i32, PISA::loadParam_v3i32, PISA::loadParam_v4i32},
{PISA::loadParam_v2i64, PISA::loadParam_v3i64, 0}}};
- const auto BitSize = DL.getTypeSizeInBits(ArgType->getScalarType());
+ const unsigned BitSize = DL.getTypeSizeInBits(ArgType->getScalarType());
// Calculate the argument size in bytes.
if (ArgType->isIntegerTy()) {
switch (BitSize) {
@@ -454,8 +461,8 @@ unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
MRI->setRegClass(VReg, &PISA::Reg64bRegClass);
Op = ParamScalar[IsKernel][3];
} else if (ArgType->isVectorTy()) {
- auto *VectorTy = cast<FixedVectorType>(ArgType);
- auto NumElts = VectorTy->getNumElements();
+ FixedVectorType *VectorTy = cast<FixedVectorType>(ArgType);
+ unsigned NumElts = VectorTy->getNumElements();
assert(((((BitSize == 8) || (BitSize == 16) || (BitSize == 64)) &&
((NumElts >= 2) && (NumElts <= 4))) ||
((BitSize == 32) &&
@@ -533,12 +540,12 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
}
MachineInstrBuilder MIB;
- const auto *TRI = static_cast<const PISARegisterInfo *>(
+ const PISARegisterInfo *TRI = static_cast<const PISARegisterInfo *>(
MIRBuilder.getMF().getSubtarget().getRegisterInfo());
if (IsIndirectCall) {
Register CalleeReg = Info.Callee.getReg();
- auto *MRI = MIRBuilder.getMRI();
+ MachineRegisterInfo *MRI = MIRBuilder.getMRI();
LLT CalleeTy = MRI->getType(CalleeReg);
Register CalleeI64Reg = MRI->createGenericVirtualRegister(LLT::integer(64));
@@ -553,18 +560,20 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
// promote i1 args to i16
SmallVector<Register, 8> ArgRegs;
- for (const auto &Arg : Info.OrigArgs) {
+ for (const ArgInfo &Arg : Info.OrigArgs) {
// Currently call args should have single vregs.
if (Arg.Regs.size() > 1)
return false;
if (Arg.Ty->getScalarSizeInBits() == 1) {
const LLT I16 = LLT::integer(16);
- auto Reg = MIRBuilder.getMRI()->createGenericVirtualRegister(I16);
+ Register Reg = MIRBuilder.getMRI()->createGenericVirtualRegister(I16);
MIRBuilder.getMRI()->setRegClass(Reg, TRI->getRegClassFromLLT(I16));
- auto Sext = llvm::any_of(Arg.Flags,
- [](const auto &Flag) { return Flag.isSExt(); });
- auto Zext = llvm::any_of(Arg.Flags,
- [](const auto &Flag) { return Flag.isZExt(); });
+ bool Sext = llvm::any_of(Arg.Flags, [](const ISD::ArgFlagsTy &Flag) {
+ return Flag.isSExt();
+ });
+ bool Zext = llvm::any_of(Arg.Flags, [](const ISD::ArgFlagsTy &Flag) {
+ return Flag.isZExt();
+ });
if (Sext) {
MIRBuilder.buildSExt(Reg, Arg.Regs[0]);
} else if (Zext) {
@@ -578,10 +587,10 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
}
}
- auto MutateI1 = false;
+ bool MutateI1 = false;
if (!Info.OrigRet.Ty->isVoidTy()) {
// select the call op according to return type and build MI
- auto RetLLT =
+ LLT RetLLT =
llvm::getLLTForType(*Info.OrigRet.Ty, MIRBuilder.getDataLayout());
unsigned CallOp = 0;
if (RetLLT.isScalar() || RetLLT.isPointer()) {
@@ -612,8 +621,8 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
break;
}
} else if (RetLLT.isVector()) {
- auto TypeBitSize = RetLLT.getElementType().getSizeInBits();
- auto NumElts = RetLLT.getNumElements();
+ uint64_t TypeBitSize = RetLLT.getElementType().getSizeInBits();
+ uint16_t NumElts = RetLLT.getNumElements();
switch (TypeBitSize) {
case 8:
switch (NumElts) {
@@ -726,7 +735,7 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
// set Ret register class
assert(!Info.OrigRet.Regs.empty());
Register ResVReg = Info.OrigRet.Regs[0];
- auto OrigRetLLT = RetLLT;
+ LLT OrigRetLLT = RetLLT;
if (MutateI1) { // will return i16 (see lowerReturn())
RetLLT = LLT::integer(16);
ResVReg = MIRBuilder.getMRI()->createGenericVirtualRegister(RetLLT);
@@ -750,7 +759,7 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
}
// add function args into MI if any
- for (auto Reg : ArgRegs) {
+ for (Register Reg : ArgRegs) {
MIB.addUse(Reg);
}
diff --git a/llvm/lib/Target/PISA/PISAISelLowering.cpp b/llvm/lib/Target/PISA/PISAISelLowering.cpp
index 377303af97321..51f00605d3969 100644
--- a/llvm/lib/Target/PISA/PISAISelLowering.cpp
+++ b/llvm/lib/Target/PISA/PISAISelLowering.cpp
@@ -20,6 +20,7 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/PISAAddrSpace.h"
+#include <functional>
#include <type_traits>
#define DEBUG_TYPE "pisa-lower"
@@ -193,7 +194,7 @@ SyncScope::ID hoistedFenceScope(Instruction *Inst) {
SyncScope::ID SSID = getAtomicSyncScopeID(Inst).value_or(SyncScope::System);
// Shared memory has no meaningful system scope, so narrow the scope used for
// hoisted fences to the widest valid shared-memory scope.
- if (const auto *RMW = dyn_cast<AtomicRMWInst>(Inst))
+ if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst))
if (RMW->getPointerAddressSpace() == SharedAS && SSID == SyncScope::System)
return Inst->getContext().getOrInsertSyncScopeID("gpu-shared");
return SSID;
@@ -287,15 +288,16 @@ bool PISATargetLowering::isCheapToSpeculateCtlz(Type *Ty) const { return true; }
bool PISATargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
Register N0, Register N1) const {
- auto GetOneDefInst = [&MRI](Register N) -> MachineInstr * {
- auto *Def = MRI.getOneDef(N);
+ std::function<MachineInstr *(Register)> GetOneDefInst =
+ [&MRI](Register N) -> MachineInstr * {
+ MachineOperand *Def = MRI.getOneDef(N);
if (Def)
return Def->getParent();
return nullptr;
};
- auto *I0 = GetOneDefInst(N0);
- auto *I1 = GetOneDefInst(N1);
+ MachineInstr *I0 = GetOneDefInst(N0);
+ MachineInstr *I1 = GetOneDefInst(N1);
if (I0 && I1) {
// Prevent reassociating the following pattern
// (add (mul a, b), (add (mul c, d), e))
@@ -306,8 +308,8 @@ bool PISATargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
std::swap(I0, I1);
if (I0->getOpcode() == TargetOpcode::G_MUL &&
I1->getOpcode() == TargetOpcode::G_ADD) {
- auto *NI0 = GetOneDefInst(I1->getOperand(1).getReg());
- auto *NI1 = GetOneDefInst(I1->getOperand(2).getReg());
+ MachineInstr *NI0 = GetOneDefInst(I1->getOperand(1).getReg());
+ MachineInstr *NI1 = GetOneDefInst(I1->getOperand(2).getReg());
if (NI0 && NI1 &&
(NI0->getOpcode() == TargetOpcode::G_MUL ||
NI1->getOpcode() == TargetOpcode::G_MUL)) {
@@ -348,7 +350,7 @@ void PISATargetLowering::getTgtMemIntrinsic(
Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Info.flags |= getTargetMMOFlags(I);
// syncscope("<target-scope>") support for atomics
- if (auto *ConstInt = dyn_cast<ConstantInt>(I.getArgOperand(3)))
+ if (ConstantInt *ConstInt = dyn_cast<ConstantInt>(I.getArgOperand(3)))
Info.order = static_cast<llvm::AtomicOrdering>(ConstInt->getZExtValue());
Infos.push_back(Info);
return;
@@ -360,10 +362,10 @@ void PISATargetLowering::getTgtMemIntrinsic(
LLT PISATargetLowering::getOptimalMemOpLLT(
const MemOp &Op, const AttributeList &FuncAttributes) const {
- auto I8 = LLT::integer(8);
- auto I16 = LLT::integer(16);
- auto I32 = LLT::integer(32);
- auto I64 = LLT::integer(64);
+ LLT I8 = LLT::integer(8);
+ LLT I16 = LLT::integer(16);
+ LLT I32 = LLT::integer(32);
+ LLT I64 = LLT::integer(64);
if (Op.size() >= 16 && Op.isAligned(Align(8)))
return LLT::fixed_vector(2, I64);
@@ -438,8 +440,8 @@ PISATargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
break;
case 'r': {
// FIXME: we already have a method to get the register class from LLT
- auto VectorSize = VT.isVector() ? VT.getVectorNumElements() : 1;
- auto ElementSize = VT.getScalarSizeInBits();
+ unsigned VectorSize = VT.isVector() ? VT.getVectorNumElements() : 1;
+ unsigned ElementSize = VT.getScalarSizeInBits();
assert(VectorSize >= 1 &&
(ElementSize == 32 ? VectorSize <= 8 || VectorSize == 16 ||
VectorSize == 32 || VectorSize == 64
@@ -479,8 +481,8 @@ PISATargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
MachineMemOperand::Flags
PISATargetLowering::getTargetMMOFlags(const Instruction &I) const {
MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
- if (auto Hint = PISA::getCacheCtrlFromMMRA(I)) {
- auto HintValue = *Hint & 0xF;
+ if (std::optional<unsigned> Hint = PISA::getCacheCtrlFromMMRA(I)) {
+ unsigned HintValue = *Hint & 0xF;
Flags |= static_cast<MachineMemOperand::Flags>(HintValue << 6);
}
return Flags;
@@ -502,7 +504,7 @@ void PISATargetLowering::computeKnownBitsForTargetInstr(
if (Intrinsic::isOverloaded(IID))
return;
- auto *Ctx = &MI->getMF()->getFunction().getContext();
+ LLVMContext *Ctx = &MI->getMF()->getFunction().getContext();
FunctionType *FT = Intrinsic::getType(*Ctx, IID);
AttributeList Attrs = Intrinsic::getAttributes(*Ctx, IID, FT);
@@ -587,7 +589,7 @@ bool PISATargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
// Use AtomicExpand fence splitting only for RMWs that become CAS loops.
// Native atomics keep their ordering.
- if (const auto *RMW = dyn_cast<AtomicRMWInst>(I))
+ if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I))
return computeRMWExpansion(RMW) == AtomicExpansionKind::CmpXChg;
return false;
}
@@ -640,7 +642,8 @@ void PISATargetLowering::emitExpandAtomicRMW(AtomicRMWInst *A) const {
IR.SetInsertPoint(JoinBB->begin());
PHINode *Res = IR.CreatePHI(A->getType(), 2);
- auto EmitArm = [&](BasicBlock *BB, unsigned AS) {
+ std::function<void(BasicBlock *, unsigned)> EmitArm =
+ [&](BasicBlock *BB, unsigned AS) {
IR.SetInsertPoint(BB);
Value *Cast = IR.CreateAddrSpaceCast(A->getPointerOperand(),
PointerType::get(Ctx, AS));
diff --git a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
index 0597d75b67edf..ae72145211323 100644
--- a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
+++ b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
@@ -53,11 +53,11 @@ constexpr LLT F64 = LLT::float64();
// return true if natively supported type
static bool isLegalType(LLT Ty, bool Vector = true) {
- auto EltSize = Ty.getScalarSizeInBits();
+ unsigned EltSize = Ty.getScalarSizeInBits();
if (Ty.isVector() && !Vector)
return false;
if (Ty.isVector()) {
- auto NumElts = Ty.getNumElements();
+ uint16_t NumElts = Ty.getNumElements();
if (EltSize == 32)
return NumElts <= 8 || NumElts == 16 || NumElts == 32 || NumElts == 64;
if (!llvm::isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 64)
@@ -94,8 +94,8 @@ LegalityPredicate isFloatingPointType(unsigned TypeIdx) {
LegalizeMutation changeElementTypeToInteger(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
- auto NewEltTy = LLT::integer(Ty.getScalarSizeInBits());
- auto NewTy = Ty.isVector()
+ llvm::LLT NewEltTy = LLT::integer(Ty.getScalarSizeInBits());
+ llvm::LLT NewTy = Ty.isVector()
? LLT::fixed_vector(Ty.getNumElements(), NewEltTy)
: NewEltTy;
return std::pair(TypeIdx, NewTy);
@@ -125,8 +125,9 @@ static bool shouldWidenLoad(unsigned int Opcode, const LLT Ty,
PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
using namespace TargetOpcode;
- auto &TM = ST.getTargetLowering()->getTargetMachine();
- auto GetPointerLlt = [&](PISAAS::AddressSpace Addrspace) {
+ const llvm::TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
+ std::function<LLT(PISAAS::AddressSpace)> GetPointerLlt =
+ [&](PISAAS::AddressSpace Addrspace) {
uint32_t NumBits =
TM.getPointerSizeInBits(static_cast<unsigned>(Addrspace));
return LLT::pointer(static_cast<unsigned>(Addrspace), NumBits);
@@ -142,9 +143,10 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
GenericPtr};
const std::initializer_list<LLT> AddrSpaces32 = {PrivatePtr, SharedPtr};
- auto AllIntegers = {I8, I16, I32, I64};
- auto AllFloats = {BF16, F16, F32, F64};
- auto AllPtrs = {PrivatePtr, GlobalPtr, ConstantPtr, SharedPtr, GenericPtr};
+ std::initializer_list<llvm::LLT> AllIntegers = {I8, I16, I32, I64};
+ std::initializer_list<llvm::LLT> AllFloats = {BF16, F16, F32, F64};
+ std::initializer_list<llvm::LLT> AllPtrs = {
+ PrivatePtr, GlobalPtr, ConstantPtr, SharedPtr, GenericPtr};
getActionDefinitionsBuilder(
{G_FADD, G_FCONSTANT, G_FSUB, G_FMUL, G_FMINNUM, G_FMAXNUM, G_FMINIMUM,
@@ -198,7 +200,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
const LLT Ty = Query.Types[0];
if (!Ty.isVector())
return false;
- auto VecBitSize = Ty.getSizeInBits();
+ llvm::TypeSize VecBitSize = Ty.getSizeInBits();
return VecBitSize == 32 || VecBitSize == 16;
})),
LegalizeMutation(([=](const LegalityQuery &Query) {
@@ -249,10 +251,10 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
.legalFor(
{{I16, I8}, {I32, I8}, {I64, I8}, {I32, I16}, {I64, I16}, {I64, I32}})
.customIf([=](const LegalityQuery &Query) {
- auto DstSize = Query.Types[0].getScalarSizeInBits();
- auto SrcSize = Query.Types[1].getScalarSizeInBits();
- auto UseSelect = SrcSize == 1;
- auto UseShuffle = (SrcSize % 8 == 0 && !isPowerOf2_32(SrcSize)) ||
+ unsigned DstSize = Query.Types[0].getScalarSizeInBits();
+ unsigned SrcSize = Query.Types[1].getScalarSizeInBits();
+ bool UseSelect = SrcSize == 1;
+ bool UseShuffle = (SrcSize % 8 == 0 && !isPowerOf2_32(SrcSize)) ||
(DstSize % 8 == 0 && !isPowerOf2_32(DstSize));
return UseSelect || UseShuffle;
})
@@ -369,7 +371,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
.customIf([](const LegalityQuery &Query) {
- auto Ty = Query.Types[0];
+ llvm::LLT Ty = Query.Types[0];
return Ty.isVector() && (Ty.getScalarSizeInBits() == 32) &&
isPowerOf2_32(Ty.getNumElements());
})
@@ -395,7 +397,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
// scalars are widened directly.
//.widenScalar does not update MI.memoperands()[0].getType(), hence
.customIf([=](const LegalityQuery &Query) -> bool {
- auto Ty = Query.Types[0];
+ llvm::LLT Ty = Query.Types[0];
if (Ty.isVector() && (Ty.getScalarSizeInBits() > 1) &&
(Ty.getScalarSizeInBits() < 8))
return true;
@@ -405,10 +407,10 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
})
.fewerElementsIf(
[=](const LegalityQuery &Query) -> bool {
- auto EltTy = Query.Types[0];
- auto BitSize = EltTy.getScalarSizeInBits();
- auto NumElts = EltTy.isVector() ? EltTy.getNumElements() : 1;
- auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ llvm::LLT EltTy = Query.Types[0];
+ unsigned BitSize = EltTy.getScalarSizeInBits();
+ int NumElts = EltTy.isVector() ? EltTy.getNumElements() : 1;
+ uint64_t AlignInBits = Query.MMODescrs[0].AlignInBits;
// small (bitsize<32) vectors with non-power-of-2 elements
// can be broken into power-of-2 vectors that can be later
// upconverted to vectors of i32 for better codegen
@@ -416,55 +418,55 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
BitSize != 1 && (BitSize < 32) && (BitSize < AlignInBits);
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
- auto EltTy = Query.Types[0];
- auto NumElts = EltTy.getNumElements();
- auto NewNumElts = PowerOf2Ceil(NumElts) / 2;
+ llvm::LLT EltTy = Query.Types[0];
+ uint16_t NumElts = EltTy.getNumElements();
+ uint64_t NewNumElts = PowerOf2Ceil(NumElts) / 2;
return std::make_pair(
0, LLT::fixed_vector(NewNumElts, EltTy.getScalarType()));
})
// split up vectors of non-standard size elements
.fewerElementsIf(
[=](const LegalityQuery &Query) -> bool {
- auto EltTy = Query.Types[0];
+ llvm::LLT EltTy = Query.Types[0];
return EltTy.isVector() &&
!isPowerOf2_32(EltTy.getScalarSizeInBits());
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
- auto EltTy = Query.Types[0];
+ llvm::LLT EltTy = Query.Types[0];
return std::make_pair(0, EltTy.getScalarType());
})
// cast non-^2 scalars to vectors of i8
.bitcastIf(
[=](const LegalityQuery &Query) -> bool {
const LLT EltTy = Query.Types[0];
- auto NumBits = EltTy.getSizeInBits();
+ llvm::TypeSize NumBits = EltTy.getSizeInBits();
return !EltTy.isVector() && !isPowerOf2_32(NumBits);
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
- auto Size = Query.Types[0].getSizeInBits();
+ llvm::TypeSize Size = Query.Types[0].getSizeInBits();
return std::pair(0, LLT::fixed_vector(Size / 8, I8));
})
// cast scalar/vector with large bitsize into <? x i32>
.bitcastIf(
[=](const LegalityQuery &Query) -> bool {
const LLT EltTy = Query.Types[0];
- auto NumBits = EltTy.getScalarSizeInBits();
- auto IsAtomic128 =
+ unsigned NumBits = EltTy.getScalarSizeInBits();
+ bool IsAtomic128 =
EltTy.isScalar() && (NumBits == 128) &&
isStrongerThanMonotonic(Query.MMODescrs[0].Ordering);
return !IsAtomic128 && (NumBits % 32 == 0) && (NumBits > 64);
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
const LLT EltTy = Query.Types[0];
- auto NumBits = EltTy.getSizeInBits();
+ llvm::TypeSize NumBits = EltTy.getSizeInBits();
return std::pair(0, LLT::fixed_vector(NumBits / 32, I32));
})
.bitcastIf(([=](const LegalityQuery &Query) -> bool {
- auto EltTy = Query.Types[0];
- auto BitSize = EltTy.getScalarSizeInBits();
- auto AccSize = EltTy.getSizeInBits();
- auto AlignInBits = Query.MMODescrs[0].AlignInBits;
- auto SmallVectorWithManyElements =
+ llvm::LLT EltTy = Query.Types[0];
+ unsigned BitSize = EltTy.getScalarSizeInBits();
+ llvm::TypeSize AccSize = EltTy.getSizeInBits();
+ uint64_t AlignInBits = Query.MMODescrs[0].AlignInBits;
+ bool SmallVectorWithManyElements =
(BitSize < 32) && EltTy.isVector() &&
(EltTy.getNumElements() > 4);
@@ -482,54 +484,54 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
((BitSize < 32) && (AlignInBits < AccSize));
}),
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
- auto EltTy = Query.Types[0];
- auto BitSize = EltTy.getScalarSizeInBits();
- auto AccSize = EltTy.getSizeInBits();
- auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ llvm::LLT EltTy = Query.Types[0];
+ unsigned BitSize = EltTy.getScalarSizeInBits();
+ llvm::TypeSize AccSize = EltTy.getSizeInBits();
+ uint64_t AlignInBits = Query.MMODescrs[0].AlignInBits;
if ((BitSize < 32) && (AccSize % 32 == 0) &&
(AlignInBits % 32 == 0))
AlignInBits = 32;
- auto NewEltTy = LLT::integer(AlignInBits);
- auto NewNumElts = AccSize / AlignInBits;
- auto NewTy = NewNumElts == 1
+ llvm::LLT NewEltTy = LLT::integer(AlignInBits);
+ uint64_t NewNumElts = AccSize / AlignInBits;
+ llvm::LLT NewTy = NewNumElts == 1
? NewEltTy
: LLT::fixed_vector(NewNumElts, NewEltTy);
return std::pair(0, NewTy);
})
// bitcast <6 x i32> to <3 x i64> if alignment is sufficient
.bitcastIf(([=](const LegalityQuery &Query) -> bool {
- auto EltTy = Query.Types[0];
+ llvm::LLT EltTy = Query.Types[0];
return EltTy.isVector() &&
(EltTy.getScalarSizeInBits() == 32) &&
(EltTy.getNumElements() == 6) &&
(Query.MMODescrs[0].AlignInBits >= 64);
}),
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
- auto NewTy = LLT::fixed_vector(3, LLT::integer(64));
+ llvm::LLT NewTy = LLT::fixed_vector(3, LLT::integer(64));
return std::pair(0, NewTy);
})
// Increase the number of elements to corresponding vector of i8
.customIf([=](const LegalityQuery &Query) {
- auto EltTy = Query.Types[0];
+ llvm::LLT EltTy = Query.Types[0];
return EltTy.getScalarSizeInBits() == 1;
})
// expand s32 vectors with 4 < elts < 8 to have 8 elements
// Enabled only for shared memory where loads of OOB accesses are
// guaranteed to return 0
.customIf([=](const LegalityQuery &Query) {
- auto EltTy = Query.Types[0];
- auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ llvm::LLT EltTy = Query.Types[0];
+ uint64_t AlignInBits = Query.MMODescrs[0].AlignInBits;
return shouldWidenLoad(Query.Opcode, EltTy,
Query.Types[1].getAddressSpace(), AlignInBits);
})
// <4 x i8> align 1 .. needs to be broken down into 4 loads
.scalarizeIf(([=](const LegalityQuery &Query) -> bool {
- auto EltTy = Query.Types[0];
- auto BitSize = EltTy.getScalarSizeInBits();
- auto AccSize = EltTy.getSizeInBits();
- auto AlignInBits = Query.MMODescrs[0].AlignInBits;
+ llvm::LLT EltTy = Query.Types[0];
+ unsigned BitSize = EltTy.getScalarSizeInBits();
+ llvm::TypeSize AccSize = EltTy.getSizeInBits();
+ uint64_t AlignInBits = Query.MMODescrs[0].AlignInBits;
return ((BitSize < 32) && (AlignInBits < AccSize));
}),
0)
@@ -551,7 +553,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
const LLT EltTy = Query.Types[0];
if (!EltTy.isVector())
return false;
- auto NumElements = EltTy.getNumElements();
+ uint16_t NumElements = EltTy.getNumElements();
if (!isPowerOf2_32(NumElements))
return !((NumElements == 3) &&
((EltTy.getScalarSizeInBits() == 32) ||
@@ -560,7 +562,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
const LLT EltTy = Query.Types[0];
- auto NewNumElts = PowerOf2Ceil(EltTy.getNumElements()) / 2;
+ uint64_t NewNumElts = PowerOf2Ceil(EltTy.getNumElements()) / 2;
return std::pair(
0, LLT::fixed_vector(NewNumElts, EltTy.getScalarType()));
})
@@ -635,8 +637,8 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
getActionDefinitionsBuilder(G_BITCAST)
// allow bitcasts between pointers and non-pointers (ptr2int/int2ptr)
.customIf([=](const LegalityQuery &Query) {
- auto DstTy = Query.Types[0];
- auto SrcTy = Query.Types[1];
+ llvm::LLT DstTy = Query.Types[0];
+ llvm::LLT SrcTy = Query.Types[1];
return (DstTy.isPointer() != SrcTy.isPointer());
})
// In cases where both source and destination operands are vectors,
@@ -644,8 +646,8 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
// divisible by each other, e.g. <4 x i32> to <8 x i16>; use custom
// legalization to handle other cases, e.g. <5 x i32> to <2 x i80>
.customIf([=](const LegalityQuery &Query) {
- auto DstTy = Query.Types[0];
- auto SrcTy = Query.Types[1];
+ llvm::LLT DstTy = Query.Types[0];
+ llvm::LLT SrcTy = Query.Types[1];
if (!SrcTy.isVector() || !DstTy.isVector())
return false;
unsigned SrcNumElts = SrcTy.getNumElements();
@@ -658,14 +660,14 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
// avoid creating illegal G_UNMERGE_VALUES on odd-sized vectors
// downstream.
.customIf([=](const LegalityQuery &Query) {
- auto DstTy = Query.Types[0];
- auto SrcTy = Query.Types[1];
+ llvm::LLT DstTy = Query.Types[0];
+ llvm::LLT SrcTy = Query.Types[1];
if (!SrcTy.isVector() || !DstTy.isVector())
return false;
- auto DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
- auto SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
- auto DstEltSize = DstTy.getScalarSizeInBits();
- auto SrcEltSize = SrcTy.getScalarSizeInBits();
+ int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
+ int SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
+ unsigned DstEltSize = DstTy.getScalarSizeInBits();
+ unsigned SrcEltSize = SrcTy.getScalarSizeInBits();
if (DstNumElts != SrcNumElts)
return false;
if (DstEltSize != SrcEltSize)
@@ -674,13 +676,13 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
!isPowerOf2_32(DstTy.getSizeInBits());
})
.legalIf([=](const LegalityQuery &Query) {
- auto DstTy = Query.Types[0];
- auto SrcTy = Query.Types[1];
- auto DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
- auto SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
- auto DstEltSize = DstTy.getScalarSizeInBits();
- auto SrcEltSize = SrcTy.getScalarSizeInBits();
- auto CastSize = DstTy.getSizeInBits();
+ llvm::LLT DstTy = Query.Types[0];
+ llvm::LLT SrcTy = Query.Types[1];
+ int DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1;
+ int SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
+ unsigned DstEltSize = DstTy.getScalarSizeInBits();
+ unsigned SrcEltSize = SrcTy.getScalarSizeInBits();
+ llvm::TypeSize CastSize = DstTy.getSizeInBits();
if (DstEltSize == 32 && SrcEltSize == 32)
// vectors of 32bit integer <=> floats
return DstNumElts <= 8 || DstNumElts == 16 || DstNumElts == 32 ||
@@ -715,13 +717,13 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
// extend vectors of i1 to have power of two elements
.moreElementsIf(
([=](const LegalityQuery &Query) {
- auto DstTy = Query.Types[SrcTyIdx];
- auto BitSize = DstTy.getSizeInBits();
+ llvm::LLT DstTy = Query.Types[SrcTyIdx];
+ llvm::TypeSize BitSize = DstTy.getSizeInBits();
return DstTy.isVector() && (DstTy.getScalarSizeInBits() == 1) &&
((BitSize < 8) || !isPowerOf2_32(BitSize));
}),
[=](const LegalityQuery &Query) {
- auto DstTy = Query.Types[SrcTyIdx];
+ llvm::LLT DstTy = Query.Types[SrcTyIdx];
unsigned NumElts = PowerOf2Ceil(DstTy.getNumElements());
NumElts = std::max(8u, NumElts);
return std::pair(
@@ -729,30 +731,30 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
})
// <? x i1>
.customIf([=](const LegalityQuery &Query) {
- auto EltSize = Query.Types[SrcTyIdx].getScalarSizeInBits();
+ unsigned EltSize = Query.Types[SrcTyIdx].getScalarSizeInBits();
return (EltSize == 1);
})
// increase to a multiple of elements, e.g. <5 x i16> => <8 x i16>
.moreElementsIf(
[=](const LegalityQuery &Query) {
- auto SrcTy = Query.Types[SrcTyIdx];
+ llvm::LLT SrcTy = Query.Types[SrcTyIdx];
unsigned NumElts = SrcTy.getNumElements();
if (SrcTy.getScalarSizeInBits() != 32)
return (NumElts > 4) && (NumElts % 4);
return (NumElts > 8) && (NumElts != 16) && (NumElts % 32);
},
[=](const LegalityQuery &Query) {
- auto SrcTy = Query.Types[SrcTyIdx];
- auto ScalarTy = SrcTy.getScalarType();
- auto NumElts = PowerOf2Ceil(SrcTy.getNumElements());
+ llvm::LLT SrcTy = Query.Types[SrcTyIdx];
+ llvm::LLT ScalarTy = SrcTy.getScalarType();
+ uint64_t NumElts = PowerOf2Ceil(SrcTy.getNumElements());
return std::pair(SrcTyIdx, LLT::fixed_vector(NumElts, ScalarTy));
})
// cast non-s32 elements to s32 vector, e.g.
// <N x s8> => <N/4 x s32>, iff the index is non-constant
.customIf([=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[SrcTyIdx];
- const auto EltSize = Ty.getScalarSizeInBits();
- const auto NumElts = Ty.getNumElements();
+ const unsigned EltSize = Ty.getScalarSizeInBits();
+ const uint16_t NumElts = Ty.getNumElements();
// Only needed for non-s32 elements
if (EltSize != 8 && EltSize != 16 && EltSize != 64)
return false;
@@ -767,13 +769,13 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
// - operation will use 'insert/extract' or swizzle
.fewerElementsIf(
[=](const LegalityQuery &Query) {
- auto SrcTy = Query.Types[SrcTyIdx];
- auto MaxElts = SrcTy.getScalarSizeInBits() == 32 ? 64 : 4;
+ llvm::LLT SrcTy = Query.Types[SrcTyIdx];
+ int MaxElts = SrcTy.getScalarSizeInBits() == 32 ? 64 : 4;
return SrcTy.getNumElements() > MaxElts;
},
[=](const LegalityQuery &Query) {
- auto SrcTy = Query.Types[SrcTyIdx];
- auto ScalarTy = SrcTy.getScalarType();
+ llvm::LLT SrcTy = Query.Types[SrcTyIdx];
+ llvm::LLT ScalarTy = SrcTy.getScalarType();
return (SrcTy.getScalarSizeInBits() == 32)
? std::pair(SrcTyIdx, LLT::fixed_vector(64, ScalarTy))
: std::pair(SrcTyIdx, LLT::fixed_vector(4, ScalarTy));
@@ -824,10 +826,10 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
})
.customIf([=](const LegalityQuery &Query) {
// return true if we want to use 'insert', instead of swizzle
- auto LitTy = Query.Types[1];
- auto BigTy = Query.Types[0];
- auto EltOk = BigTy.getScalarSizeInBits() == 32;
- auto VecOk = LitTy.isVector() && BigTy.isVector();
+ llvm::LLT LitTy = Query.Types[1];
+ llvm::LLT BigTy = Query.Types[0];
+ bool EltOk = BigTy.getScalarSizeInBits() == 32;
+ bool VecOk = LitTy.isVector() && BigTy.isVector();
return EltOk && VecOk &&
((LitTy.getNumElements() > 4) || (BigTy.getNumElements() > 4));
})
@@ -843,14 +845,14 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
// fewerElementsVectorReductions does not handle (SrcElts % DstElts != 0)
.moreElementsIf(([=](const LegalityQuery &Query) {
- auto NumElts = Query.Types[1].getNumElements();
+ uint16_t NumElts = Query.Types[1].getNumElements();
return NumElts > 4 && NumElts % 4 != 0;
}),
[=](const LegalityQuery &Query) {
- auto SrcTy = Query.Types[1];
- auto NewNumElts =
+ llvm::LLT SrcTy = Query.Types[1];
+ uint64_t NewNumElts =
llvm::PowerOf2Ceil(SrcTy.getNumElements());
- auto NewSrcTy = LLT::fixed_vector(
+ llvm::LLT NewSrcTy = LLT::fixed_vector(
NewNumElts, SrcTy.getScalarType());
return std::pair(1, NewSrcTy);
})
@@ -863,10 +865,10 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
unsigned BigTyIdx = Op == G_UNMERGE_VALUES ? 1 : 0;
unsigned LitTyIdx = Op == G_UNMERGE_VALUES ? 0 : 1;
- auto &Builder = getActionDefinitionsBuilder(Op);
+ llvm::LegalizeRuleSet &Builder = getActionDefinitionsBuilder(Op);
Builder.customIf([=](const LegalityQuery &Query) {
// return true if we want to use 'extract', instead of swizzle
- auto BigTy = Query.Types[BigTyIdx];
+ llvm::LLT BigTy = Query.Types[BigTyIdx];
return BigTy.isVector() && (BigTy.getScalarSizeInBits() == 32) &&
(BigTy.getNumElements() > 4);
});
@@ -874,9 +876,9 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
Builder
.legalIf([=](const LegalityQuery &Query) {
// vector(big) <=> scalar/vector(lit)
- auto BigTy = Query.Types[BigTyIdx];
- auto LitTy = Query.Types[LitTyIdx];
- auto LitTyValid = LitTy.isScalar() ? BigTy.getScalarType() == LitTy
+ llvm::LLT BigTy = Query.Types[BigTyIdx];
+ llvm::LLT LitTy = Query.Types[LitTyIdx];
+ bool LitTyValid = LitTy.isScalar() ? BigTy.getScalarType() == LitTy
: LitTy.getScalarSizeInBits() >= 8;
// No register class exists for vectors with elements wider than
// 64 bits, so <N x i128> and friends must not be marked legal here
@@ -889,13 +891,13 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
})
.widenScalarIf(
[=](const LegalityQuery &Query) {
- auto BigTy = Query.Types[BigTyIdx];
- auto BigTySize = BigTy.getSizeInBits();
+ llvm::LLT BigTy = Query.Types[BigTyIdx];
+ llvm::TypeSize BigTySize = BigTy.getSizeInBits();
return BigTy.isScalar() && BigTySize > 64 &&
!isPowerOf2_32(BigTySize);
},
[=](const LegalityQuery &Query) {
- auto BigTy = Query.Types[BigTyIdx];
+ llvm::LLT BigTy = Query.Types[BigTyIdx];
unsigned NewSizeInBits =
1 << Log2_32_Ceil(BigTy.getSizeInBits() + 1);
return std::pair(BigTyIdx, LLT::integer(NewSizeInBits));
@@ -903,9 +905,9 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
.lowerIf([=](const LegalityQuery &Query) {
// lower to shift/mask if conversion would
// result in a vector with >4 elements
- auto BigTy = Query.Types[BigTyIdx];
- auto LitTy = Query.Types[LitTyIdx];
- auto NumElts = BigTy.getSizeInBits() / LitTy.getScalarSizeInBits();
+ llvm::LLT BigTy = Query.Types[BigTyIdx];
+ llvm::LLT LitTy = Query.Types[LitTyIdx];
+ uint64_t NumElts = BigTy.getSizeInBits() / LitTy.getScalarSizeInBits();
return BigTy.isScalar() && (NumElts > 4);
})
.lowerIf(all(vectorElementCountIsGreaterThan(LitTyIdx, 4),
@@ -1150,10 +1152,10 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
static SmallVector<MachineInstr *> scalarizeIntrinsic(MachineInstr &MI) {
Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
MachineIRBuilder B(MI);
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
SmallVector<MachineInstr *> NewMIs;
- auto DstTy = MRI.getType(MI.getOperand(0).getReg());
+ llvm::LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
if (!DstTy.isVector()) {
NewMIs.push_back(&MI);
return NewMIs;
@@ -1163,12 +1165,12 @@ static SmallVector<MachineInstr *> scalarizeIntrinsic(MachineInstr &MI) {
for (unsigned I = 0; I < DstTy.getNumElements(); I++) {
SmallVector<MachineOperand, 4> Opnds;
for (unsigned J = 2; J < MI.getNumOperands(); J++) { // dst, iid
- auto Opnd = MI.getOperand(J);
+ llvm::MachineOperand Opnd = MI.getOperand(J);
if (Opnd.isReg()) {
- auto ArgTy = MRI.getType(Opnd.getReg());
+ llvm::LLT ArgTy = MRI.getType(Opnd.getReg());
if (ArgTy.isVector()) {
ArgTy = ArgTy.getScalarType();
- auto ArgReg = MRI.createGenericVirtualRegister(ArgTy);
+ llvm::Register ArgReg = MRI.createGenericVirtualRegister(ArgTy);
B.buildExtractVectorElementConstant(ArgReg, Opnd, I);
Opnds.push_back(MachineOperand::CreateReg(ArgReg, false));
} else { // use register operand as-is
@@ -1178,12 +1180,12 @@ static SmallVector<MachineInstr *> scalarizeIntrinsic(MachineInstr &MI) {
Opnds.push_back(Opnd);
}
}
- auto DstReg = MRI.createGenericVirtualRegister(DstTy.getScalarType());
- auto Res = B.buildIntrinsic(IntrinsicID, DstReg);
+ llvm::Register DstReg = MRI.createGenericVirtualRegister(DstTy.getScalarType());
+ llvm::MachineInstrBuilder Res = B.buildIntrinsic(IntrinsicID, DstReg);
NewMIs.push_back(Res);
Res.setMIFlags(MI.getFlags());
- for (auto It = Opnds.begin(), Ite = Opnds.end(); It != Ite; ++It)
- Res.add(*It);
+ for (MachineOperand &Opnd : Opnds)
+ Res.add(Opnd);
VecRegs.push_back(DstReg);
}
B.buildBuildVector(MI.getOperand(0), VecRegs);
@@ -1199,14 +1201,14 @@ static bool legalizeGFlog(MachineInstr &MI, MachineIRBuilder &B,
LLT Ty = B.getMRI()->getType(Dst);
unsigned Flags = MI.getFlags();
- auto &Semantics = getFltSemanticForLLT(Ty.getScalarType());
+ const llvm::fltSemantics &Semantics = getFltSemanticForLLT(Ty.getScalarType());
APFloat APFLog2BaseInverted(Log2BaseInverted);
bool LosesInfo; // ignored
APFLog2BaseInverted.convert(Semantics, APFloat::rmNearestTiesToEven,
&LosesInfo);
- auto Log2Operand = B.buildFLog2(Ty, Src, Flags);
- auto Log2BaseInvertedOperand = B.buildFConstant(Ty, APFLog2BaseInverted);
+ llvm::MachineInstrBuilder Log2Operand = B.buildFLog2(Ty, Src, Flags);
+ llvm::MachineInstrBuilder Log2BaseInvertedOperand = B.buildFConstant(Ty, APFLog2BaseInverted);
B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
MI.eraseFromParent();
@@ -1221,13 +1223,13 @@ static bool legalizeGFexp(MachineInstr &MI, MachineIRBuilder &B,
unsigned Flags = MI.getFlags();
LLT Ty = B.getMRI()->getType(Dst);
- auto &Semantics = getFltSemanticForLLT(Ty.getScalarType());
+ const llvm::fltSemantics &Semantics = getFltSemanticForLLT(Ty.getScalarType());
APFloat APFMultiplicand(Multiplicand);
bool LosesInfo; // ignored
APFMultiplicand.convert(Semantics, APFloat::rmNearestTiesToEven, &LosesInfo);
- auto K = B.buildFConstant(Ty, APFMultiplicand);
- auto Mul = B.buildFMul(Ty, Src, K, Flags);
+ llvm::MachineInstrBuilder K = B.buildFConstant(Ty, APFMultiplicand);
+ llvm::MachineInstrBuilder Mul = B.buildFMul(Ty, Src, K, Flags);
B.buildFExp2(Dst, Mul, Flags);
MI.eraseFromParent();
return true;
@@ -1238,7 +1240,7 @@ static bool legalizeGFexp(MachineInstr &MI, MachineIRBuilder &B,
// if and when it is available. For now, we custom legalize it based upon the
// approach in TargetLowering::LegalizeSetCCCondCode().
static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
- auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
+ llvm::CmpInst::Predicate Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
Register Dst = MI.getOperand(0).getReg();
Register Op0 = MI.getOperand(2).getReg();
Register Op1 = MI.getOperand(3).getReg();
@@ -1258,11 +1260,11 @@ static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
// G_ANYEXT the G_FCMP compare result to i16. Given that a .reg destination
// for fcmp is only available for 32-bit, we explicitly extend it here
// so we can fold the resulting select into the fcmp.
- auto LHS =
+ llvm::MachineInstrBuilder LHS =
B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OGT, I1, Op0, Op1, Flags));
- auto RHS =
+ llvm::MachineInstrBuilder RHS =
B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OLT, I1, Op0, Op1, Flags));
- auto Result = B.buildOr(I32, LHS, RHS);
+ llvm::MachineInstrBuilder Result = B.buildOr(I32, LHS, RHS);
if (Pred == CmpInst::FCMP_UEQ)
Result = B.buildNot(I32, Result);
B.buildICmp(CmpInst::ICMP_EQ, Dst, Result, B.buildConstant(I32, -1));
@@ -1270,11 +1272,11 @@ static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
break;
}
case CmpInst::FCMP_ORD: {
- auto LHS =
+ llvm::MachineInstrBuilder LHS =
B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OEQ, I1, Op0, Op0, Flags));
- auto RHS =
+ llvm::MachineInstrBuilder RHS =
B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_OEQ, I1, Op1, Op1, Flags));
- auto Result = B.buildAnd(I32, LHS, RHS);
+ llvm::MachineInstrBuilder Result = B.buildAnd(I32, LHS, RHS);
B.buildICmp(CmpInst::ICMP_EQ, Dst, Result, B.buildConstant(I32, -1));
MI.eraseFromParent();
break;
@@ -1284,23 +1286,25 @@ static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
// fcmp.uno with a non-NaN constant (usually zero). In that case, we don't
// need to generate two fcmps because only the non-const parameter is
// relevant to this comparison
- auto Op0Cst = getFConstantVRegValWithLookThrough(Op0, *B.getMRI());
+ std::optional<llvm::FPValueAndVReg> Op0Cst =
+ getFConstantVRegValWithLookThrough(Op0, *B.getMRI());
bool Op0IsOrdConstant = Op0Cst && !Op0Cst.value().Value.isNaN();
- auto Op1Cst = getFConstantVRegValWithLookThrough(Op1, *B.getMRI());
+ std::optional<llvm::FPValueAndVReg> Op1Cst =
+ getFConstantVRegValWithLookThrough(Op1, *B.getMRI());
bool Op1IsOrdConstant = Op1Cst && !Op1Cst.value().Value.isNaN();
if (Op0IsOrdConstant || Op1IsOrdConstant) {
- auto Reg = Op1IsOrdConstant ? Op0 : Op1;
+ llvm::Register Reg = Op1IsOrdConstant ? Op0 : Op1;
B.buildFCmp(CmpInst::FCMP_UNE, Dst, Reg, Reg, Flags);
} else {
// If the operands are both non-constant, we need to split this into two
// fcmps to ensure it returns false if they are unequal
- auto LHS =
+ llvm::MachineInstrBuilder LHS =
B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_UNE, I1, Op0, Op0, Flags));
- auto RHS =
+ llvm::MachineInstrBuilder RHS =
B.buildSExt(I32, B.buildFCmp(CmpInst::FCMP_UNE, I1, Op1, Op1, Flags));
- auto Result = B.buildOr(I32, LHS, RHS);
+ llvm::MachineInstrBuilder Result = B.buildOr(I32, LHS, RHS);
B.buildICmp(CmpInst::ICMP_EQ, Dst, Result, B.buildConstant(I32, -1));
}
MI.eraseFromParent();
@@ -1310,9 +1314,10 @@ static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
case CmpInst::FCMP_UGE:
case CmpInst::FCMP_ULT:
case CmpInst::FCMP_ULE: {
- auto Cmp = B.buildSExt(I32, B.buildFCmp(FCmpInst::getInversePredicate(Pred),
- I1, Op0, Op1, Flags));
- auto Not = B.buildNot(I32, Cmp);
+ llvm::MachineInstrBuilder Cmp = B.buildSExt(
+ I32, B.buildFCmp(FCmpInst::getInversePredicate(Pred), I1, Op0, Op1,
+ Flags));
+ llvm::MachineInstrBuilder Not = B.buildNot(I32, Cmp);
B.buildICmp(CmpInst::ICMP_EQ, Dst, Not, B.buildConstant(I32, -1));
MI.eraseFromParent();
break;
@@ -1324,20 +1329,22 @@ static bool legalizeGFcmp(MachineInstr &MI, MachineIRBuilder &B) {
}
static bool legalizeGTrunc(MachineInstr &MI, MachineIRBuilder &B) {
- [[maybe_unused]] auto &MRI = *B.getMRI();
- auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ [[maybe_unused]] llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ Register Dst, Src;
+ LLT DstTy, SrcTy;
+ std::tie(Dst, DstTy, Src, SrcTy) = MI.getFirst2RegLLTs();
if (DstTy.getSizeInBits() == 1) {
// truncate ??? to i1
// Since PISA does not support truncs to i1 (i8 is the minimum), we must
// turn it into an i1 by using an icmp instruction.
- auto Zero = B.buildConstant(SrcTy, 0);
- auto One = B.buildConstant(SrcTy, 1);
- auto And = B.buildAnd(SrcTy, Src, One);
+ llvm::MachineInstrBuilder Zero = B.buildConstant(SrcTy, 0);
+ llvm::MachineInstrBuilder One = B.buildConstant(SrcTy, 1);
+ llvm::MachineInstrBuilder And = B.buildAnd(SrcTy, Src, One);
B.buildICmp(CmpInst::ICMP_NE, Dst, And, Zero);
} else {
// truncate i128 to ???
assert(SrcTy.getSizeInBits() == 128);
- auto Unmerge = B.buildUnmerge(I64, Src);
+ llvm::MachineInstrBuilder Unmerge = B.buildUnmerge(I64, Src);
if (DstTy.getSizeInBits() == 64)
B.buildCopy(Dst, Unmerge.getReg(0));
else
@@ -1348,25 +1355,27 @@ static bool legalizeGTrunc(MachineInstr &MI, MachineIRBuilder &B) {
}
static bool legalizeGExt(MachineInstr &MI, MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
- auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ Register Dst, Src;
+ LLT DstTy, SrcTy;
+ std::tie(Dst, DstTy, Src, SrcTy) = MI.getFirst2RegLLTs();
if (MRI.getType(Src).getSizeInBits() == 1) {
// i8 = G_*EXT i1
- auto Zero = B.buildConstant(DstTy, 0);
+ llvm::MachineInstrBuilder Zero = B.buildConstant(DstTy, 0);
int64_t ExtendedVal = (MI.getOpcode() == TargetOpcode::G_SEXT) ||
(MI.getOpcode() == TargetOpcode::G_ANYEXT)
? -1
: 1;
- auto One = B.buildConstant(DstTy, ExtendedVal);
+ llvm::MachineInstrBuilder One = B.buildConstant(DstTy, ExtendedVal);
B.buildSelect(Dst, Src, One, Zero);
} else {
// any G_*EXT where source and destination are byte size
- auto DstSize = DstTy.getScalarSizeInBits();
- auto SrcSize = SrcTy.getScalarSizeInBits();
+ unsigned DstSize = DstTy.getScalarSizeInBits();
+ unsigned SrcSize = SrcTy.getScalarSizeInBits();
assert((DstSize % 8 == 0) && "destination size is not byte size");
assert((SrcSize % 8 == 0) && "source size is not byte size");
- auto EltSize =
+ int EltSize =
((DstSize % 32 == 0) && (SrcSize % 32 == 0))
? 32
: (((DstSize % 16 == 0) && (SrcSize % 16 == 0)) ? 16 : 8);
@@ -1375,7 +1384,7 @@ static bool legalizeGExt(MachineInstr &MI, MachineIRBuilder &B) {
LLT EltTy = LLT::integer(EltSize);
LLT VecDstTy = LLT::fixed_vector(NumDstElts, EltTy);
- auto VecZero = MRI.createGenericVirtualRegister(VecDstTy);
+ llvm::Register VecZero = MRI.createGenericVirtualRegister(VecDstTy);
SmallVector<APInt> Zeros(NumDstElts, APInt(EltSize, 0));
B.buildBuildVectorConstant(VecZero, Zeros);
@@ -1396,13 +1405,13 @@ static bool legalizeGExt(MachineInstr &MI, MachineIRBuilder &B) {
: MRI.getType(VecSrc).getNumElements());
}
- auto VecDst = MRI.createGenericVirtualRegister(VecDstTy);
+ llvm::Register VecDst = MRI.createGenericVirtualRegister(VecDstTy);
B.buildShuffleVector(VecDst, VecSrc, VecZero, Mask);
if (MI.getOpcode() == TargetOpcode::G_SEXT) {
- auto CastReg = MRI.createGenericVirtualRegister(DstTy);
- auto ShiftReg = MRI.createGenericVirtualRegister(DstTy);
- auto ShiftAmt = B.buildConstant(I32, DstSize - SrcSize);
+ llvm::Register CastReg = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ShiftReg = MRI.createGenericVirtualRegister(DstTy);
+ llvm::MachineInstrBuilder ShiftAmt = B.buildConstant(I32, DstSize - SrcSize);
B.buildBitcast(CastReg, VecDst);
B.buildShl(ShiftReg, CastReg, ShiftAmt);
B.buildAShr(Dst, ShiftReg, ShiftAmt);
@@ -1415,7 +1424,9 @@ static bool legalizeGExt(MachineInstr &MI, MachineIRBuilder &B) {
}
static bool legalizeGItofp(MachineInstr &MI, MachineIRBuilder &B) {
- auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ Register Dst, Src;
+ LLT DstTy, SrcTy;
+ std::tie(Dst, DstTy, Src, SrcTy) = MI.getFirst2RegLLTs();
assert(SrcTy.isScalar() && SrcTy.getSizeInBits() == 1 &&
"Unexpected source type");
assert(DstTy.isScalar() && DstTy.getSizeInBits() == 16 &&
@@ -1428,12 +1439,12 @@ static bool legalizeGItofp(MachineInstr &MI, MachineIRBuilder &B) {
const fltSemantics &Semantics =
DstTy == LLT::bfloat16() ? APFloat::BFloat() : APFloat::IEEEhalf();
- auto TrueVal =
+ llvm::APFloat TrueVal =
APFloat::getOne(Semantics, /*Negative=*/Opc == TargetOpcode::G_SITOFP);
- auto FalseVal = APFloat::getZero(Semantics);
+ llvm::APFloat FalseVal = APFloat::getZero(Semantics);
- auto True = B.buildFConstant(DstTy, TrueVal);
- auto False = B.buildFConstant(DstTy, FalseVal);
+ llvm::MachineInstrBuilder True = B.buildFConstant(DstTy, TrueVal);
+ llvm::MachineInstrBuilder False = B.buildFConstant(DstTy, FalseVal);
B.buildSelect(Dst, Src, True, False);
MI.eraseFromParent();
return true;
@@ -1442,21 +1453,21 @@ static bool legalizeGItofp(MachineInstr &MI, MachineIRBuilder &B) {
static void updateRegInDebugValue(Register OriginalVal, Register NewVal,
MachineRegisterInfo &MRI) {
llvm::SmallVector<MachineOperand *, 5> Opnds;
- for (auto &Instr : MRI.use_instructions(OriginalVal)) {
+ for (llvm::MachineInstr &Instr : MRI.use_instructions(OriginalVal)) {
if (!Instr.isDebugValue())
continue;
- for (auto &Opnd : Instr.operands()) {
+ for (llvm::MachineOperand &Opnd : Instr.operands()) {
if (Opnd.isReg() && Opnd.getReg() == OriginalVal)
Opnds.push_back(&Opnd);
}
}
- for (auto *Opnd : Opnds)
+ for (llvm::MachineOperand *Opnd : Opnds)
Opnd->setReg(NewVal);
return;
}
static bool legalizeGExtload(MachineInstr &MI, MachineIRBuilder &B) {
- auto &LoadMI = cast<GExtLoad>(MI);
+ llvm::GExtLoad &LoadMI = cast<GExtLoad>(MI);
Register DstReg = LoadMI.getDstReg();
Register PtrReg = LoadMI.getPointerReg();
LLT MemTy = LoadMI.getMMO().getMemoryType();
@@ -1469,7 +1480,7 @@ static bool legalizeGExtload(MachineInstr &MI, MachineIRBuilder &B) {
// Narrow load + extension: G_{S,Z}EXTLOAD(DstTy, ptr) ->
// %narrow = G_LOAD MemTy, ptr
// DstReg = G_{S,Z}EXT DstTy, %narrow
- auto NarrowLoad = B.buildLoad(MemTy, PtrReg, LoadMI.getMMO());
+ llvm::MachineInstrBuilder NarrowLoad = B.buildLoad(MemTy, PtrReg, LoadMI.getMMO());
if (isa<GSExtLoad>(MI))
B.buildSExt(DstReg, NarrowLoad);
else
@@ -1480,19 +1491,19 @@ static bool legalizeGExtload(MachineInstr &MI, MachineIRBuilder &B) {
static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
LegalizerHelper &Helper) {
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
GISelChangeObserver &Observer = Helper.Observer;
- auto &ValMO = MI.getOperand(0);
+ llvm::MachineOperand &ValMO = MI.getOperand(0);
Register Val = ValMO.getReg();
MachineMemOperand &MMO = **MI.memoperands_begin();
unsigned AddressSpace = MMO.getAddrSpace();
LLT CurTy = MRI.getType(Val);
- auto CurTySize = CurTy.getSizeInBits();
+ llvm::TypeSize CurTySize = CurTy.getSizeInBits();
if (!CurTy.isVector() && ((CurTySize % 8) != 0)) {
// Widen sub-byte scalar load/store to multiple of 8 bits.
- auto NewSize = (CurTySize + 7) & ~7;
- auto NewTy = LLT::integer(NewSize);
+ uint64_t NewSize = (CurTySize + 7) & ~7;
+ llvm::LLT NewTy = LLT::integer(NewSize);
if (MI.getOpcode() == TargetOpcode::G_LOAD) {
// For loads: widen the load and truncate result.
Helper.widenScalar(MI, 0, NewTy);
@@ -1552,7 +1563,7 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
// e.g. <2 x i4> -> i8, <2 x i2> -> i4 -> i8, <65 x i2> -> i130 -> i136
unsigned ScalarSize = CurTySize;
unsigned NewSize = std::max(8u, ((ScalarSize + 7) & ~7u));
- auto NewTy = LLT::integer(NewSize);
+ llvm::LLT NewTy = LLT::integer(NewSize);
if (MI.getOpcode() == TargetOpcode::G_LOAD) {
Register NewVal = MRI.createGenericVirtualRegister(NewTy);
Observer.changingInstr(MI);
@@ -1563,7 +1574,7 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
if (ScalarSize == NewSize) {
B.buildBitcast(Val, NewVal);
} else {
- auto Trunc = B.buildTrunc(LLT::integer(ScalarSize), NewVal);
+ llvm::MachineInstrBuilder Trunc = B.buildTrunc(LLT::integer(ScalarSize), NewVal);
B.buildBitcast(Val, Trunc);
}
} else {
@@ -1605,13 +1616,13 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
Align AlignInBytes = MMO.getAlign();
Register OriginalVal = ValMO.getReg();
Register NewVal;
- auto NumElts = CurTy.getNumElements();
+ uint16_t NumElts = CurTy.getNumElements();
// If alignment is at least 8 bytes and number of elements is 5 or 6,
// widen to 3 elements of i64. Otherwise, widen to 8 elements of i32.
bool CanWidenToI64 = (AlignInBytes.value() >= 8) && (NumElts <= 6);
- auto NewTy = CanWidenToI64 ? LLT::fixed_vector(3, LLT::integer(64))
+ llvm::LLT NewTy = CanWidenToI64 ? LLT::fixed_vector(3, LLT::integer(64))
: LLT::fixed_vector(8, LLT::integer(32));
- auto VecN32Ty = LLT::fixed_vector(NumElts, LLT::integer(32));
+ llvm::LLT VecN32Ty = LLT::fixed_vector(NumElts, LLT::integer(32));
// Create the new widened load/store
Observer.changingInstr(MI);
@@ -1627,8 +1638,8 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
if (CanWidenToI64) {
// Handle vectors with 5 or 6 elements of i32 with alignment >= 8 bytes
// Legalize to 3 elements of i64 for better hardware utilization
- auto V6i32Ty = LLT::fixed_vector(6, LLT::integer(32));
- auto ExtrVal = MRI.createGenericVirtualRegister(V6i32Ty);
+ llvm::LLT V6i32Ty = LLT::fixed_vector(6, LLT::integer(32));
+ llvm::Register ExtrVal = MRI.createGenericVirtualRegister(V6i32Ty);
// Bitcast to 6xi32 first, then extract the
// first 5 elements
B.buildBitcast(ExtrVal, NewVal);
@@ -1648,7 +1659,7 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
}
} else if (CurTy.getScalarType().isPointer()) {
// load/store of ptr requires inttoptr/ptrtoint
- auto EltSize = CurTy.getScalarSizeInBits();
+ unsigned EltSize = CurTy.getScalarSizeInBits();
LLT NewTy = CurTy.changeElementType(LLT::integer(EltSize));
Register NewVal = MRI.createGenericVirtualRegister(NewTy);
MMO.setType(NewTy);
@@ -1660,9 +1671,9 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
B.buildPtrToInt(NewVal, Val);
}
} else if (CurTy.getScalarSizeInBits() == 1) {
- auto BitSize = CurTy.getSizeInBits();
- auto NumEltsI8 = (BitSize + 7) / 8;
- auto NewBitSize = NumEltsI8 * 8;
+ llvm::TypeSize BitSize = CurTy.getSizeInBits();
+ uint64_t NumEltsI8 = (BitSize + 7) / 8;
+ uint64_t NewBitSize = NumEltsI8 * 8;
assert(CurTy.isVector() &&
"Expected only vector of i1 to reach here, scalar was extended to "
"i8 on widen scalars to be multiple of 8");
@@ -1702,39 +1713,39 @@ static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
}
static bool legalizeGFrem(MachineInstr &MI, MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
Register Src0Reg = MI.getOperand(1).getReg();
Register Src1Reg = MI.getOperand(2).getReg();
- auto Flags = MI.getFlags();
- auto FmAfn = Flags & MachineInstr::FmAfn;
+ unsigned Flags = MI.getFlags();
+ unsigned FmAfn = Flags & MachineInstr::FmAfn;
LLT Ty = MRI.getType(DstReg);
- auto DivFlags = Flags;
+ unsigned DivFlags = Flags;
if (FmAfn) {
DivFlags &= ~MachineInstr::FmAfn;
DivFlags |= MachineInstr::FmArcp;
}
- auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, DivFlags);
- auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
- auto Neg = B.buildFNeg(Ty, Trunc, Flags);
+ llvm::MachineInstrBuilder Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, DivFlags);
+ llvm::MachineInstrBuilder Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
+ llvm::MachineInstrBuilder Neg = B.buildFNeg(Ty, Trunc, Flags);
if (!FmAfn) {
- auto FMA = B.buildFMA(Ty, Neg, Src1Reg, Src0Reg, Flags);
+ llvm::MachineInstrBuilder FMA = B.buildFMA(Ty, Neg, Src1Reg, Src0Reg, Flags);
- auto &Semantics = getFltSemanticForLLT(Ty.getScalarType());
- auto InfC = B.buildFConstant(Ty, APFloat::getInf(Semantics));
+ const llvm::fltSemantics &Semantics = getFltSemanticForLLT(Ty.getScalarType());
+ llvm::MachineInstrBuilder InfC = B.buildFConstant(Ty, APFloat::getInf(Semantics));
- auto XAbs = B.buildIntrinsic(Intrinsic::pisa_fabs, {Ty})
+ llvm::MachineInstrBuilder XAbs = B.buildIntrinsic(Intrinsic::pisa_fabs, {Ty})
.addUse(Src0Reg)
.setMIFlags(Flags);
- auto YAbs = B.buildIntrinsic(Intrinsic::pisa_fabs, {Ty})
+ llvm::MachineInstrBuilder YAbs = B.buildIntrinsic(Intrinsic::pisa_fabs, {Ty})
.addUse(Src1Reg)
.setMIFlags(Flags);
// Using pisa_fabs is safe here: the result is only compared against Inf
// via OEQ, which is false for any NaN regardless of signaling/quiet.
- auto XFCmp = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, XAbs, InfC, Flags);
- auto YFCmp = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, YAbs, InfC, Flags);
- auto Sel = B.buildSelect(Ty, YFCmp, Src0Reg, FMA);
+ llvm::MachineInstrBuilder XFCmp = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, XAbs, InfC, Flags);
+ llvm::MachineInstrBuilder YFCmp = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, YAbs, InfC, Flags);
+ llvm::MachineInstrBuilder Sel = B.buildSelect(Ty, YFCmp, Src0Reg, FMA);
B.buildSelect(DstReg, XFCmp, FMA, Sel);
} else {
B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
@@ -1755,10 +1766,10 @@ static bool legalizeFAbs(MachineInstr &MI, MachineIRBuilder &B) {
// <2 x half> / <2 x bfloat>: pack into a single 32-bit AND.
if (Ty.isVector() && Ty.getNumElements() == 2 &&
Ty.getScalarSizeInBits() == 16) {
- auto Src32 = B.buildBitcast(I32, SrcReg);
+ llvm::MachineInstrBuilder Src32 = B.buildBitcast(I32, SrcReg);
// 0x7FFF7FFF: clears the sign bit of each 16-bit element.
- auto Mask = B.buildConstant(I32, 0x7FFF7FFF);
- auto And = B.buildAnd(I32, Src32, Mask);
+ llvm::MachineInstrBuilder Mask = B.buildConstant(I32, 0x7FFF7FFF);
+ llvm::MachineInstrBuilder And = B.buildAnd(I32, Src32, Mask);
B.buildBitcast(DstReg, And);
MI.eraseFromParent();
return true;
@@ -1780,8 +1791,8 @@ static bool legalizeFAbs(MachineInstr &MI, MachineIRBuilder &B) {
if (IsTypedFloat)
IntSrc = B.buildBitcast(IntTy, SrcReg).getReg(0);
- auto MaskCst = B.buildConstant(IntTy, Mask);
- auto And = B.buildAnd(IntTy, IntSrc, MaskCst);
+ llvm::MachineInstrBuilder MaskCst = B.buildConstant(IntTy, Mask);
+ llvm::MachineInstrBuilder And = B.buildAnd(IntTy, IntSrc, MaskCst);
if (IsTypedFloat)
B.buildBitcast(DstReg, And);
@@ -1801,16 +1812,17 @@ static bool legalizeGFdiv(MachineInstr &MI, MachineIRBuilder &B) {
return true;
// perform converts
- auto &MRI = *B.getMRI();
- auto [DstReg, Src0Reg, Src1Reg] = MI.getFirst3Regs();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ Register DstReg, Src0Reg, Src1Reg;
+ std::tie(DstReg, Src0Reg, Src1Reg) = MI.getFirst3Regs();
- auto Src0Tmp = MRI.createGenericVirtualRegister(F32);
- auto Src1Tmp = MRI.createGenericVirtualRegister(F32);
- auto DstTmp = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Src0Tmp = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Src1Tmp = MRI.createGenericVirtualRegister(F32);
+ llvm::Register DstTmp = MRI.createGenericVirtualRegister(F32);
- auto FPExt0 = B.buildFPExt(Src0Tmp, Src0Reg);
- auto FPExt1 = B.buildFPExt(Src1Tmp, Src1Reg);
- auto FDiv = B.buildFDiv(DstTmp, FPExt0, FPExt1);
+ llvm::MachineInstrBuilder FPExt0 = B.buildFPExt(Src0Tmp, Src0Reg);
+ llvm::MachineInstrBuilder FPExt1 = B.buildFPExt(Src1Tmp, Src1Reg);
+ llvm::MachineInstrBuilder FDiv = B.buildFDiv(DstTmp, FPExt0, FPExt1);
B.buildFPTrunc(DstReg, FDiv);
MI.eraseFromParent();
@@ -1819,7 +1831,7 @@ static bool legalizeGFdiv(MachineInstr &MI, MachineIRBuilder &B) {
static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
Register SrcReg = MI.getOperand(1).getReg();
Register EltReg = MI.getOperand(2).getReg();
Register IndexReg = MI.getOperand(3).getReg();
@@ -1838,24 +1850,24 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
LLT ScalarTy = LLT::integer(Size);
- auto SScalarReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto DScalarReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto MaskReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto NotReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto AndReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto ShiftReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto EltZExtReg = MRI.createGenericVirtualRegister(ScalarTy);
- auto ShiftAmountReg = IndexReg;
+ llvm::Register SScalarReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register DScalarReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register MaskReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register NotReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register AndReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register ShiftReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register EltZExtReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register ShiftAmountReg = IndexReg;
B.buildBitcast(SScalarReg, SrcReg);
- auto Value = getIConstantVRegValWithLookThrough(IndexReg, MRI);
+ std::optional<llvm::ValueAndVReg> Value = getIConstantVRegValWithLookThrough(IndexReg, MRI);
if (Value.has_value()) { // constant index
B.buildConstant(MaskReg, 1ull << Value->Value.getZExtValue());
ShiftAmountReg = MRI.createGenericVirtualRegister(ScalarTy);
B.buildConstant(ShiftAmountReg, Value->Value.getZExtValue());
} else { // non-constant index
- auto ConstReg = MRI.createGenericVirtualRegister(ScalarTy);
+ llvm::Register ConstReg = MRI.createGenericVirtualRegister(ScalarTy);
B.buildConstant(ConstReg, 1ull);
B.buildShl(MaskReg, ConstReg, IndexReg);
}
@@ -1874,7 +1886,8 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
"unexpected element size");
// If the index is constant, narrow the vector down to 4 elements.
- if (auto MaybeValue = getIConstantVRegValWithLookThrough(IndexReg, MRI)) {
+ if (std::optional<llvm::ValueAndVReg> MaybeValue =
+ getIConstantVRegValWithLookThrough(IndexReg, MRI)) {
if (NumElts <= 4) {
SmallVector<Register, 4> Elements;
for (int I = 0; I < NumElts; ++I)
@@ -1888,7 +1901,8 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
}
// Narrow to 4 elements.
- auto Res = Helper.fewerElementsVector(MI, 0, LLT::fixed_vector(4, EltTy));
+ llvm::LegalizerHelper::LegalizeResult Res =
+ Helper.fewerElementsVector(MI, 0, LLT::fixed_vector(4, EltTy));
return Res != LegalizerHelper::UnableToLegalize;
}
@@ -1901,7 +1915,7 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
if (NumElts * EltSize % 32 != 0) {
int NewNumElts = alignTo(NumElts, 32 / EltSize);
LLT NewVecTy = LLT::fixed_vector(NewNumElts, EltTy);
- auto Res = Helper.moreElementsVector(MI, 0, NewVecTy);
+ llvm::LegalizerHelper::LegalizeResult Res = Helper.moreElementsVector(MI, 0, NewVecTy);
if (Res == LegalizerHelper::UnableToLegalize)
return false;
NumElts = NewNumElts;
@@ -1937,7 +1951,7 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
}
int NewNumElts = NumElts * EltSize / 32;
LLT NewVecTy = NewNumElts == 1 ? I32 : LLT::fixed_vector(NewNumElts, I32);
- auto Res = Helper.bitcastInsertVectorElt(MI, 0, NewVecTy);
+ llvm::LegalizerHelper::LegalizeResult Res = Helper.bitcastInsertVectorElt(MI, 0, NewVecTy);
if (NeedFloatBitcast && Res != LegalizerHelper::UnableToLegalize) {
MachineInstr *DefMI = MRI.getVRegDef(IntDst);
B.setInsertPt(*DefMI->getParent(), std::next(DefMI->getIterator()));
@@ -1950,23 +1964,23 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
LLT NewVecTy = LLT::fixed_vector(NumElts * 2, I32);
// Compute the low and high indices as low = index * 2, high = low + 1
- auto One = B.buildConstant(IndexTy, 1).getReg(0);
- auto LowIndexReg = B.buildShl(IndexTy, IndexReg, One).getReg(0);
- auto HighIndexReg = B.buildAdd(IndexTy, LowIndexReg, One).getReg(0);
+ llvm::Register One = B.buildConstant(IndexTy, 1).getReg(0);
+ llvm::Register LowIndexReg = B.buildShl(IndexTy, IndexReg, One).getReg(0);
+ llvm::Register HighIndexReg = B.buildAdd(IndexTy, LowIndexReg, One).getReg(0);
// Split the 64-bit element into two 32-bit elements
- auto EltLowReg = MRI.createGenericVirtualRegister(I32);
- auto EltHighReg = MRI.createGenericVirtualRegister(I32);
+ llvm::Register EltLowReg = MRI.createGenericVirtualRegister(I32);
+ llvm::Register EltHighReg = MRI.createGenericVirtualRegister(I32);
B.buildUnmerge({EltLowReg, EltHighReg}, EltReg);
// Bitcast the source vector to s32 vector
- auto BitcastSrcReg = B.buildBitcast(NewVecTy, SrcReg).getReg(0);
+ llvm::Register BitcastSrcReg = B.buildBitcast(NewVecTy, SrcReg).getReg(0);
// Insert the low and high parts
- auto InsertLowReg = B.buildInsertVectorElement(NewVecTy, BitcastSrcReg,
+ llvm::Register InsertLowReg = B.buildInsertVectorElement(NewVecTy, BitcastSrcReg,
EltLowReg, LowIndexReg)
.getReg(0);
- auto InsertHighReg = B.buildInsertVectorElement(NewVecTy, InsertLowReg,
+ llvm::Register InsertHighReg = B.buildInsertVectorElement(NewVecTy, InsertLowReg,
EltHighReg, HighIndexReg)
.getReg(0);
@@ -1978,7 +1992,7 @@ static bool legalizeGInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
Register VecReg = MI.getOperand(1).getReg();
LLT VecTy = MRI.getType(VecReg);
LLT EltTy = VecTy.getScalarType();
@@ -1987,10 +2001,10 @@ static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
if (EltSize == 1) {
// Handle extraction of <n x i1>
- auto Size = MRI.getType(VecReg).getSizeInBits();
- auto CastReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
+ llvm::TypeSize Size = MRI.getType(VecReg).getSizeInBits();
+ llvm::Register CastReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
B.buildBitcast(CastReg, VecReg);
- auto ShiftReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
+ llvm::Register ShiftReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
B.buildLShr(ShiftReg, CastReg, MI.getOperand(2));
B.buildTrunc(MI.getOperand(0), ShiftReg);
MI.eraseFromParent();
@@ -2002,7 +2016,8 @@ static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
// If the index is constant, narrow the vector down to 4 elements.
Register IndexReg = MI.getOperand(2).getReg();
- if (auto MaybeValue = getIConstantVRegValWithLookThrough(IndexReg, MRI)) {
+ if (std::optional<llvm::ValueAndVReg> MaybeValue =
+ getIConstantVRegValWithLookThrough(IndexReg, MRI)) {
if (NumElts <= 4) {
Register UnmergeReg = B.buildUnmerge(EltTy, VecReg)
.getReg(MaybeValue->Value.getZExtValue());
@@ -2012,7 +2027,8 @@ static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
}
// Narrow to 4 elements.
- auto Res = Helper.fewerElementsVector(MI, 1, LLT::fixed_vector(4, EltTy));
+ llvm::LegalizerHelper::LegalizeResult Res =
+ Helper.fewerElementsVector(MI, 1, LLT::fixed_vector(4, EltTy));
return Res != LegalizerHelper::UnableToLegalize;
}
@@ -2024,7 +2040,7 @@ static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
if (NumElts * EltSize % 32 != 0) {
int NewNumElts = alignTo(NumElts, 32 / EltSize);
LLT NewVecTy = LLT::fixed_vector(NewNumElts, EltTy);
- auto Res = Helper.moreElementsVector(MI, 1, NewVecTy);
+ llvm::LegalizerHelper::LegalizeResult Res = Helper.moreElementsVector(MI, 1, NewVecTy);
if (Res == LegalizerHelper::UnableToLegalize)
return false;
NumElts = NewNumElts;
@@ -2052,7 +2068,7 @@ static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
}
int NewNumElts = NumElts * EltSize / 32;
LLT NewVecTy = NewNumElts == 1 ? I32 : LLT::fixed_vector(NewNumElts, I32);
- auto Res = Helper.bitcastExtractVectorElt(MI, 1, NewVecTy);
+ llvm::LegalizerHelper::LegalizeResult Res = Helper.bitcastExtractVectorElt(MI, 1, NewVecTy);
if (NeedFloatBitcast && Res != LegalizerHelper::UnableToLegalize) {
// MI has been erased by the helper. IntDst now has an integer-typed def
// from the helper's lowered sequence. Bitcast it back to the original
@@ -2065,8 +2081,9 @@ static bool legalizeGExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI,
}
static bool legalizeGBswap(MachineInstr &MI, MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
- auto [Dst, Src] = MI.getFirst2Regs();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ Register Dst, Src;
+ std::tie(Dst, Src) = MI.getFirst2Regs();
const LLT Ty = MRI.getType(Src);
unsigned BitSize = Ty.getScalarSizeInBits();
@@ -2083,14 +2100,18 @@ static bool legalizeGBswap(MachineInstr &MI, MachineIRBuilder &B) {
// a chunk.
// - ChunkShuffleVecTy: the vector type used for shuffling bytes within a
// chunk.
- auto GetSwapProps =
+ std::function<std::tuple<unsigned, ArrayRef<int>, LLT>(unsigned)>
+ GetSwapProps =
[&](unsigned BitSize) -> std::tuple<unsigned, ArrayRef<int>, LLT> {
return (BitSize % 32 == 0)
? std::make_tuple(32u, ArrayRef<int>(SwapMask32), V4I8)
: std::make_tuple(16u, ArrayRef<int>(SwapMask16), V2I8);
};
- auto [ChunkSize, ChunkByteSwapMask, ChunkShuffleVecTy] =
+ unsigned ChunkSize;
+ ArrayRef<int> ChunkByteSwapMask;
+ LLT ChunkShuffleVecTy;
+ std::tie(ChunkSize, ChunkByteSwapMask, ChunkShuffleVecTy) =
GetSwapProps(BitSize);
// For Src types that are multiples of 32 bits, the value is divided into
@@ -2101,28 +2122,28 @@ static bool legalizeGBswap(MachineInstr &MI, MachineIRBuilder &B) {
if (BitSize == 16 || BitSize == 32) {
assert(ChunkSize == BitSize &&
"Single chunk case: ChunkSize must equal BitSize");
- auto VecReg = B.buildBitcast(ChunkShuffleVecTy, Src);
- auto ShufReg = B.buildShuffleVector(ChunkShuffleVecTy, VecReg, VecReg,
+ llvm::MachineInstrBuilder VecReg = B.buildBitcast(ChunkShuffleVecTy, Src);
+ llvm::MachineInstrBuilder ShufReg = B.buildShuffleVector(ChunkShuffleVecTy, VecReg, VecReg,
ChunkByteSwapMask);
B.buildBitcast(Dst, ShufReg);
} else {
unsigned NumChunks = BitSize / ChunkSize;
LLT ChunkTy = LLT::integer(ChunkSize);
LLT VecTy = LLT::fixed_vector(NumChunks, ChunkTy);
- auto VecReg = B.buildBitcast(VecTy, Src);
+ llvm::MachineInstrBuilder VecReg = B.buildBitcast(VecTy, Src);
SmallVector<Register, 8> SwappedChunks;
for (int I = NumChunks - 1; I >= 0; --I) {
- auto Index = B.buildConstant(I32, I);
- auto ChunkReg = B.buildExtractVectorElement(ChunkTy, VecReg, Index);
- auto ChunkVec = B.buildBitcast(ChunkShuffleVecTy, ChunkReg);
- auto SwappedVec = B.buildShuffleVector(ChunkShuffleVecTy, ChunkVec,
+ llvm::MachineInstrBuilder Index = B.buildConstant(I32, I);
+ llvm::MachineInstrBuilder ChunkReg = B.buildExtractVectorElement(ChunkTy, VecReg, Index);
+ llvm::MachineInstrBuilder ChunkVec = B.buildBitcast(ChunkShuffleVecTy, ChunkReg);
+ llvm::MachineInstrBuilder SwappedVec = B.buildShuffleVector(ChunkShuffleVecTy, ChunkVec,
ChunkVec, ChunkByteSwapMask);
- auto SwappedChunk = B.buildBitcast(ChunkTy, SwappedVec);
+ llvm::MachineInstrBuilder SwappedChunk = B.buildBitcast(ChunkTy, SwappedVec);
SwappedChunks.push_back(SwappedChunk.getReg(0));
}
- auto FinalVec = B.buildBuildVector(VecTy, SwappedChunks);
+ llvm::MachineInstrBuilder FinalVec = B.buildBuildVector(VecTy, SwappedChunks);
B.buildBitcast(Dst, FinalVec);
}
@@ -2131,23 +2152,24 @@ static bool legalizeGBswap(MachineInstr &MI, MachineIRBuilder &B) {
}
static bool legalizeGFpow(MachineInstr &MI, MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
- auto [Dst, Src0, Src1] = MI.getFirst3Regs();
- auto DstTy = MRI.getType(Dst);
+ Register Dst, Src0, Src1;
+ std::tie(Dst, Src0, Src1) = MI.getFirst3Regs();
+ llvm::LLT DstTy = MRI.getType(Dst);
assert(DstTy.isScalar() &&
(DstTy.getSizeInBits() == 32 || DstTy.getSizeInBits() == 16));
unsigned Flags = MI.getFlags();
// can only do approximation of pow()
- auto AllowApprox = MI.getFlag(MachineInstr::FmAfn);
+ bool AllowApprox = MI.getFlag(MachineInstr::FmAfn);
if (!AllowApprox)
llvm_unreachable("not implemented (fpow)");
- auto LogReg = MRI.createGenericVirtualRegister(DstTy);
- auto MulReg = MRI.createGenericVirtualRegister(DstTy);
- auto FExp2Reg = Dst;
+ llvm::Register LogReg = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register MulReg = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register FExp2Reg = Dst;
B.buildFLog2(LogReg, Src0, Flags);
B.buildFMul(MulReg, LogReg, Src1, Flags);
@@ -2158,21 +2180,22 @@ static bool legalizeGFpow(MachineInstr &MI, MachineIRBuilder &B) {
}
static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
- auto [Dst, Src0, Src1] = MI.getFirst3Regs();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ Register Dst, Src0, Src1;
+ std::tie(Dst, Src0, Src1) = MI.getFirst3Regs();
unsigned Flags = MI.getFlags();
LLT XTy = MRI.getType(Src0);
LLT NTy = MRI.getType(Src1);
LLT Src1Ty = MRI.getType(Src1);
- auto AllowApprox =
+ bool AllowApprox =
XTy.getSizeInBits() <= 32 && MI.getFlag(MachineInstr::FmAfn);
- auto IsBFloat16 = XTy == LLT::bfloat16();
+ bool IsBFloat16 = XTy == LLT::bfloat16();
if (AllowApprox) {
- auto RegLLT = MRI.getType(Dst);
- auto FpReg = MRI.createGenericVirtualRegister(RegLLT);
- auto ExpReg = MRI.createGenericVirtualRegister(RegLLT);
+ llvm::LLT RegLLT = MRI.getType(Dst);
+ llvm::Register FpReg = MRI.createGenericVirtualRegister(RegLLT);
+ llvm::Register ExpReg = MRI.createGenericVirtualRegister(RegLLT);
B.buildSITOFP(FpReg, Src1);
B.buildFExp2(ExpReg, FpReg, Flags);
@@ -2207,13 +2230,13 @@ static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
// For FP64, |n|>1024+1022+52 will definitely lead to overflow/underflow
// |n|<=1022*3 is a sufficiently wide range for n (and FP64 x)
- auto ClampMax = B.buildConstant(NTy, -NClampRangeVal * 3);
- auto NClampedMax = B.buildSMax(NTy, Src1, ClampMax);
- auto ClampMin = B.buildConstant(NTy, NClampRangeVal * 3);
- auto NClamped = B.buildSMin(NTy, NClampedMax, ClampMin);
+ llvm::MachineInstrBuilder ClampMax = B.buildConstant(NTy, -NClampRangeVal * 3);
+ llvm::MachineInstrBuilder NClampedMax = B.buildSMax(NTy, Src1, ClampMax);
+ llvm::MachineInstrBuilder ClampMin = B.buildConstant(NTy, NClampRangeVal * 3);
+ llvm::MachineInstrBuilder NClamped = B.buildSMin(NTy, NClampedMax, ClampMin);
- auto AddConst = B.buildConstant(NTy, (NClampRangeVal + 1) * 3);
- auto N = B.buildAdd(NTy, NClamped, AddConst);
+ llvm::MachineInstrBuilder AddConst = B.buildConstant(NTy, (NClampRangeVal + 1) * 3);
+ llvm::MachineInstrBuilder N = B.buildAdd(NTy, NClamped, AddConst);
if (XTy.getSizeInBits() == 16 && !IsBFloat16) {
NTy = I16;
N = B.buildTrunc(NTy, N);
@@ -2222,13 +2245,13 @@ static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
// for fp16, n/3 performed as a 8x8-bit->16-bit integer MUL and SHR by 8.
// for others, n/3, performed as a 16x16-bit->32-bit integer MUL and SHR by 16
// (both LSHR or ASHR work, n is positive at this point)
- auto MulConst = B.buildConstant(NTy, NDivBy3MulVal);
- auto NMul = B.buildMul(NTy, N, MulConst);
- auto ShrConst = B.buildConstant(I32, NDivBy3ShiftVal);
- auto K0 = B.buildLShr(NTy, NMul, ShrConst);
+ llvm::MachineInstrBuilder MulConst = B.buildConstant(NTy, NDivBy3MulVal);
+ llvm::MachineInstrBuilder NMul = B.buildMul(NTy, N, MulConst);
+ llvm::MachineInstrBuilder ShrConst = B.buildConstant(I32, NDivBy3ShiftVal);
+ llvm::MachineInstrBuilder K0 = B.buildLShr(NTy, NMul, ShrConst);
- auto NMinusK0 = B.buildSub(NTy, N, K0);
- auto K1 = B.buildSub(NTy, NMinusK0, K0);
+ llvm::MachineInstrBuilder NMinusK0 = B.buildSub(NTy, N, K0);
+ llvm::MachineInstrBuilder K1 = B.buildSub(NTy, NMinusK0, K0);
if (XTy.getSizeInBits() == 64) {
NTy = I64;
@@ -2240,11 +2263,11 @@ static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
K1 = B.buildTrunc(NTy, K1);
}
- auto ShlConst = B.buildConstant(I32, NShiftVal);
- auto SK0I = B.buildShl(NTy, K0, ShlConst);
- auto SK1I = B.buildShl(NTy, K1, ShlConst);
- auto SK0 = B.buildBitcast(XTy, SK0I);
- auto SK1 = B.buildBitcast(XTy, SK1I);
+ llvm::MachineInstrBuilder ShlConst = B.buildConstant(I32, NShiftVal);
+ llvm::MachineInstrBuilder SK0I = B.buildShl(NTy, K0, ShlConst);
+ llvm::MachineInstrBuilder SK1I = B.buildShl(NTy, K1, ShlConst);
+ llvm::MachineInstrBuilder SK0 = B.buildBitcast(XTy, SK0I);
+ llvm::MachineInstrBuilder SK1 = B.buildBitcast(XTy, SK1I);
SrcOp SwapperX(Src0), SwapperSK1(SK1);
if (XTy.getSizeInBits() > 16) {
@@ -2252,17 +2275,17 @@ static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
// overflow (inf). This prevents a potential underflow that can happen with
// Src0 * SK0 * SK0.
int SmallThresholdVal = NClampRangeVal / 3;
- auto SmallThresholdConst = B.buildConstant(Src1Ty, SmallThresholdVal);
- auto Src1Abs = B.buildAbs(Src1Ty, NClamped);
- auto IsSrc1Small = B.buildICmp(CmpInst::Predicate::ICMP_SLT, I1, Src1Abs,
+ llvm::MachineInstrBuilder SmallThresholdConst = B.buildConstant(Src1Ty, SmallThresholdVal);
+ llvm::MachineInstrBuilder Src1Abs = B.buildAbs(Src1Ty, NClamped);
+ llvm::MachineInstrBuilder IsSrc1Small = B.buildICmp(CmpInst::Predicate::ICMP_SLT, I1, Src1Abs,
SmallThresholdConst);
SwapperX = B.buildSelect(XTy, IsSrc1Small, SK1, Src0);
SwapperSK1 = B.buildSelect(XTy, IsSrc1Small, Src0, SK1);
}
- auto Res0 = B.buildFMul(XTy, SwapperX, SK0, Flags);
- auto Res1 = B.buildFMul(XTy, Res0, SK0, Flags);
+ llvm::MachineInstrBuilder Res0 = B.buildFMul(XTy, SwapperX, SK0, Flags);
+ llvm::MachineInstrBuilder Res1 = B.buildFMul(XTy, Res0, SK0, Flags);
B.buildFMul(Dst, Res1, SwapperSK1, Flags);
MI.eraseFromParent();
@@ -2275,7 +2298,7 @@ static bool legalizeGFldexp(MachineInstr &MI, MachineIRBuilder &B) {
// - truncate to 16bit value
static bool legalizeIntrinsicFSqrt(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
SmallVector<MachineInstr *, 4> MIs;
Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
@@ -2285,19 +2308,18 @@ static bool legalizeIntrinsicFSqrt(LegalizerHelper &Helper, MachineInstr &MI) {
MIs.push_back(&MI);
}
- for (auto It = MIs.begin(), Ite = MIs.end(); It != Ite; ++It) {
- auto *MI = *It;
+ for (MachineInstr *MI : MIs) {
MachineIRBuilder MIB(*MI);
- auto Dst = MI->getOperand(0).getReg();
- auto Src = MI->getOperand(2).getReg();
- auto Imm = MI->getOperand(3).getImm();
+ llvm::Register Dst = MI->getOperand(0).getReg();
+ llvm::Register Src = MI->getOperand(2).getReg();
+ int64_t Imm = MI->getOperand(3).getImm();
if (MRI.getType(Dst).getScalarSizeInBits() != 16)
continue; // already legal
- auto Src32 = MRI.createGenericVirtualRegister(F32);
- auto Dst32 = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Src32 = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Dst32 = MRI.createGenericVirtualRegister(F32);
MIB.buildFPExt(Src32, Src);
MIB.buildIntrinsic(IntrinsicID, Dst32).addReg(Src32).addImm(Imm);
MIB.buildFPTrunc(Dst, Dst32);
@@ -2308,19 +2330,18 @@ static bool legalizeIntrinsicFSqrt(LegalizerHelper &Helper, MachineInstr &MI) {
static bool legalizeIntrinsicFDiv(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
SmallVector<MachineInstr *> MIs = scalarizeIntrinsic(MI);
// fdiv only supports 32/64 width
- for (auto It = MIs.begin(), Ite = MIs.end(); It != Ite; ++It) {
- auto *MI = *It;
+ for (MachineInstr *MI : MIs) {
MachineIRBuilder MIB(*MI);
- auto Dst = MI->getOperand(0).getReg();
- auto Src0 = MI->getOperand(2).getReg();
- auto Src1 = MI->getOperand(3).getReg();
+ llvm::Register Dst = MI->getOperand(0).getReg();
+ llvm::Register Src0 = MI->getOperand(2).getReg();
+ llvm::Register Src1 = MI->getOperand(3).getReg();
if (MRI.getType(Dst).getScalarSizeInBits() != 16)
continue;
@@ -2330,9 +2351,9 @@ static bool legalizeIntrinsicFDiv(LegalizerHelper &Helper, MachineInstr &MI) {
// => s32 C' = FEXT s16 C
// => s32 A' = FDIV s32 B', s32 C'
// => s16 A = FTRUNC s32 A'
- auto Src032 = MRI.createGenericVirtualRegister(F32);
- auto Src132 = MRI.createGenericVirtualRegister(F32);
- auto Dst32 = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Src032 = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Src132 = MRI.createGenericVirtualRegister(F32);
+ llvm::Register Dst32 = MRI.createGenericVirtualRegister(F32);
MIB.buildFPExt(Src032, Src0);
MIB.buildFPExt(Src132, Src1);
MIB.buildIntrinsic(IntrinsicID, Dst32)
@@ -2348,15 +2369,15 @@ static bool legalizeIntrinsicFDiv(LegalizerHelper &Helper, MachineInstr &MI) {
static SmallVector<Register> splitVectorByGrain(MachineIRBuilder &B,
Register Src, unsigned Grain) {
- auto &MRI = *B.getMRI();
- auto SrcTy = MRI.getType(Src);
- auto EltTy = SrcTy.getScalarType();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ llvm::LLT SrcTy = MRI.getType(Src);
+ llvm::LLT EltTy = SrcTy.getScalarType();
- auto SliceTy = LLT::fixed_vector(Grain, EltTy);
+ llvm::LLT SliceTy = LLT::fixed_vector(Grain, EltTy);
if (SrcTy.isScalar()) {
- auto SliceUndef = MRI.createGenericVirtualRegister(SliceTy);
- auto Slice = MRI.createGenericVirtualRegister(SliceTy);
+ llvm::Register SliceUndef = MRI.createGenericVirtualRegister(SliceTy);
+ llvm::Register Slice = MRI.createGenericVirtualRegister(SliceTy);
B.buildUndef(SliceUndef);
B.buildInsertVectorElement(Slice, SliceUndef, Src, B.buildConstant(I32, 0));
return {Slice};
@@ -2366,12 +2387,12 @@ static SmallVector<Register> splitVectorByGrain(MachineIRBuilder &B,
SmallVector<Register> Elts;
for (unsigned I = 0; I < NumElts; I += Grain) {
- auto Slice = MRI.createGenericVirtualRegister(SliceTy);
+ llvm::Register Slice = MRI.createGenericVirtualRegister(SliceTy);
B.buildUndef(Slice);
for (unsigned J = 0; J < std::min(Grain, NumElts - I); ++J) {
- auto Idx = B.buildConstant(I32, J).getReg(0);
- auto Elt =
+ llvm::Register Idx = B.buildConstant(I32, J).getReg(0);
+ llvm::Register Elt =
B.buildExtractVectorElementConstant(EltTy, Src, I + J).getReg(0);
Slice = B.buildInsertVectorElement(SliceTy, Slice, Elt, Idx).getReg(0);
}
@@ -2382,26 +2403,26 @@ static SmallVector<Register> splitVectorByGrain(MachineIRBuilder &B,
static void joinVectorByGrain(MachineIRBuilder &B, Register Dst,
ArrayRef<Register> Srcs, unsigned Grain) {
- auto &MRI = *B.getMRI();
- auto DstTy = MRI.getType(Dst);
- auto EltTy = DstTy.getScalarType();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ llvm::LLT DstTy = MRI.getType(Dst);
+ llvm::LLT EltTy = DstTy.getScalarType();
if (DstTy.isScalar()) {
- auto Src = Srcs[0];
+ llvm::Register Src = Srcs[0];
B.buildExtractVectorElementConstant(Dst, Src, 0);
return;
}
const unsigned NumElts = DstTy.getNumElements();
- auto TmpDst = B.buildUndef(DstTy).getReg(0);
+ llvm::Register TmpDst = B.buildUndef(DstTy).getReg(0);
for (unsigned I = 0; I < NumElts; I += Grain) {
- auto &Src = Srcs[I / Grain];
+ const llvm::Register &Src = Srcs[I / Grain];
for (unsigned J = 0; J < std::min(Grain, NumElts - I); ++J) {
- auto Idx = B.buildConstant(I32, I + J).getReg(0);
- auto Elt = B.buildExtractVectorElementConstant(EltTy, Src, J).getReg(0);
+ llvm::Register Idx = B.buildConstant(I32, I + J).getReg(0);
+ llvm::Register Elt = B.buildExtractVectorElementConstant(EltTy, Src, J).getReg(0);
TmpDst = B.buildInsertVectorElement(DstTy, TmpDst, Elt, Idx).getReg(0);
}
@@ -2412,42 +2433,45 @@ static void joinVectorByGrain(MachineIRBuilder &B, Register Dst,
static bool legalizeIntrinsicBfn(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
- auto &MRI = *B.getMRI();
- auto IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ unsigned IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
assert(IntrinsicID == Intrinsic::pisa_bfn);
- const auto BfnOpcode = MI.getOperand(2);
+ const llvm::MachineOperand BfnOpcode = MI.getOperand(2);
- const auto OrigDst = MI.getOperand(0).getReg();
- const auto OrigSrc0 = MI.getOperand(3).getReg();
- const auto OrigSrc1 = MI.getOperand(4).getReg();
- const auto OrigSrc2 = MI.getOperand(5).getReg();
+ const llvm::Register OrigDst = MI.getOperand(0).getReg();
+ const llvm::Register OrigSrc0 = MI.getOperand(3).getReg();
+ const llvm::Register OrigSrc1 = MI.getOperand(4).getReg();
+ const llvm::Register OrigSrc2 = MI.getOperand(5).getReg();
- const auto Ty = MRI.getType(OrigDst);
+ const llvm::LLT Ty = MRI.getType(OrigDst);
- const auto BitWidth = Ty.getScalarSizeInBits();
+ const unsigned BitWidth = Ty.getScalarSizeInBits();
switch (BitWidth) {
default:
llvm_unreachable("unexpected bitwidth");
case 8:
case 16: {
- const auto Grain = 32 / BitWidth;
+ const unsigned Grain = 32 / BitWidth;
MachineIRBuilder MIB(MI);
- auto Srcs0 = splitVectorByGrain(MIB, OrigSrc0, Grain);
- auto Srcs1 = splitVectorByGrain(MIB, OrigSrc1, Grain);
- auto Srcs2 = splitVectorByGrain(MIB, OrigSrc2, Grain);
+ llvm::SmallVector<llvm::Register, 12> Srcs0 = splitVectorByGrain(MIB, OrigSrc0, Grain);
+ llvm::SmallVector<llvm::Register, 12> Srcs1 = splitVectorByGrain(MIB, OrigSrc1, Grain);
+ llvm::SmallVector<llvm::Register, 12> Srcs2 = splitVectorByGrain(MIB, OrigSrc2, Grain);
- auto GrainTy = MRI.getType(Srcs0[0]);
+ llvm::LLT GrainTy = MRI.getType(Srcs0[0]);
SmallVector<Register> Dsts;
- for (auto [Src0, Src1, Src2] : zip(Srcs0, Srcs1, Srcs2)) {
- auto Dst = MRI.createGenericVirtualRegister(I32);
- auto Src0Cast = MRI.createGenericVirtualRegister(I32);
- auto Src1Cast = MRI.createGenericVirtualRegister(I32);
- auto Src2Cast = MRI.createGenericVirtualRegister(I32);
+ for (size_t I = 0; I < Srcs0.size(); ++I) {
+ Register Src0 = Srcs0[I];
+ Register Src1 = Srcs1[I];
+ Register Src2 = Srcs2[I];
+ llvm::Register Dst = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Src0Cast = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Src1Cast = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Src2Cast = MRI.createGenericVirtualRegister(I32);
- auto DstCast = MRI.createGenericVirtualRegister(GrainTy);
+ llvm::Register DstCast = MRI.createGenericVirtualRegister(GrainTy);
MIB.buildBitcast(Src0Cast, Src0);
MIB.buildBitcast(Src1Cast, Src1);
@@ -2470,19 +2494,19 @@ static bool legalizeIntrinsicBfn(LegalizerHelper &Helper, MachineInstr &MI) {
scalarizeIntrinsic(MI);
return true;
case 64: {
- auto MIs = scalarizeIntrinsic(MI);
+ llvm::SmallVector<llvm::MachineInstr*, 6> MIs = scalarizeIntrinsic(MI);
- for (auto *MI : MIs) {
+ for (llvm::MachineInstr *MI : MIs) {
MachineIRBuilder MIB(*MI);
- auto Dst = MI->getOperand(0).getReg();
- auto Src0 = MI->getOperand(3).getReg();
- auto Src1 = MI->getOperand(4).getReg();
- auto Src2 = MI->getOperand(5).getReg();
+ llvm::Register Dst = MI->getOperand(0).getReg();
+ llvm::Register Src0 = MI->getOperand(3).getReg();
+ llvm::Register Src1 = MI->getOperand(4).getReg();
+ llvm::Register Src2 = MI->getOperand(5).getReg();
- auto DstV2I32 = MRI.createGenericVirtualRegister(V2I32);
- auto Src0V2I32 = MRI.createGenericVirtualRegister(V2I32);
- auto Src1V2I32 = MRI.createGenericVirtualRegister(V2I32);
- auto Src2V2I32 = MRI.createGenericVirtualRegister(V2I32);
+ llvm::Register DstV2I32 = MRI.createGenericVirtualRegister(V2I32);
+ llvm::Register Src0V2I32 = MRI.createGenericVirtualRegister(V2I32);
+ llvm::Register Src1V2I32 = MRI.createGenericVirtualRegister(V2I32);
+ llvm::Register Src2V2I32 = MRI.createGenericVirtualRegister(V2I32);
MIB.buildUndef(DstV2I32);
@@ -2491,11 +2515,11 @@ static bool legalizeIntrinsicBfn(LegalizerHelper &Helper, MachineInstr &MI) {
MIB.buildBitcast(Src2V2I32, Src2);
for (int I = 0; I < 2; I++) {
- auto DstI32 = MRI.createGenericVirtualRegister(I32);
- auto Src0I32 = MRI.createGenericVirtualRegister(I32);
- auto Src1I32 = MRI.createGenericVirtualRegister(I32);
- auto Src2I32 = MRI.createGenericVirtualRegister(I32);
- auto Idx = MRI.createGenericVirtualRegister(I32);
+ llvm::Register DstI32 = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Src0I32 = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Src1I32 = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Src2I32 = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Idx = MRI.createGenericVirtualRegister(I32);
MIB.buildExtractVectorElementConstant(Src0I32, Src0V2I32, I);
MIB.buildExtractVectorElementConstant(Src1I32, Src1V2I32, I);
@@ -2509,7 +2533,7 @@ static bool legalizeIntrinsicBfn(LegalizerHelper &Helper, MachineInstr &MI) {
MIB.buildConstant(Idx, I);
- auto DstNext = MRI.createGenericVirtualRegister(V2I32);
+ llvm::Register DstNext = MRI.createGenericVirtualRegister(V2I32);
MIB.buildInsertVectorElement(DstNext, DstV2I32, DstI32, Idx);
DstV2I32 = DstNext;
}
@@ -2526,7 +2550,7 @@ static bool legalizeIntrinsicBfn(LegalizerHelper &Helper, MachineInstr &MI) {
static bool legalizeIntrinsicRE(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
- auto RndMode = MI.getOperand(MI.getNumOperands() - 1).getImm();
+ int64_t RndMode = MI.getOperand(MI.getNumOperands() - 1).getImm();
if (static_cast<RoundingMode>(RndMode) != RoundingMode::NearestTiesToEven)
return false; // only .re is supported
@@ -2574,28 +2598,27 @@ static bool legalizeIntrinsicRE(LegalizerHelper &Helper, MachineInstr &MI) {
static bool legalizeIntrinsicI2F(LegalizerHelper &Helper, MachineInstr &MI) {
MachineIRBuilder &B = Helper.MIRBuilder;
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
SmallVector<MachineInstr *> MIs = scalarizeIntrinsic(MI);
// @llvm.experimental.constrained.sitofp.f32.i1
- for (auto It = MIs.begin(), Ite = MIs.end(); It != Ite; ++It) {
- auto *MI = *It;
+ for (MachineInstr *MI : MIs) {
MachineIRBuilder MIB(*MI);
- auto SrcReg = MI->getOperand(2).getReg();
- auto SrcTy = MRI.getType(SrcReg);
+ llvm::Register SrcReg = MI->getOperand(2).getReg();
+ llvm::LLT SrcTy = MRI.getType(SrcReg);
if (SrcTy.getSizeInBits() >= 8)
continue;
- auto ExtReg = MRI.createGenericVirtualRegister(I8);
+ llvm::Register ExtReg = MRI.createGenericVirtualRegister(I8);
if (IntrinsicID == Intrinsic::pisa_uitofp)
MIB.buildZExt(ExtReg, SrcReg);
else
MIB.buildSExt(ExtReg, SrcReg);
- auto NewMI = MIB.buildIntrinsic(IntrinsicID, MI->getOperand(0).getReg())
+ llvm::MachineInstrBuilder NewMI = MIB.buildIntrinsic(IntrinsicID, MI->getOperand(0).getReg())
.addReg(ExtReg)
.add(MI->getOperand(3))
.add(MI->getOperand(4));
@@ -2615,7 +2638,7 @@ static bool legalizeIntrinsicDp4a(LegalizerHelper &Helper, MachineInstr &MI) {
return true;
MachineIRBuilder &B = Helper.MIRBuilder;
- auto &MRI = *B.getMRI();
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
B.setInstrAndDebugLoc(MI);
Register Dst = MI.getOperand(0).getReg();
@@ -2683,19 +2706,19 @@ bool PISALegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
}
static bool legalizeGConcatVectors(MachineInstr &MI, MachineIRBuilder &B) {
- auto *MRI = B.getMRI();
+ llvm::MachineRegisterInfo *MRI = B.getMRI();
- auto Dst = MI.getOperand(0).getReg();
- auto DstTy = MRI->getType(Dst);
+ llvm::Register Dst = MI.getOperand(0).getReg();
+ llvm::LLT DstTy = MRI->getType(Dst);
assert(DstTy.getScalarSizeInBits() == 32);
unsigned Idx = 0;
- auto TDst = MRI->createGenericVirtualRegister(DstTy);
+ llvm::Register TDst = MRI->createGenericVirtualRegister(DstTy);
B.buildInstr(TargetOpcode::IMPLICIT_DEF).addDef(TDst);
for (unsigned I = 1; I < MI.getNumOperands(); I++) {
- auto Src = MI.getOperand(I).getReg();
- auto SrcTy = B.getMRI()->getType(Src);
- auto NewDst = MRI->createGenericVirtualRegister(DstTy);
+ llvm::Register Src = MI.getOperand(I).getReg();
+ llvm::LLT SrcTy = B.getMRI()->getType(Src);
+ llvm::Register NewDst = MRI->createGenericVirtualRegister(DstTy);
B.buildInsertSubvector(NewDst, TDst, Src, Idx);
Idx += SrcTy.getNumElements();
TDst = NewDst;
@@ -2707,15 +2730,15 @@ static bool legalizeGConcatVectors(MachineInstr &MI, MachineIRBuilder &B) {
static bool legalizeGUnmergeValues(LegalizerHelper &Helper, MachineInstr &MI,
MachineIRBuilder &B) {
- auto *MRI = B.getMRI();
+ llvm::MachineRegisterInfo *MRI = B.getMRI();
- auto Src = MI.getOperand(MI.getNumOperands() - 1).getReg();
- auto DstTy = MRI->getType(MI.getOperand(0).getReg());
+ llvm::Register Src = MI.getOperand(MI.getNumOperands() - 1).getReg();
+ llvm::LLT DstTy = MRI->getType(MI.getOperand(0).getReg());
assert(MRI->getType(Src).getScalarSizeInBits() == 32);
unsigned Idx = 0;
for (unsigned I = 0; I < MI.getNumOperands() - 1; I++) {
- auto Dst = MI.getOperand(I).getReg();
+ llvm::Register Dst = MI.getOperand(I).getReg();
if (DstTy.isVector()) {
// <2 x s32>, <2 x s32> = G_UNMERGE_VALUES <4 x s32>
B.buildExtractSubvector(Dst, Src, Idx);
@@ -2732,12 +2755,12 @@ static bool legalizeGUnmergeValues(LegalizerHelper &Helper, MachineInstr &MI,
// NOLINTNEXTLINE(readability-identifier-naming)
static bool legalizeGInsertSubvector(MachineInstr &MI, MachineIRBuilder &B) {
- auto *MRI = B.getMRI();
+ llvm::MachineRegisterInfo *MRI = B.getMRI();
- auto DstReg = MI.getOperand(0).getReg();
- auto VecReg = MI.getOperand(1).getReg();
- auto SubVecReg = MI.getOperand(2).getReg();
- auto Idx = MI.getOperand(3).getImm();
+ llvm::Register DstReg = MI.getOperand(0).getReg();
+ llvm::Register VecReg = MI.getOperand(1).getReg();
+ llvm::Register SubVecReg = MI.getOperand(2).getReg();
+ int64_t Idx = MI.getOperand(3).getImm();
LLT DstTy = MRI->getType(DstReg);
@@ -2752,17 +2775,17 @@ static bool legalizeGInsertSubvector(MachineInstr &MI, MachineIRBuilder &B) {
LLT::fixed_vector(2 * DstTy.getNumElements(), LLT::integer(32));
// Update vec
- auto CastedVecReg = MRI->createGenericVirtualRegister(CastedVecTy);
+ llvm::Register CastedVecReg = MRI->createGenericVirtualRegister(CastedVecTy);
B.buildBitcast(CastedVecReg, VecReg);
MI.getOperand(1).setReg(CastedVecReg);
// Update subvec
- auto CastedSubVecReg = MRI->createGenericVirtualRegister(CastedSubVecTy);
+ llvm::Register CastedSubVecReg = MRI->createGenericVirtualRegister(CastedSubVecTy);
B.buildBitcast(CastedSubVecReg, SubVecReg);
MI.getOperand(2).setReg(CastedSubVecReg);
// Update dst
- auto CastedDstReg = MRI->createGenericVirtualRegister(CastedDstTy);
+ llvm::Register CastedDstReg = MRI->createGenericVirtualRegister(CastedDstTy);
MI.getOperand(0).setReg(CastedDstReg);
// Update index
@@ -2817,11 +2840,11 @@ static bool legalizeGInsertSubvector(MachineInstr &MI, MachineIRBuilder &B) {
// NOLINTNEXTLINE(readability-identifier-naming)
static bool legalizeGExtractSubvector(MachineInstr &MI, MachineIRBuilder &B) {
- auto *MRI = B.getMRI();
+ llvm::MachineRegisterInfo *MRI = B.getMRI();
- auto DstReg = MI.getOperand(0).getReg();
- auto SrcReg = MI.getOperand(1).getReg();
- auto Idx = MI.getOperand(2).getImm();
+ llvm::Register DstReg = MI.getOperand(0).getReg();
+ llvm::Register SrcReg = MI.getOperand(1).getReg();
+ int64_t Idx = MI.getOperand(2).getImm();
LLT SrcTy = MRI->getType(SrcReg);
@@ -2833,12 +2856,12 @@ static bool legalizeGExtractSubvector(MachineInstr &MI, MachineIRBuilder &B) {
LLT::fixed_vector(2 * DstTy.getNumElements(), LLT::integer(32));
// Update src
- auto CastedSrcReg = MRI->createGenericVirtualRegister(CastedSrcTy);
+ llvm::Register CastedSrcReg = MRI->createGenericVirtualRegister(CastedSrcTy);
B.buildBitcast(CastedSrcReg, SrcReg);
MI.getOperand(1).setReg(CastedSrcReg);
// Update dst
- auto CastedDstReg = MRI->createGenericVirtualRegister(CastedDstTy);
+ llvm::Register CastedDstReg = MRI->createGenericVirtualRegister(CastedDstTy);
MI.getOperand(0).setReg(CastedDstReg);
// Update index
@@ -2886,7 +2909,7 @@ static bool legalizeGExtractSubvector(MachineInstr &MI, MachineIRBuilder &B) {
static StringRef getSyncScopeStr(LLVMContext &Ctx, SyncScope::ID ScopeID) {
// Map dynamically assigned PISA SyncScope ID to its scope name.
static DenseMap<SyncScope::ID, StringRef> ScopeID2Name;
- auto InitializeScopeID2Name = [&]() {
+ std::function<void()> InitializeScopeID2Name = [&]() {
static const StringMap<StringRef> ScopeName2EncodeName = {
{"workgroup", "workgroup"},
{"gpu", "gpu"},
@@ -2895,8 +2918,10 @@ static StringRef getSyncScopeStr(LLVMContext &Ctx, SyncScope::ID ScopeID) {
{"subgroup", "workgroup"},
{"workitem", "workgroup"},
};
- for (const auto &[Name, EncodeName] : ScopeName2EncodeName) {
- auto ID = Ctx.getOrInsertSyncScopeID(Name);
+ for (const StringMapEntry<StringRef> &Entry : ScopeName2EncodeName) {
+ StringRef Name = Entry.first();
+ StringRef EncodeName = Entry.second;
+ SyncScope::ID ID = Ctx.getOrInsertSyncScopeID(Name);
ScopeID2Name.emplace_or_assign(ID, EncodeName);
}
};
@@ -2904,7 +2929,8 @@ static StringRef getSyncScopeStr(LLVMContext &Ctx, SyncScope::ID ScopeID) {
std::call_once(InitializeScopeID2NameFlag, InitializeScopeID2Name);
// Use the original SyncScope ID to look up its scope name in the map.
- auto It = ScopeID2Name.find(ScopeID);
+ DenseMap<SyncScope::ID, StringRef>::iterator It =
+ ScopeID2Name.find(ScopeID);
return It != ScopeID2Name.end() ? It->second : StringRef("gpu");
}
@@ -2917,7 +2943,7 @@ static bool legalizeGAtomicrmw(MachineInstr &MI, MachineIRBuilder &B) {
isAtLeastOrStrongerThan(AtomicOrdering::Release, AOF))
return true;
- auto &Ctx = B.getMF().getFunction().getContext();
+ llvm::LLVMContext &Ctx = B.getMF().getFunction().getContext();
llvm::SmallString<16> FenceScopeStr =
getSyncScopeStr(Ctx, MemOp->getSyncScopeID());
unsigned AddressSpace = MemOp->getAddrSpace();
@@ -2963,11 +2989,11 @@ static bool legalizeGAtomicrmw(MachineInstr &MI, MachineIRBuilder &B) {
// NOLINTNEXTLINE(readability-identifier-naming)
static bool legalizeGAtomicrmwXchg(MachineInstr &MI, MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
- auto &Dst = MI.getOperand(0);
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ llvm::MachineOperand &Dst = MI.getOperand(0);
LLT CurTy = MRI.getType(Dst.getReg());
if (CurTy.getScalarType().isPointer()) {
- auto &Src = MI.getOperand(2);
+ llvm::MachineOperand &Src = MI.getOperand(2);
LLT NewTy = LLT::integer(CurTy.getScalarSizeInBits());
Register NewSrc = MRI.createGenericVirtualRegister(NewTy);
Register NewDst = MRI.createGenericVirtualRegister(NewTy);
@@ -2985,15 +3011,15 @@ static bool legalizeGAtomicrmwXchg(MachineInstr &MI, MachineIRBuilder &B) {
// NOLINTNEXTLINE(readability-identifier-naming)
static bool legalizeGShuffleVector(LegalizerHelper &Helper, MachineInstr &MI,
MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
- auto &Dst = MI.getOperand(0);
- auto &Src0 = MI.getOperand(1);
- auto &Src1 = MI.getOperand(2);
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ llvm::MachineOperand &Dst = MI.getOperand(0);
+ llvm::MachineOperand &Src0 = MI.getOperand(1);
+ llvm::MachineOperand &Src1 = MI.getOperand(2);
ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
assert(MRI.getType(Dst.getReg()).getScalarSizeInBits() == 32);
assert(isPowerOf2_32(MRI.getType(Dst.getReg()).getNumElements()));
- auto UseExtract = true;
+ bool UseExtract = true;
// indices must be consecutive
int PrevIdx = -1;
for (int Idx : Mask) {
@@ -3007,7 +3033,7 @@ static bool legalizeGShuffleVector(LegalizerHelper &Helper, MachineInstr &MI,
UseExtract = false;
// indices can not straddle the arguments
- auto SrcSize = MRI.getType(Src0.getReg()).getNumElements();
+ uint16_t SrcSize = MRI.getType(Src0.getReg()).getNumElements();
if ((Mask[0] < SrcSize) && ((Mask[0] + Mask.size()) > SrcSize))
UseExtract = false;
@@ -3015,26 +3041,28 @@ static bool legalizeGShuffleVector(LegalizerHelper &Helper, MachineInstr &MI,
UseExtract = false;
if (UseExtract) {
- auto Src = (Mask[0] < SrcSize) ? Src0 : Src1;
- auto Idx = (Mask[0] < SrcSize) ? Mask[0] : Mask[0] - SrcSize;
+ llvm::MachineOperand Src = (Mask[0] < SrcSize) ? Src0 : Src1;
+ int Idx = (Mask[0] < SrcSize) ? Mask[0] : Mask[0] - SrcSize;
B.buildExtractSubvector(Dst, Src, Idx);
MI.eraseFromParent();
return true;
}
// lower if unable to use extract/insert
- auto Res = Helper.lowerShuffleVector(MI);
+ llvm::LegalizerHelper::LegalizeResult Res = Helper.lowerShuffleVector(MI);
return Res != LegalizerHelper::UnableToLegalize;
}
// NOLINTNEXTLINE(readability-identifier-naming)
static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
MachineIRBuilder &MIRBuilder) {
- auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
+ Register DstReg, SrcReg;
+ LLT DstTy, SrcTy;
+ std::tie(DstReg, DstTy, SrcReg, SrcTy) = MI.getFirst2RegLLTs();
FPClassTest OriginalMask =
static_cast<FPClassTest>(MI.getOperand(2).getImm());
- auto Mask = OriginalMask;
- auto IsInvertedCheck = false;
+ llvm::FPClassTest Mask = OriginalMask;
+ bool IsInvertedCheck = false;
if (Mask == fcNone) {
MIRBuilder.buildConstant(DstReg, 0);
@@ -3048,13 +3076,13 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
}
// support bfloat types
- auto &Semantics = getFltSemanticForLLT(SrcTy.getScalarType());
+ const llvm::fltSemantics &Semantics = getFltSemanticForLLT(SrcTy.getScalarType());
unsigned BitSize = SrcTy.getScalarSizeInBits();
LLT IntTy = LLT::integer(BitSize);
if (SrcTy.isVector())
IntTy = LLT::vector(SrcTy.getElementCount(), IntTy);
- auto AsInt = MIRBuilder.buildBitcast(IntTy, SrcReg);
+ llvm::MachineInstrBuilder AsInt = MIRBuilder.buildBitcast(IntTy, SrcReg);
// Various masks.
APInt SignBit = APInt::getSignMask(BitSize);
@@ -3066,20 +3094,21 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
APInt::getOneBitSet(BitSize, AllOneMantissa.getActiveBits() - 1);
APInt InvertionMask = APInt::getAllOnes(DstTy.getScalarSizeInBits());
- auto SignBitC = MIRBuilder.buildConstant(IntTy, SignBit);
- auto ValueMaskC = MIRBuilder.buildConstant(IntTy, ValueMask);
- auto InfC = MIRBuilder.buildConstant(IntTy, Inf);
- auto ExpMaskC = MIRBuilder.buildConstant(IntTy, ExpMask);
- auto ZeroC = MIRBuilder.buildConstant(IntTy, 0);
+ llvm::MachineInstrBuilder SignBitC = MIRBuilder.buildConstant(IntTy, SignBit);
+ llvm::MachineInstrBuilder ValueMaskC = MIRBuilder.buildConstant(IntTy, ValueMask);
+ llvm::MachineInstrBuilder InfC = MIRBuilder.buildConstant(IntTy, Inf);
+ llvm::MachineInstrBuilder ExpMaskC = MIRBuilder.buildConstant(IntTy, ExpMask);
+ llvm::MachineInstrBuilder ZeroC = MIRBuilder.buildConstant(IntTy, 0);
- auto Abs = MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC);
- auto Sign =
+ llvm::MachineInstrBuilder Abs = MIRBuilder.buildAnd(IntTy, AsInt, ValueMaskC);
+ llvm::MachineInstrBuilder Sign =
MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_NE, DstTy, AsInt, Abs);
- auto Res = MIRBuilder.buildConstant(DstTy, 0);
+ llvm::MachineInstrBuilder Res = MIRBuilder.buildConstant(DstTy, 0);
// Clang doesn't support capture of structured bindings:
LLT DstTyCopy = DstTy;
- const auto AppendToRes = [&](MachineInstrBuilder ToAppend) {
+ const std::function<void(MachineInstrBuilder)> AppendToRes =
+ [&](MachineInstrBuilder ToAppend) {
Res = MIRBuilder.buildOr(DstTyCopy, Res, ToAppend);
};
@@ -3096,9 +3125,9 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
Mask &= ~fcPosFinite;
} else if ((Mask & fcFinite) == fcNegFinite) {
// finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
- auto Cmp = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs,
+ llvm::MachineInstrBuilder Cmp = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, Abs,
ExpMaskC);
- auto And = MIRBuilder.buildAnd(DstTy, Cmp, Sign);
+ llvm::MachineInstrBuilder And = MIRBuilder.buildAnd(DstTy, Cmp, Sign);
AppendToRes(And);
Mask &= ~fcNegFinite;
}
@@ -3108,7 +3137,7 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
// TODO: Handle sign bit specific cases
// TODO: Handle inverted case
if (PartialCheck == (fcZero | fcSubnormal)) {
- auto ExpBits = MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC);
+ llvm::MachineInstrBuilder ExpBits = MIRBuilder.buildAnd(IntTy, AsInt, ExpMaskC);
AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
ExpBits, ZeroC));
Mask &= ~PartialCheck;
@@ -3118,7 +3147,7 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
if (Mask == OriginalMask) {
// combination of classes above did not yield any
// optimizations, see if inverse will be less ops
- auto InvertedMask = (unsigned)~Mask;
+ unsigned InvertedMask = (unsigned)~Mask;
if (llvm::popcount((unsigned)Mask) > llvm::popcount(InvertedMask)) {
Mask = ~Mask;
IsInvertedCheck = true;
@@ -3141,10 +3170,10 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
if (FPClassTest PartialCheck = Mask & fcSubnormal) {
// issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
// issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
- auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
- auto OneC = MIRBuilder.buildConstant(IntTy, 1);
- auto VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC);
- auto SubnormalRes =
+ llvm::MachineInstrBuilder V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
+ llvm::MachineInstrBuilder OneC = MIRBuilder.buildConstant(IntTy, 1);
+ llvm::MachineInstrBuilder VMinusOne = MIRBuilder.buildSub(IntTy, V, OneC);
+ llvm::MachineInstrBuilder SubnormalRes =
MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, VMinusOne,
MIRBuilder.buildConstant(IntTy, AllOneMantissa));
if (PartialCheck == fcNegSubnormal)
@@ -3161,14 +3190,14 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy, Abs, InfC));
else { // fcNegInf
APInt NegInf = APFloat::getInf(Semantics, true).bitcastToAPInt();
- auto NegInfC = MIRBuilder.buildConstant(IntTy, NegInf);
+ llvm::MachineInstrBuilder NegInfC = MIRBuilder.buildConstant(IntTy, NegInf);
AppendToRes(MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, DstTy,
AsInt, NegInfC));
}
}
if (FPClassTest PartialCheck = Mask & fcNan) {
- auto InfWithQnanBitC =
+ llvm::MachineInstrBuilder InfWithQnanBitC =
MIRBuilder.buildConstant(IntTy, std::move(Inf) | QNaNBitMask);
if (PartialCheck == fcNan) {
// isnan(V) ==> abs(V) u> int(inf)
@@ -3181,10 +3210,10 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
} else { // fcSNan
// issignaling(V) ==> abs(V) u> unsigned(Inf) &&
// abs(V) u< (unsigned(Inf) | quiet_bit)
- auto IsNan =
+ llvm::MachineInstrBuilder IsNan =
MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_UGT, DstTy, Abs, InfC);
- auto IsNotQnan = MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy,
- Abs, InfWithQnanBitC);
+ llvm::MachineInstrBuilder IsNotQnan = MIRBuilder.buildICmp(
+ CmpInst::Predicate::ICMP_ULT, DstTy, Abs, InfWithQnanBitC);
AppendToRes(MIRBuilder.buildAnd(DstTy, IsNan, IsNotQnan));
}
}
@@ -3193,16 +3222,16 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
// isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
// (max_exp-1))
APInt ExpLSB = ExpMask & ~(ExpMask.shl(1));
- auto ExpMinusOne = MIRBuilder.buildSub(
+ llvm::MachineInstrBuilder ExpMinusOne = MIRBuilder.buildSub(
IntTy, Abs, MIRBuilder.buildConstant(IntTy, ExpLSB));
APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
- auto NormalRes =
+ llvm::MachineInstrBuilder NormalRes =
MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_ULT, DstTy, ExpMinusOne,
MIRBuilder.buildConstant(IntTy, MaxExpMinusOne));
if (PartialCheck == fcNegNormal)
NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, Sign);
else if (PartialCheck == fcPosNormal) {
- auto PosSign = MIRBuilder.buildXor(
+ llvm::MachineInstrBuilder PosSign = MIRBuilder.buildXor(
DstTy, Sign, MIRBuilder.buildConstant(DstTy, InvertionMask));
NormalRes = MIRBuilder.buildAnd(DstTy, NormalRes, PosSign);
}
@@ -3219,34 +3248,35 @@ static bool legalizeGIsFpclass(LegalizerHelper &Helper, MachineInstr &MI,
static bool legalizeGMulh(LegalizerHelper &Helper, MachineInstr &MI,
MachineIRBuilder &B) {
- auto &MRI = *B.getMRI();
- auto [Dst, Src0, Src1] = MI.getFirst3Regs();
- auto IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
- auto DstTy = MRI.getType(Dst);
+ llvm::MachineRegisterInfo &MRI = *B.getMRI();
+ Register Dst, Src0, Src1;
+ std::tie(Dst, Src0, Src1) = MI.getFirst3Regs();
+ bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
+ llvm::LLT DstTy = MRI.getType(Dst);
assert(DstTy.getSizeInBits() == 64);
- auto SourceA = MRI.createGenericVirtualRegister(DstTy);
- auto SourceB = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register SourceA = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register SourceB = MRI.createGenericVirtualRegister(DstTy);
- auto Const32 = MRI.createGenericVirtualRegister(DstTy);
- auto Const63 = MRI.createGenericVirtualRegister(DstTy);
- auto Const0 = MRI.createGenericVirtualRegister(DstTy);
- auto Mask32 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register Const32 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register Const63 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register Const0 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register Mask32 = MRI.createGenericVirtualRegister(DstTy);
B.buildConstant(Const32, 32);
B.buildConstant(Const63, 63);
B.buildConstant(Const0, 0);
B.buildConstant(Mask32, 0xFFFFFFFF);
- auto ASign = MRI.createGenericVirtualRegister(DstTy);
- auto BSign = MRI.createGenericVirtualRegister(DstTy);
- auto ResultSign = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ASign = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register BSign = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ResultSign = MRI.createGenericVirtualRegister(DstTy);
B.buildAShr(ASign, Src0, Const63);
B.buildAShr(BSign, Src1, Const63);
B.buildXor(ResultSign, ASign, BSign);
if (IsSigned) {
- auto ASignXor = MRI.createGenericVirtualRegister(DstTy);
- auto BSignXor = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ASignXor = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register BSignXor = MRI.createGenericVirtualRegister(DstTy);
B.buildXor(ASignXor, Src0, ASign);
B.buildXor(BSignXor, Src1, BSign);
B.buildSub(SourceA, ASignXor, ASign);
@@ -3256,40 +3286,40 @@ static bool legalizeGMulh(LegalizerHelper &Helper, MachineInstr &MI,
B.buildCopy(SourceB, Src1);
}
- auto LoSrc0 = MRI.createGenericVirtualRegister(DstTy);
- auto HiSrc0 = MRI.createGenericVirtualRegister(DstTy);
- auto LoSrc1 = MRI.createGenericVirtualRegister(DstTy);
- auto HiSrc1 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register LoSrc0 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register HiSrc0 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register LoSrc1 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register HiSrc1 = MRI.createGenericVirtualRegister(DstTy);
B.buildLShr(HiSrc0, SourceA, Const32);
B.buildLShr(HiSrc1, SourceB, Const32);
B.buildAnd(LoSrc0, SourceA, Mask32);
B.buildAnd(LoSrc1, SourceB, Mask32);
- auto ALobLo = MRI.createGenericVirtualRegister(DstTy);
- auto ALobHi = MRI.createGenericVirtualRegister(DstTy);
- auto AHibLo = MRI.createGenericVirtualRegister(DstTy);
- auto AHibHi = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ALobLo = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ALobHi = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register AHibLo = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register AHibHi = MRI.createGenericVirtualRegister(DstTy);
B.buildMul(AHibHi, HiSrc0, HiSrc1);
B.buildMul(AHibLo, HiSrc0, LoSrc1);
B.buildMul(ALobHi, LoSrc0, HiSrc1);
B.buildMul(ALobLo, LoSrc0, LoSrc1);
- auto ALobLoHi = MRI.createGenericVirtualRegister(DstTy);
- auto ALobHiLo = MRI.createGenericVirtualRegister(DstTy);
- auto AHibLoSum0 = MRI.createGenericVirtualRegister(DstTy);
- auto AHibLoSum1 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ALobLoHi = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ALobHiLo = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register AHibLoSum0 = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register AHibLoSum1 = MRI.createGenericVirtualRegister(DstTy);
B.buildLShr(ALobLoHi, ALobLo, Const32);
B.buildAnd(ALobHiLo, ALobHi, Mask32);
B.buildAdd(AHibLoSum0, ALobLoHi, ALobHiLo);
B.buildAdd(AHibLoSum1, AHibLo, AHibLoSum0);
- auto ALobLoMasked = MRI.createGenericVirtualRegister(DstTy);
- auto AHibLoShiftedL = MRI.createGenericVirtualRegister(DstTy);
- auto ALobHiShiftedR = MRI.createGenericVirtualRegister(DstTy);
- auto AHibLoShiftedR = MRI.createGenericVirtualRegister(DstTy);
- auto ShiftedSum = MRI.createGenericVirtualRegister(DstTy);
- auto DstLo = MRI.createGenericVirtualRegister(DstTy);
- auto DstHi = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ALobLoMasked = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register AHibLoShiftedL = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ALobHiShiftedR = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register AHibLoShiftedR = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register ShiftedSum = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register DstLo = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register DstHi = MRI.createGenericVirtualRegister(DstTy);
B.buildAnd(ALobLoMasked, ALobLo, Mask32);
B.buildShl(AHibLoShiftedL, AHibLoSum1, Const32);
@@ -3305,15 +3335,15 @@ static bool legalizeGMulh(LegalizerHelper &Helper, MachineInstr &MI,
// lo = lo ^ mask;
// lo += resultSign; // Add 1 if resultSign is negative, otherwise add 0
// hi += (lo < resultSign); // Adjust hi if lo overflowed
- auto Mask = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register Mask = MRI.createGenericVirtualRegister(DstTy);
B.buildNeg(Mask, ResultSign);
- auto HiXorMask = MRI.createGenericVirtualRegister(DstTy);
- auto LoXorMask = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register HiXorMask = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register LoXorMask = MRI.createGenericVirtualRegister(DstTy);
B.buildXor(HiXorMask, DstHi, Mask);
B.buildXor(LoXorMask, DstLo, Mask);
- auto LoAddResult = MRI.createGenericVirtualRegister(DstTy);
- auto LoAddShiftResult = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register LoAddResult = MRI.createGenericVirtualRegister(DstTy);
+ llvm::Register LoAddShiftResult = MRI.createGenericVirtualRegister(DstTy);
B.buildAdd(LoAddResult, LoXorMask, ResultSign);
B.buildShl(LoAddShiftResult, LoAddResult, ResultSign);
B.buildAdd(Dst, HiXorMask, LoAddShiftResult);
@@ -3328,7 +3358,7 @@ static bool legalizeGMulh(LegalizerHelper &Helper, MachineInstr &MI,
// spaces. Ensures that null pointers are preserved during the cast by replacing
// the addrspacecast with a null pointer of the destination type.
static bool legalizeGAddrspaceCast(MachineInstr &MI, MachineIRBuilder &B) {
- auto &Dst = MI.getOperand(0);
+ llvm::MachineOperand &Dst = MI.getOperand(0);
LLT DstTy = B.getMRI()->getType(Dst.getReg());
B.buildConstant(
Dst, PISATargetMachine::getNullPointerValue(DstTy.getAddressSpace()));
@@ -3338,17 +3368,19 @@ static bool legalizeGAddrspaceCast(MachineInstr &MI, MachineIRBuilder &B) {
// NOLINTNEXTLINE(readability-identifier-naming)
static bool legalizeGBitcast(MachineInstr &MI, MachineIRBuilder &B) {
- auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
- auto *MRI = B.getMRI();
+ Register DstReg, SrcReg;
+ LLT DstTy, SrcTy;
+ std::tie(DstReg, DstTy, SrcReg, SrcTy) = MI.getFirst2RegLLTs();
+ llvm::MachineRegisterInfo *MRI = B.getMRI();
assert(SrcTy.getSizeInBits() == DstTy.getSizeInBits());
if (SrcTy.isPointer() != DstTy.isPointer()) {
// legalize bitcast between pointers and non-pointers
if (SrcTy.isPointer()) {
// <2 x i32> G_BITCAST (p1)
- auto IntSize = SrcTy.getSizeInBits();
- auto IntTy = LLT::integer(IntSize);
- auto IntReg = MRI->createGenericVirtualRegister(IntTy);
+ llvm::TypeSize IntSize = SrcTy.getSizeInBits();
+ llvm::LLT IntTy = LLT::integer(IntSize);
+ llvm::Register IntReg = MRI->createGenericVirtualRegister(IntTy);
B.buildPtrToInt(IntReg, SrcReg);
if (IntTy == DstTy)
B.buildCopy(DstReg, IntReg);
@@ -3356,9 +3388,9 @@ static bool legalizeGBitcast(MachineInstr &MI, MachineIRBuilder &B) {
B.buildBitcast(DstReg, IntReg);
} else {
// (p1) G_BITCAST <2 x i32>
- auto IntSize = DstTy.getSizeInBits();
- auto IntTy = LLT::integer(IntSize);
- auto IntReg = MRI->createGenericVirtualRegister(IntTy);
+ llvm::TypeSize IntSize = DstTy.getSizeInBits();
+ llvm::LLT IntTy = LLT::integer(IntSize);
+ llvm::Register IntReg = MRI->createGenericVirtualRegister(IntTy);
if (IntTy == SrcTy)
B.buildCopy(IntReg, SrcReg);
else
@@ -3378,10 +3410,10 @@ static bool legalizeGBitcast(MachineInstr &MI, MachineIRBuilder &B) {
// and reinterpret as the destination scalar type.
SmallVector<Register, 4> DstElements;
for (unsigned I = 0; I < SrcTy.getNumElements(); I++) {
- auto SrcElemReg =
+ llvm::Register SrcElemReg =
MRI->createGenericVirtualRegister(SrcTy.getScalarType());
B.buildExtractVectorElementConstant(SrcElemReg, SrcReg, I);
- auto DstElemReg =
+ llvm::Register DstElemReg =
MRI->createGenericVirtualRegister(DstTy.getScalarType());
B.buildBitcast(DstElemReg, SrcElemReg);
DstElements.push_back(DstElemReg);
@@ -3394,10 +3426,10 @@ static bool legalizeGBitcast(MachineInstr &MI, MachineIRBuilder &B) {
SmallVector<Register, 4> ScalarPieces;
for (unsigned SrcElemIdx = 0; SrcElemIdx < SrcTy.getNumElements();
SrcElemIdx++) {
- auto SrcElemReg =
+ llvm::Register SrcElemReg =
MRI->createGenericVirtualRegister(SrcTy.getScalarType());
B.buildExtractVectorElementConstant(SrcElemReg, SrcReg, SrcElemIdx);
- auto Unmerge =
+ llvm::MachineInstrBuilder Unmerge =
B.buildUnmerge(LLT::integer(CommonPieceSize), SrcElemReg);
for (unsigned PieceIdx = 0; PieceIdx != Unmerge->getNumOperands() - 1;
PieceIdx++)
@@ -3407,7 +3439,7 @@ static bool legalizeGBitcast(MachineInstr &MI, MachineIRBuilder &B) {
SmallVector<Register, 4> DstElements;
for (unsigned PieceStartIdx = 0; PieceStartIdx < ScalarPieces.size();
PieceStartIdx += NumPiecesPerDstElem) {
- auto DstElemReg =
+ llvm::Register DstElemReg =
MRI->createGenericVirtualRegister(DstTy.getScalarType());
SmallVector<Register> DstElemPieces(
ScalarPieces.begin() + PieceStartIdx,
diff --git a/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h b/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h
index 34c3a88176984..2a60e537b8d50 100644
--- a/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h
+++ b/llvm/lib/Target/PISA/PISAMachineFunctionInfo.h
@@ -24,7 +24,7 @@ class PISAMachineFunctionInfo : public MachineFunctionInfo {
~PISAMachineFunctionInfo() override;
ArgInfo getArgInfo(unsigned Slot) const {
- auto I = ArgInfos.find(Slot);
+ DenseMap<unsigned, ArgInfo>::const_iterator I = ArgInfos.find(Slot);
if (I == ArgInfos.end())
return {0, false};
return I->second;
diff --git a/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp b/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
index b7ab3b214d7ee..a3f66f9f42e8f 100644
--- a/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
@@ -222,16 +222,16 @@ bool PISAPostLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
void PISAPostLegalizerCombinerImpl::applyBuildVectorWithConstants(
MachineInstr &MI) const {
- auto EltSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
- auto Size = EltSize * (MI.getNumOperands() - 1);
+ llvm::TypeSize EltSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
+ llvm::TypeSize Size = EltSize * (MI.getNumOperands() - 1);
assert((Size <= 64) && "vector size too large");
- auto NewReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
+ llvm::Register NewReg = MRI.createGenericVirtualRegister(LLT::integer(Size));
uint64_t Value = 0;
for (unsigned I = MI.getNumOperands() - 1; I > 0; I--) {
- auto Reg = MI.getOperand(I).getReg();
- auto CValue = getAnyConstantVRegValWithLookThrough(Reg, MRI);
+ llvm::Register Reg = MI.getOperand(I).getReg();
+ std::optional<llvm::ValueAndVReg> CValue = getAnyConstantVRegValWithLookThrough(Reg, MRI);
assert(CValue.has_value() && "expected const vreg val");
APInt IValue = CValue->Value;
Value <<= EltSize;
@@ -251,16 +251,16 @@ void PISAPostLegalizerCombinerImpl::applyBuildVectorWithConstants(
// i? S = G_SELECT i1 C, i? LHS, i? RHS
// => S = sel.? RHS, LHS, A
bool PISAPostLegalizerCombinerImpl::matchCompareSelect(MachineInstr *MI) const {
- auto *SelectMI = MI;
- auto *CmpMI = getDefIgnoringCopies(SelectMI->getOperand(1).getReg(), MRI);
+ llvm::MachineInstr *SelectMI = MI;
+ llvm::MachineInstr *CmpMI = getDefIgnoringCopies(SelectMI->getOperand(1).getReg(), MRI);
if (CmpMI->getOpcode() == TargetOpcode::G_ICMP) {
- auto CC = CmpMI->getOperand(1).getPredicate();
- auto CReg = CmpMI->getOperand(3).getReg();
- auto CVal = getIConstantVRegValWithLookThrough(CReg, MRI);
+ unsigned CC = CmpMI->getOperand(1).getPredicate();
+ llvm::Register CReg = CmpMI->getOperand(3).getReg();
+ std::optional<llvm::ValueAndVReg> CVal = getIConstantVRegValWithLookThrough(CReg, MRI);
if (CVal.has_value() && (CVal->Value == 0) &&
((CC == CmpInst::ICMP_EQ) || (CC == CmpInst::ICMP_NE))) {
- auto CmpTy = MRI.getType(CmpMI->getOperand(2).getReg());
- auto SelTy = MRI.getType(SelectMI->getOperand(2).getReg());
+ llvm::LLT CmpTy = MRI.getType(CmpMI->getOperand(2).getReg());
+ llvm::LLT SelTy = MRI.getType(SelectMI->getOperand(2).getReg());
if ((CmpTy.isScalar() && !CmpTy.isPointer()) &&
(SelTy.isScalar() && !SelTy.isPointer()) &&
CmpTy.getScalarSizeInBits() == SelTy.getScalarSizeInBits()) {
@@ -271,11 +271,11 @@ bool PISAPostLegalizerCombinerImpl::matchCompareSelect(MachineInstr *MI) const {
return false;
}
void PISAPostLegalizerCombinerImpl::applyCompareSelect(MachineInstr *MI) const {
- auto *SelectMI = MI;
- auto *CmpMI = getDefIgnoringCopies(SelectMI->getOperand(1).getReg(), MRI);
- auto CC = CmpMI->getOperand(1).getPredicate();
- auto DstReg = SelectMI->getOperand(0).getReg();
- auto MIB = B.buildInstr(PISA::G_PISA_SELECT)
+ llvm::MachineInstr *SelectMI = MI;
+ llvm::MachineInstr *CmpMI = getDefIgnoringCopies(SelectMI->getOperand(1).getReg(), MRI);
+ unsigned CC = CmpMI->getOperand(1).getPredicate();
+ llvm::Register DstReg = SelectMI->getOperand(0).getReg();
+ llvm::MachineInstrBuilder MIB = B.buildInstr(PISA::G_PISA_SELECT)
.addDef(DstReg)
.add(CmpMI->getOperand(2));
if (CC == CmpInst::ICMP_NE)
@@ -315,7 +315,7 @@ bool PISAPostLegalizerCombinerImpl::matchFCmpInvertedCond(
if (BrCondMI->getOpcode() != TargetOpcode::G_BRCOND)
return false;
- auto *CmpMI = MRI.getVRegDef(BrCondMI->getOperand(0).getReg());
+ llvm::MachineInstr *CmpMI = MRI.getVRegDef(BrCondMI->getOperand(0).getReg());
Register FCC;
CmpInst::Predicate Pred;
@@ -343,11 +343,11 @@ bool PISAPostLegalizerCombinerImpl::matchFCmpInvertedCond(
return false;
// %9:_(s1) = G_FCMP floatpred(oge), %1:reg32b(s32), %2:reg32b
- auto *FCmpMI = MRI.getVRegDef(FCC);
+ llvm::MachineInstr *FCmpMI = MRI.getVRegDef(FCC);
if (FCmpMI->getOpcode() != TargetOpcode::G_FCMP)
return false;
- auto FCCTy = MRI.getType(FCC);
+ llvm::LLT FCCTy = MRI.getType(FCC);
if (!(FCCTy.isScalar() && FCCTy.getScalarSizeInBits() == 1))
return false;
@@ -359,8 +359,8 @@ void PISAPostLegalizerCombinerImpl::applyFCmpInvertedCond(
MachineInstr *BrCond;
Register FCC;
std::tie(BrCond, FCC) = MatchInfo;
- auto *BrCondBB = BrCond->getOperand(1).getMBB();
- auto *BrBB = MI.getOperand(0).getMBB();
+ llvm::MachineBasicBlock *BrCondBB = BrCond->getOperand(1).getMBB();
+ llvm::MachineBasicBlock *BrBB = MI.getOperand(0).getMBB();
Observer.changingInstr(MI);
MI.getOperand(0).setMBB(BrCondBB);
@@ -386,13 +386,13 @@ bool PISAPostLegalizerCombinerImpl::matchShlAddToMad(
m_GAdd(m_GShl(m_Reg(MulReg), m_ICst(Shift)), m_Reg(AddReg))))
return false;
- auto Ty = MRI.getType(MulReg);
+ llvm::LLT Ty = MRI.getType(MulReg);
if (Shift.getZExtValue() >= Ty.getSizeInBits())
return false;
MatchInfo = [Ty, &AddMI, AddReg, MulReg,
Shift = std::move(Shift)](MachineIRBuilder &B) {
- auto MulOp2 = B.buildConstant(
+ llvm::MachineInstrBuilder MulOp2 = B.buildConstant(
Ty, APInt::getOneBitSet(Ty.getSizeInBits(), Shift.getZExtValue()));
B.buildIntrinsic(Intrinsic::pisa_smad, AddMI.getOperand(0).getReg())
.addUse(MulReg)
@@ -440,7 +440,7 @@ bool PISAPostLegalizerCombinerImpl::matchFDivToRcpFMul(
uint16_t Flags = MI.getFlags();
MatchInfo = [=](MachineIRBuilder &B) {
- auto Rcp = B.buildIntrinsic(Intrinsic::pisa_frcp, {DstTy})
+ llvm::MachineInstrBuilder Rcp = B.buildIntrinsic(Intrinsic::pisa_frcp, {DstTy})
.addUse(Src1Reg)
.setMIFlags(Flags);
@@ -452,13 +452,17 @@ bool PISAPostLegalizerCombinerImpl::matchFDivToRcpFMul(
void PISAPostLegalizerCombinerImpl::applyShiftOfConstants(
MachineInstr *MI) const {
- auto [SrcMI, SrcRegIdx] =
+ MachineInstr *SrcMI;
+ unsigned SrcRegIdx;
+ std::tie(SrcMI, SrcRegIdx) =
PISA::getDefIgnoringBitcasts(MI->getOperand(1).getReg(), MRI);
- auto [ShiftMI, ShiftRegIdx] =
+ MachineInstr *ShiftMI;
+ unsigned ShiftRegIdx;
+ std::tie(ShiftMI, ShiftRegIdx) =
PISA::getDefIgnoringBitcasts(MI->getOperand(2).getReg(), MRI);
- const auto *CImm = SrcMI->getOperand(1).getCImm();
- auto Shift = ShiftMI->getOperand(1).getCImm()->getZExtValue();
+ const llvm::ConstantInt *CImm = SrcMI->getOperand(1).getCImm();
+ uint64_t Shift = ShiftMI->getOperand(1).getCImm()->getZExtValue();
int64_t NewVal = 0;
switch (MI->getOpcode()) {
default:
@@ -485,19 +489,19 @@ void PISAPostLegalizerCombinerImpl::applyShiftOfConstants(
// => E(32) = COPY ARG(32)
bool PISAPostLegalizerCombinerImpl::matchRedundantMovesPost(
MachineInstr &MI) const {
- auto &BitcastMI = MI;
- auto DstReg = BitcastMI.getOperand(0).getReg();
- auto SrcReg = BitcastMI.getOperand(1).getReg();
+ llvm::MachineInstr &BitcastMI = MI;
+ llvm::Register DstReg = BitcastMI.getOperand(0).getReg();
+ llvm::Register SrcReg = BitcastMI.getOperand(1).getReg();
if (MRI.getType(DstReg).isVector() || !MRI.getType(SrcReg).isVector())
return false;
- auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::MachineInstr &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
if (BuildVecMI.getOpcode() != TargetOpcode::G_BUILD_VECTOR)
return false;
Register UnmergeReg = 0;
for (unsigned I = 1; I < BuildVecMI.getNumOperands(); I++) {
- auto &UnmergeMI =
+ llvm::MachineInstr &UnmergeMI =
*getDefIgnoringCopies(BuildVecMI.getOperand(I).getReg(), MRI);
if (UnmergeMI.getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
return false;
@@ -510,16 +514,16 @@ bool PISAPostLegalizerCombinerImpl::matchRedundantMovesPost(
return false;
}
// order to extracted operands should match
- auto VecEltReg = BuildVecMI.getOperand(I).getReg();
+ llvm::Register VecEltReg = BuildVecMI.getOperand(I).getReg();
if (UnmergeMI.getOperand(I - 1).getReg() != VecEltReg)
return false;
}
- auto &SrcBitcastMI = *getDefIgnoringCopies(UnmergeReg, MRI);
+ llvm::MachineInstr &SrcBitcastMI = *getDefIgnoringCopies(UnmergeReg, MRI);
if (SrcBitcastMI.getOpcode() != TargetOpcode::G_BITCAST)
return false;
- auto SrcBitcastReg = SrcBitcastMI.getOperand(1).getReg();
+ llvm::Register SrcBitcastReg = SrcBitcastMI.getOperand(1).getReg();
if (MRI.getType(DstReg).getSizeInBits() !=
MRI.getType(SrcBitcastReg).getSizeInBits())
return false;
@@ -528,15 +532,15 @@ bool PISAPostLegalizerCombinerImpl::matchRedundantMovesPost(
void PISAPostLegalizerCombinerImpl::applyRedundantMovesPost(
MachineInstr &MI) const {
- auto &BitcastMI = MI;
- auto DstReg = BitcastMI.getOperand(0).getReg();
- auto SrcReg = BitcastMI.getOperand(1).getReg();
- auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
- auto &UnmergeMI =
+ llvm::MachineInstr &BitcastMI = MI;
+ llvm::Register DstReg = BitcastMI.getOperand(0).getReg();
+ llvm::Register SrcReg = BitcastMI.getOperand(1).getReg();
+ llvm::MachineInstr &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::MachineInstr &UnmergeMI =
*getDefIgnoringCopies(BuildVecMI.getOperand(1).getReg(), MRI);
- auto &SrcBitcastMI = *getDefIgnoringCopies(
+ llvm::MachineInstr &SrcBitcastMI = *getDefIgnoringCopies(
UnmergeMI.getOperand(UnmergeMI.getNumOperands() - 1).getReg(), MRI);
- auto SrcBitcastReg = SrcBitcastMI.getOperand(1).getReg();
+ llvm::Register SrcBitcastReg = SrcBitcastMI.getOperand(1).getReg();
B.buildCopy(DstReg, SrcBitcastReg);
MI.eraseFromParent();
}
@@ -554,9 +558,9 @@ void PISAPostLegalizerCombinerImpl::applyRedundantMovesPost(
// => Y(<N x s16>) = COPY ARG(<N x s16>)
bool PISAPostLegalizerCombinerImpl::matchExtractAllToBuildVector(
MachineInstr &MI, Register &Replacement) const {
- auto &BuildVecMI = MI;
- auto DstReg = BuildVecMI.getOperand(0).getReg();
- auto SrcReg = BuildVecMI.getOperand(1).getReg();
+ llvm::MachineInstr &BuildVecMI = MI;
+ llvm::Register DstReg = BuildVecMI.getOperand(0).getReg();
+ llvm::Register SrcReg = BuildVecMI.getOperand(1).getReg();
if (MRI.getType(SrcReg).isVector())
return false;
@@ -566,8 +570,8 @@ bool PISAPostLegalizerCombinerImpl::matchExtractAllToBuildVector(
Register SrcVecReg;
bool AllExtracts = true;
for (unsigned I = 1; I < BuildVecMI.getNumOperands(); I++) {
- auto EltReg = BuildVecMI.getOperand(I).getReg();
- auto &ExtractMI = *getDefIgnoringCopies(EltReg, MRI);
+ llvm::Register EltReg = BuildVecMI.getOperand(I).getReg();
+ llvm::MachineInstr &ExtractMI = *getDefIgnoringCopies(EltReg, MRI);
if (ExtractMI.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT) {
AllExtracts = false;
break;
@@ -580,8 +584,8 @@ bool PISAPostLegalizerCombinerImpl::matchExtractAllToBuildVector(
break;
}
- auto IndexReg = ExtractMI.getOperand(2).getReg();
- auto CValue = getIConstantVRegValWithLookThrough(IndexReg, MRI);
+ llvm::Register IndexReg = ExtractMI.getOperand(2).getReg();
+ std::optional<llvm::ValueAndVReg> CValue = getIConstantVRegValWithLookThrough(IndexReg, MRI);
if (!CValue.has_value() || CValue->Value != (I - 1)) {
AllExtracts = false;
break;
@@ -597,21 +601,21 @@ bool PISAPostLegalizerCombinerImpl::matchExtractAllToBuildVector(
// Branch B: all operands are the sequential defs of a single G_UNMERGE_VALUES
// whose source vector type matches the G_BUILD_VECTOR dest type.
unsigned NumElts = BuildVecMI.getNumOperands() - 1;
- auto FirstDefAndReg =
+ std::optional<llvm::DefinitionAndSourceRegister> FirstDefAndReg =
getDefSrcRegIgnoringCopies(BuildVecMI.getOperand(1).getReg(), MRI);
if (!FirstDefAndReg)
return false;
- auto *UnmergeMI = FirstDefAndReg->MI;
+ llvm::MachineInstr *UnmergeMI = FirstDefAndReg->MI;
if (UnmergeMI->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
return false;
if (UnmergeMI->getNumOperands() - 1 != NumElts)
return false; // G_UNMERGE_VALUES produces a different number of elements
- auto UnmergeSrcReg = UnmergeMI->getOperand(NumElts).getReg();
+ llvm::Register UnmergeSrcReg = UnmergeMI->getOperand(NumElts).getReg();
if (MRI.getType(DstReg) != MRI.getType(UnmergeSrcReg))
return false;
for (unsigned I = 0; I < NumElts; I++) {
- auto DefAndReg =
+ std::optional<llvm::DefinitionAndSourceRegister> DefAndReg =
getDefSrcRegIgnoringCopies(BuildVecMI.getOperand(I + 1).getReg(), MRI);
if (!DefAndReg || DefAndReg->MI != UnmergeMI)
return false; // different producer
@@ -624,8 +628,8 @@ bool PISAPostLegalizerCombinerImpl::matchExtractAllToBuildVector(
}
void PISAPostLegalizerCombinerImpl::applyExtractAllToBuildVector(
MachineInstr &MI, Register Replacement) const {
- auto &BuildVecMI = MI;
- auto DstReg = BuildVecMI.getOperand(0).getReg();
+ llvm::MachineInstr &BuildVecMI = MI;
+ llvm::Register DstReg = BuildVecMI.getOperand(0).getReg();
B.buildCopy(DstReg, Replacement);
MI.eraseFromParent();
}
@@ -640,7 +644,7 @@ bool PISAPostLegalizerCombinerImpl::matchOrAndToBfi(
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
// BFI supports only 32 bits of width, start with this simple check to
// exclude unsupported shifts early.
- auto BitWidth = MRI.getType(OrMI.getOperand(0).getReg()).getSizeInBits();
+ llvm::TypeSize BitWidth = MRI.getType(OrMI.getOperand(0).getReg()).getSizeInBits();
if (BitWidth != 32)
return false;
@@ -665,7 +669,9 @@ bool PISAPostLegalizerCombinerImpl::matchOrAndToBfi(
return false;
}
- auto [ShiftedDataMI, ShiftedDataRegIdx] =
+ MachineInstr *ShiftedDataMI;
+ unsigned ShiftedDataRegIdx;
+ std::tie(ShiftedDataMI, ShiftedDataRegIdx) =
PISA::getDefIgnoringBitcasts(ShiftedData, MRI);
Register Data;
@@ -674,8 +680,8 @@ bool PISAPostLegalizerCombinerImpl::matchOrAndToBfi(
return false;
MatchInfo = [Data, Base, Width, Offset, &OrMI](MachineIRBuilder &B) {
- auto WidthReg = B.buildConstant(I32, Width);
- auto OffsetReg = B.buildConstant(I32, Offset);
+ llvm::MachineInstrBuilder WidthReg = B.buildConstant(I32, Width);
+ llvm::MachineInstrBuilder OffsetReg = B.buildConstant(I32, Offset);
B.buildIntrinsic(Intrinsic::pisa_bfi, {OrMI.getOperand(0)})
.addUse(Base)
.addUse(Data)
@@ -695,12 +701,14 @@ bool PISAPostLegalizerCombinerImpl::matchShiftSubToBfe(
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
// BFE supports only 32 bits of width, start with this simple check to
// exclude unsupported shifts early.
- auto BitWidth = MRI.getType(ShrMI.getOperand(0).getReg()).getSizeInBits();
+ llvm::TypeSize BitWidth = MRI.getType(ShrMI.getOperand(0).getReg()).getSizeInBits();
if (BitWidth != 32)
return false;
- auto MakeApplyHandler = [&ShrMI](Register Data, Register Width,
- Register Offset) {
+ std::function<std::function<void(MachineIRBuilder &)>(Register, Register,
+ Register)>
+ MakeApplyHandler = [&ShrMI](Register Data, Register Width,
+ Register Offset) {
return [&ShrMI, Data, Width, Offset](MachineIRBuilder &B) {
B.buildIntrinsic(Intrinsic::pisa_ubfe, {ShrMI.getOperand(0)})
.addUse(Data)
@@ -769,11 +777,11 @@ bool PISAPostLegalizerCombinerImpl::matchShiftSubToBfe(
// =>
// s32 %andwidth = G_AND i32 %width, 31
// bfe %data %andwidth %offset
- auto WidthType = MRI.getType(Width);
+ llvm::LLT WidthType = MRI.getType(Width);
MatchInfo = [Data, Width, WidthType, Offset,
&ShrMI](MachineIRBuilder &B) {
- auto Mask = B.buildConstant(WidthType, ShiftInstMask);
- auto MaskedWidth = B.buildAnd(WidthType, Width, Mask);
+ llvm::MachineInstrBuilder Mask = B.buildConstant(WidthType, ShiftInstMask);
+ llvm::MachineInstrBuilder MaskedWidth = B.buildAnd(WidthType, Width, Mask);
B.buildIntrinsic(Intrinsic::pisa_ubfe, {ShrMI.getOperand(0)})
.addUse(Data)
.addUse(MaskedWidth->getOperand(0).getReg())
@@ -795,7 +803,7 @@ bool PISAPostLegalizerCombinerImpl::matchAndBitfieldToBitfield(
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
Register Data, Width, Offset;
Intrinsic::ID II;
- if (auto *MIIntrinsic = dyn_cast<GIntrinsic>(&MI)) {
+ if (llvm::GIntrinsic *MIIntrinsic = dyn_cast<GIntrinsic>(&MI)) {
II = MIIntrinsic->getIntrinsicID();
if (II == Intrinsic::pisa_ubfe) {
Data = MI.getOperand(2).getReg();
@@ -872,7 +880,7 @@ bool PISAPostLegalizerCombinerImpl::matchAndBitfieldToBitfield(
bool PISAPostLegalizerCombinerImpl::matchV2i1ZextToBfeBfi(
MachineInstr &BitcastMI, Register &BitcastInput) const {
- auto VectorElementSize =
+ unsigned VectorElementSize =
MRI.getType(BitcastMI.getOperand(0).getReg()).getScalarSizeInBits();
// We only support an i8 zext
if (VectorElementSize != 8)
@@ -910,8 +918,8 @@ bool PISAPostLegalizerCombinerImpl::matchV2i1ZextToBfeBfi(
code example), and one unmerge instruction with both values being used
(i.e. UpperUnmerge =?= LowerUnmerge)
*/
- auto *UpperUnmergeInstr = MRI.getUniqueVRegDef(UpperCmpRes);
- auto *LowerUnmergeInstr = MRI.getUniqueVRegDef(LowerCmpRes);
+ llvm::MachineInstr *UpperUnmergeInstr = MRI.getUniqueVRegDef(UpperCmpRes);
+ llvm::MachineInstr *LowerUnmergeInstr = MRI.getUniqueVRegDef(LowerCmpRes);
if (!UpperUnmergeInstr || !LowerUnmergeInstr ||
UpperUnmergeInstr->getOpcode() != TargetOpcode::G_UNMERGE_VALUES ||
LowerUnmergeInstr->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
@@ -923,12 +931,12 @@ bool PISAPostLegalizerCombinerImpl::matchV2i1ZextToBfeBfi(
if (LowerUnmergeInstr->getOperand(1).getReg() != LowerCmpRes)
return false;
- auto BitcastRes = UpperUnmergeInstr->getOperand(2);
+ llvm::MachineOperand BitcastRes = UpperUnmergeInstr->getOperand(2);
if (!BitcastRes.isIdenticalTo(LowerUnmergeInstr->getOperand(2)))
return false;
// Verify that this inst is a G_BITCAST, and get the input register
- auto *BitcastInst = MRI.getUniqueVRegDef(BitcastRes.getReg());
+ llvm::MachineInstr *BitcastInst = MRI.getUniqueVRegDef(BitcastRes.getReg());
if (!BitcastInst || BitcastInst->getOpcode() != TargetOpcode::G_BITCAST)
return false;
@@ -939,15 +947,15 @@ bool PISAPostLegalizerCombinerImpl::matchV2i1ZextToBfeBfi(
void PISAPostLegalizerCombinerImpl::applyV2i1ZextToBfeBfi(
MachineInstr &BitcastMI, Register BitcastInput) const {
- auto *MRI = B.getMRI();
- auto Bfe1Res = MRI->createGenericVirtualRegister(I32);
- auto Bfe2Res = MRI->createGenericVirtualRegister(I32);
- auto BfiRes = MRI->createGenericVirtualRegister(I32);
+ llvm::MachineRegisterInfo *MRI = B.getMRI();
+ llvm::Register Bfe1Res = MRI->createGenericVirtualRegister(I32);
+ llvm::Register Bfe2Res = MRI->createGenericVirtualRegister(I32);
+ llvm::Register BfiRes = MRI->createGenericVirtualRegister(I32);
- auto Zero = B.buildConstant(I32, 0);
- auto One = B.buildConstant(I32, 1);
- auto Eight = B.buildConstant(I32, 8);
- auto Sixteen = B.buildConstant(I32, 16);
+ llvm::MachineInstrBuilder Zero = B.buildConstant(I32, 0);
+ llvm::MachineInstrBuilder One = B.buildConstant(I32, 1);
+ llvm::MachineInstrBuilder Eight = B.buildConstant(I32, 8);
+ llvm::MachineInstrBuilder Sixteen = B.buildConstant(I32, 16);
B.buildIntrinsic(Intrinsic::pisa_ubfe, {DstOp(Bfe1Res)})
.addUse(BitcastInput)
@@ -985,7 +993,7 @@ void PISAPostLegalizerCombinerImpl::applyV2i1ZextToBfeBfi(
In all other cases, insert a G_BITCAST to <2 x s8>.
*/
- auto StoreInput = BitcastMI.getOperand(0).getReg();
+ llvm::Register StoreInput = BitcastMI.getOperand(0).getReg();
if (MRI->hasOneNonDBGUse(StoreInput) &&
MRI->use_instr_nodbg_begin(StoreInput)->getOpcode() ==
TargetOpcode::G_STORE) {
@@ -993,12 +1001,12 @@ void PISAPostLegalizerCombinerImpl::applyV2i1ZextToBfeBfi(
B.buildInstr(TargetOpcode::G_TRUNC).addDef(StoreInput).addUse(BfiRes);
MRI->setType(StoreInput, I16);
- for (auto *MemOp : MRI->use_instr_nodbg_begin(StoreInput)->memoperands())
+ for (llvm::MachineMemOperand *MemOp : MRI->use_instr_nodbg_begin(StoreInput)->memoperands())
MemOp->setType(I16);
} else {
// We either have multiple users or the following user is not a G_STORE
// Insert a bitcast to <2 x s8> so that nothing breaks
- auto TruncRes = MRI->createGenericVirtualRegister(I16);
+ llvm::Register TruncRes = MRI->createGenericVirtualRegister(I16);
B.buildInstr(TargetOpcode::G_TRUNC).addDef(TruncRes).addUse(BfiRes);
B.buildInstr(TargetOpcode::G_BITCAST).addDef(StoreInput).addUse(TruncRes);
@@ -1016,21 +1024,21 @@ void PISAPostLegalizerCombinerImpl::applyV2i1ZextToBfeBfi(
// => %405:_(s16) = COPY %375
bool PISAPostLegalizerCombinerImpl::matchAndSelect(
MachineInstr &MI, Register &Replacement) const {
- auto &SelectMI = MI;
+ llvm::MachineInstr &SelectMI = MI;
- auto Const0 =
+ std::optional<llvm::ValueAndVReg> Const0 =
getIConstantVRegValWithLookThrough(SelectMI.getOperand(3).getReg(), MRI);
- auto Const1 =
+ std::optional<llvm::ValueAndVReg> Const1 =
getIConstantVRegValWithLookThrough(SelectMI.getOperand(2).getReg(), MRI);
if (!Const0.has_value() || !Const1.has_value())
return false;
if ((Const0->Value != 0) || (Const1->Value != 1))
return false;
- auto &AndMI = *getDefIgnoringCopies(SelectMI.getOperand(1).getReg(), MRI);
+ llvm::MachineInstr &AndMI = *getDefIgnoringCopies(SelectMI.getOperand(1).getReg(), MRI);
if (AndMI.getOpcode() != TargetOpcode::G_AND)
return false;
- auto AndConst =
+ std::optional<llvm::ValueAndVReg> AndConst =
getIConstantVRegValWithLookThrough(AndMI.getOperand(2).getReg(), MRI);
if (!AndConst.has_value() || (AndConst->Value != 1))
return false;
@@ -1063,35 +1071,35 @@ void PISAPostLegalizerCombinerImpl::applyAndSelect(MachineInstr &MI,
// => %141 = G_OR %A, %B
bool PISAPostLegalizerCombinerImpl::matchBuildRegFrom2(
MachineInstr &MI, Reg2MatchInfo &MatchInfo) const {
- auto &BitcastMI = MI;
+ llvm::MachineInstr &BitcastMI = MI;
- auto DstTy = MRI.getType(BitcastMI.getOperand(0).getReg());
+ llvm::LLT DstTy = MRI.getType(BitcastMI.getOperand(0).getReg());
if (DstTy.isVector() || (DstTy.getScalarSizeInBits() != 32))
return false;
- auto SrcReg = BitcastMI.getOperand(1).getReg();
- auto SrcTy = MRI.getType(SrcReg);
+ llvm::Register SrcReg = BitcastMI.getOperand(1).getReg();
+ llvm::LLT SrcTy = MRI.getType(SrcReg);
if (!SrcTy.isVector() || (SrcTy.getScalarSizeInBits() != 8))
return false;
- auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::MachineInstr &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
if (BuildVecMI.getOpcode() != TargetOpcode::G_BUILD_VECTOR)
return false;
- for (auto I = 0; I < 2; I++) {
- auto LoReg = BuildVecMI.getOperand(1 + (I * 2)).getReg();
- auto HiReg = BuildVecMI.getOperand(2 + (I * 2)).getReg();
- auto &LoTruncMI = *getDefIgnoringCopies(LoReg, MRI);
- auto &HiTruncMI = *getDefIgnoringCopies(HiReg, MRI);
+ for (int I = 0; I < 2; I++) {
+ llvm::Register LoReg = BuildVecMI.getOperand(1 + (I * 2)).getReg();
+ llvm::Register HiReg = BuildVecMI.getOperand(2 + (I * 2)).getReg();
+ llvm::MachineInstr &LoTruncMI = *getDefIgnoringCopies(LoReg, MRI);
+ llvm::MachineInstr &HiTruncMI = *getDefIgnoringCopies(HiReg, MRI);
if (LoTruncMI.getOpcode() != TargetOpcode::G_TRUNC)
return false;
if (HiTruncMI.getOpcode() != TargetOpcode::G_TRUNC)
return false;
- auto &HiShiftMI =
+ llvm::MachineInstr &HiShiftMI =
*getDefIgnoringCopies(HiTruncMI.getOperand(1).getReg(), MRI);
if (HiShiftMI.getOpcode() != TargetOpcode::G_LSHR)
return false;
if (LoTruncMI.getOperand(1).getReg() != HiShiftMI.getOperand(1).getReg())
return false;
- auto ShiftConst = getIConstantVRegValWithLookThrough(
+ std::optional<llvm::ValueAndVReg> ShiftConst = getIConstantVRegValWithLookThrough(
HiShiftMI.getOperand(2).getReg(), MRI);
if (!ShiftConst.has_value() || (ShiftConst->Value != 8))
return false;
@@ -1104,15 +1112,15 @@ bool PISAPostLegalizerCombinerImpl::matchBuildRegFrom2(
}
void PISAPostLegalizerCombinerImpl::applyBuildRegFrom2(
MachineInstr &MI, Reg2MatchInfo &MatchInfo) const {
- auto MaskReg = MRI.createGenericVirtualRegister(LLT::integer(32));
- auto Mask = B.buildConstant(MaskReg, 0xFFFF);
- auto ShiftReg = MRI.createGenericVirtualRegister(LLT::integer(32));
- auto Shift = B.buildConstant(ShiftReg, 16);
+ llvm::Register MaskReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ llvm::MachineInstrBuilder Mask = B.buildConstant(MaskReg, 0xFFFF);
+ llvm::Register ShiftReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ llvm::MachineInstrBuilder Shift = B.buildConstant(ShiftReg, 16);
- auto AReg = MRI.createGenericVirtualRegister(LLT::integer(32));
- auto BReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ llvm::Register AReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ llvm::Register BReg = MRI.createGenericVirtualRegister(LLT::integer(32));
- auto RegSize = MRI.getType(MatchInfo.Reg0).getSizeInBits();
+ llvm::TypeSize RegSize = MRI.getType(MatchInfo.Reg0).getSizeInBits();
assert((RegSize == 16 || RegSize == 32 || RegSize == 64) &&
"only supporting 16/32/64bit registers");
Register ASrcReg, BSrcReg;
@@ -1154,7 +1162,8 @@ bool PISAPostLegalizerCombinerImpl::matchBuildVectorFromUnmergeLanes(
Register SrcReg;
int64_t BaseLane = -1;
for (unsigned I = 0; I < NumElts; ++I) {
- auto Def = getDefSrcRegIgnoringCopies(MI.getOperand(1 + I).getReg(), MRI);
+ std::optional<llvm::DefinitionAndSourceRegister> Def =
+ getDefSrcRegIgnoringCopies(MI.getOperand(1 + I).getReg(), MRI);
if (!Def)
return false;
const MachineInstr *D = Def->MI;
@@ -1201,7 +1210,8 @@ bool PISAPostLegalizerCombinerImpl::matchBuildVectorFromUnmergeLanes(
void PISAPostLegalizerCombinerImpl::applyBuildVectorFromUnmergeLanes(
MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {
- auto [SrcReg, BaseLane] = MatchInfo;
+ Register SrcReg = std::get<0>(MatchInfo);
+ int64_t BaseLane = std::get<1>(MatchInfo);
B.buildExtractSubvector(MI.getOperand(0).getReg(), SrcReg, BaseLane);
MI.eraseFromParent();
}
@@ -1224,7 +1234,7 @@ bool PISAPostLegalizerCombinerImpl::matchBuildVectorConcatSubvectors(
const MachineInstr *Unmerge = nullptr;
Register SrcReg;
for (unsigned J = 0; J < M; ++J) {
- auto Def = getDefSrcRegIgnoringCopies(
+ std::optional<llvm::DefinitionAndSourceRegister> Def = getDefSrcRegIgnoringCopies(
MI.getOperand(1 + G * M + J).getReg(), MRI);
if (!Def)
return false;
@@ -1281,7 +1291,7 @@ bool PISAPostLegalizerCombinerImpl::matchCmpAndAllOnes(
(Pred != CmpInst::ICMP_NE))
return false;
- auto BitWidth = MRI.getType(SrcReg).getSizeInBits();
+ llvm::TypeSize BitWidth = MRI.getType(SrcReg).getSizeInBits();
if (BitWidth < 16)
return false;
@@ -1338,7 +1348,7 @@ bool PISAPostLegalizerCombinerImpl::matchSinkTrunc(
return false;
// We don't want to create 64-bit boolean operations
- auto BitWidth = MRI.getType(SrcReg).getSizeInBits();
+ llvm::TypeSize BitWidth = MRI.getType(SrcReg).getSizeInBits();
if (BitWidth > 32)
return false;
@@ -1350,9 +1360,9 @@ void PISAPostLegalizerCombinerImpl::applySinkTrunc(
int64_t Mask;
Register SrcReg;
std::tie(SrcReg, Mask) = MatchInfo;
- auto SrcTy = MRI.getType(SrcReg);
- auto MaskReg = MRI.createGenericVirtualRegister(SrcTy);
- auto DstReg = MRI.createGenericVirtualRegister(SrcTy);
+ llvm::LLT SrcTy = MRI.getType(SrcReg);
+ llvm::Register MaskReg = MRI.createGenericVirtualRegister(SrcTy);
+ llvm::Register DstReg = MRI.createGenericVirtualRegister(SrcTy);
B.buildConstant(MaskReg, Mask);
B.buildInstr(MI.getOpcode()).addDef(DstReg).addUse(SrcReg).addUse(MaskReg);
B.buildInstr(TargetOpcode::G_TRUNC)
@@ -1396,7 +1406,7 @@ bool PISAPostLegalizerCombinerImpl::matchAbsRedMaxToRedAbsMax(
default:
return false;
case TargetOpcode::G_INTRINSIC_CONVERGENT: {
- auto II = cast<GIntrinsic>(MI).getIntrinsicID();
+ unsigned II = cast<GIntrinsic>(MI).getIntrinsicID();
if (II == Intrinsic::pisa_ired) {
if (MI.getOperand(2).getImm() != pisa::IRedOp::UMAX)
return false;
@@ -1415,7 +1425,9 @@ bool PISAPostLegalizerCombinerImpl::matchAbsRedMaxToRedAbsMax(
}
}
unsigned SrcOpIdx = IntrID ? 3 : 2;
- auto [SrcMI, SrcRegIdx] =
+ MachineInstr *SrcMI;
+ unsigned SrcRegIdx;
+ std::tie(SrcMI, SrcRegIdx) =
PISA::getDefIgnoringBitcasts(MI.getOperand(SrcOpIdx).getReg(), MRI);
MachineInstr *SrcMIPtr = SrcMI;
if (SrcMIPtr->getOpcode() != AbsOpcode)
@@ -1428,7 +1440,7 @@ bool PISAPostLegalizerCombinerImpl::matchAbsRedMaxToRedAbsMax(
if (IntrID)
MatchInfo = [SrcMIPtr, AbsSrcIdx, &MI, IntrID,
TargetOpType](MachineIRBuilder &B) {
- auto MIB = B.buildIntrinsic(IntrID, {MI.getOperand(0)});
+ llvm::MachineInstrBuilder MIB = B.buildIntrinsic(IntrID, {MI.getOperand(0)});
MIB.addImm(TargetOpType);
MIB.addUse(SrcMIPtr->getOperand(AbsSrcIdx).getReg());
MIB.addUse(MI.getOperand(4).getReg());
@@ -1469,9 +1481,9 @@ bool PISAPostLegalizerCombinerImpl::matchShiftTrueFalse(
// Result can be only all zeros or all ones. But if sources are also only
// all zeros/ones, then shl/ashr is a redundant artifact from legalization.
- auto DstReg = AShrMI.getOperand(0).getReg();
- auto DstTy = MRI.getType(DstReg);
- auto ShiftAmount = DstTy.getScalarSizeInBits() - 1;
+ llvm::Register DstReg = AShrMI.getOperand(0).getReg();
+ llvm::LLT DstTy = MRI.getType(DstReg);
+ unsigned ShiftAmount = DstTy.getScalarSizeInBits() - 1;
Register SrcReg;
if (!mi_match(AShrMI, MRI,
@@ -1480,11 +1492,11 @@ bool PISAPostLegalizerCombinerImpl::matchShiftTrueFalse(
return false;
std::function<bool(Register)> Match = [&](Register Reg) {
- auto CValue = getIConstantVRegValWithLookThrough(Reg, MRI);
+ std::optional<llvm::ValueAndVReg> CValue = getIConstantVRegValWithLookThrough(Reg, MRI);
if (CValue.has_value())
return CValue->Value.isZero() || CValue->Value.isAllOnes();
- auto *MI = getDefIgnoringCopies(Reg, MRI);
+ llvm::MachineInstr *MI = getDefIgnoringCopies(Reg, MRI);
if (!MI)
return false;
@@ -1510,7 +1522,7 @@ bool PISAPostLegalizerCombinerImpl::matchShiftTrueFalse(
MatchInfo = [DstReg, SrcReg, this](MachineIRBuilder &B) {
if (MRI.hasOneNonDBGUse(SrcReg)) {
- auto *MI = getDefIgnoringCopies(SrcReg, MRI);
+ llvm::MachineInstr *MI = getDefIgnoringCopies(SrcReg, MRI);
if (MI) {
MI->getOperand(0).setReg(DstReg);
return;
@@ -1536,26 +1548,35 @@ bool PISAPostLegalizerCombinerImpl::matchAddInt8Reduction(
MachineInstr &MI,
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
- auto Dst = MI.getOperand(0).getReg();
- auto DstTy = MRI.getType(Dst);
+ llvm::Register Dst = MI.getOperand(0).getReg();
+ llvm::LLT DstTy = MRI.getType(Dst);
if (!DstTy.isScalar() || DstTy.getSizeInBits() > 32)
return false;
std::array<Register, 4> Srcs;
// Helper to match anyext or zext of a register
- auto MatchExt = [](Register &Reg) {
- return m_any_of(m_GAnyExt(m_Reg(Reg)), m_GZExt(m_Reg(Reg)));
- };
if (!mi_match(MI, MRI,
- m_any_of(m_GAdd(m_OneNonDBGUse(m_GAdd(MatchExt(Srcs[0]),
- MatchExt(Srcs[1]))),
- m_OneNonDBGUse(m_GAdd(MatchExt(Srcs[2]),
- MatchExt(Srcs[3])))),
+ m_any_of(m_GAdd(
+ m_OneNonDBGUse(m_GAdd(
+ m_any_of(m_GAnyExt(m_Reg(Srcs[0])),
+ m_GZExt(m_Reg(Srcs[0]))),
+ m_any_of(m_GAnyExt(m_Reg(Srcs[1])),
+ m_GZExt(m_Reg(Srcs[1]))))),
+ m_OneNonDBGUse(m_GAdd(
+ m_any_of(m_GAnyExt(m_Reg(Srcs[2])),
+ m_GZExt(m_Reg(Srcs[2]))),
+ m_any_of(m_GAnyExt(m_Reg(Srcs[3])),
+ m_GZExt(m_Reg(Srcs[3])))))),
m_GAdd(m_OneNonDBGUse(m_GAdd(
- m_OneNonDBGUse(m_GAdd(MatchExt(Srcs[0]),
- MatchExt(Srcs[1]))),
- MatchExt(Srcs[2]))),
- MatchExt(Srcs[3])))))
+ m_OneNonDBGUse(m_GAdd(
+ m_any_of(m_GAnyExt(m_Reg(Srcs[0])),
+ m_GZExt(m_Reg(Srcs[0]))),
+ m_any_of(m_GAnyExt(m_Reg(Srcs[1])),
+ m_GZExt(m_Reg(Srcs[1]))))),
+ m_any_of(m_GAnyExt(m_Reg(Srcs[2])),
+ m_GZExt(m_Reg(Srcs[2]))))),
+ m_any_of(m_GAnyExt(m_Reg(Srcs[3])),
+ m_GZExt(m_Reg(Srcs[3])))))))
return false;
// All registers must be unique.
@@ -1564,8 +1585,8 @@ bool PISAPostLegalizerCombinerImpl::matchAddInt8Reduction(
return false;
// All register must be i8 from the same unmerge instruction.
- auto GetUnmerge = [=](Register &Reg) {
- auto *UnmergeMI = MRI.getVRegDef(Reg);
+ std::function<MachineInstr *(Register &)> GetUnmerge = [=](Register &Reg) {
+ llvm::MachineInstr *UnmergeMI = MRI.getVRegDef(Reg);
return UnmergeMI->getOpcode() == TargetOpcode::G_UNMERGE_VALUES ? UnmergeMI
: nullptr;
};
@@ -1578,16 +1599,16 @@ bool PISAPostLegalizerCombinerImpl::matchAddInt8Reduction(
return false;
}
- auto VectorReg =
+ llvm::Register VectorReg =
UnmergeMI->getOperand(UnmergeMI->getNumOperands() - 1).getReg();
if (MRI.getType(VectorReg) !=
LLT::vector(ElementCount::getFixed(4), LLT::integer(8)))
return false;
MatchInfo = [=](MachineIRBuilder &B) {
- auto Acc = MRI.createGenericVirtualRegister(I32);
- auto X = MRI.createGenericVirtualRegister(I32);
- auto Y = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Acc = MRI.createGenericVirtualRegister(I32);
+ llvm::Register X = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Y = MRI.createGenericVirtualRegister(I32);
B.buildConstant(Acc, 0);
B.buildBitcast(X, VectorReg);
@@ -1601,7 +1622,7 @@ bool PISAPostLegalizerCombinerImpl::matchAddInt8Reduction(
.addUse(Y)
.addImm(0);
} else {
- auto Dst32 = MRI.createGenericVirtualRegister(I32);
+ llvm::Register Dst32 = MRI.createGenericVirtualRegister(I32);
B.buildIntrinsic(Intrinsic::pisa_dp4a_uu, {DstOp(Dst32)})
.addUse(Acc)
.addUse(X)
@@ -1622,13 +1643,15 @@ bool PISAPostLegalizerCombinerImpl::matchUnmergeBitcastBuildVectorToBitcast(
MachineInstr &MI, Register &MatchInfo) const {
Register UnmergeSrc;
for (unsigned I = 1, E = MI.getNumOperands(); I < E; I++) {
- auto [SrcMI, SrcRegIdx] =
+ MachineInstr *SrcMI;
+ unsigned SrcRegIdx;
+ std::tie(SrcMI, SrcRegIdx) =
PISA::getDefIgnoringBitcasts(MI.getOperand(I).getReg(), MRI, true);
if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
return false;
if (SrcMI->getNumOperands() != MI.getNumOperands())
return false;
- auto SrcReg = SrcMI->getOperand(SrcMI->getNumOperands() - 1).getReg();
+ llvm::Register SrcReg = SrcMI->getOperand(SrcMI->getNumOperands() - 1).getReg();
if (SrcRegIdx != (I - 1))
return false;
if (I == 1)
@@ -1641,7 +1664,7 @@ bool PISAPostLegalizerCombinerImpl::matchUnmergeBitcastBuildVectorToBitcast(
}
void PISAPostLegalizerCombinerImpl::applyUnmergeBitcastBuildVectorToBitcast(
MachineInstr &MI, Register &MatchInfo) const {
- auto DstReg = MI.getOperand(0).getReg();
+ llvm::Register DstReg = MI.getOperand(0).getReg();
if (MRI.getType(DstReg) != MRI.getType(MatchInfo))
B.buildBitcast(MI.getOperand(0).getReg(), MatchInfo);
else
@@ -1653,11 +1676,12 @@ void PISAPostLegalizerCombinerImpl::applyUnmergeBitcastBuildVectorToBitcast(
// These are execution barriers, not memory ordering fences, and should
// be preserved.
static bool isSubgroupFence(const MachineInstr &MI) {
- auto ScopeID = static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
+ SyncScope::ID ScopeID =
+ static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
if (ScopeID == SyncScope::SingleThread)
return true;
- auto &Ctx = MI.getMF()->getFunction().getContext();
- if (auto Name = Ctx.getSyncScopeName(ScopeID))
+ llvm::LLVMContext &Ctx = MI.getMF()->getFunction().getContext();
+ if (std::optional<llvm::StringRef> Name = Ctx.getSyncScopeName(ScopeID))
return Name->starts_with("subgroup") || Name->starts_with("workitem");
return false;
}
@@ -1666,9 +1690,10 @@ static bool isSubgroupFence(const MachineInstr &MI) {
// include an address space suffix (e.g. "workgroup-shared", "workgroup-global",
// "workgroup-generic", or just "workgroup").
static StringRef getBaseScopeName(const MachineInstr &MI) {
- auto ScopeID = static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
- auto &Ctx = MI.getMF()->getFunction().getContext();
- auto ScopeName = Ctx.getSyncScopeName(ScopeID);
+ SyncScope::ID ScopeID =
+ static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
+ llvm::LLVMContext &Ctx = MI.getMF()->getFunction().getContext();
+ std::optional<llvm::StringRef> ScopeName = Ctx.getSyncScopeName(ScopeID);
if (!ScopeName)
return StringRef();
@@ -1687,7 +1712,7 @@ bool PISAPostLegalizerCombinerImpl::matchRedundantFence(
unsigned Scope = MI.getOperand(1).getImm();
- auto It = MI.getIterator();
+ MachineBasicBlock::iterator It = MI.getIterator();
const MachineBasicBlock &MBB = *MI.getParent();
while (It != MBB.begin()) {
--It;
@@ -1712,13 +1737,13 @@ void PISAPostLegalizerCombinerImpl::applyRedundantFence(
MachineInstr &MI, MachineInstr *&PrevFence) const {
assert(PrevFence && "Expected a preceding compatible fence");
- auto CurOrd = static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
- auto PrevOrd = static_cast<AtomicOrdering>(PrevFence->getOperand(0).getImm());
+ llvm::AtomicOrdering CurOrd = static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
+ llvm::AtomicOrdering PrevOrd = static_cast<AtomicOrdering>(PrevFence->getOperand(0).getImm());
// Pick the stronger ordering (e.g. choose seq_cst over acquire).
// Acquire and release cannot be compared; use acq_rel for the
// resulting fence.
- auto Merged = getMergedAtomicOrdering(PrevOrd, CurOrd);
+ llvm::AtomicOrdering Merged = getMergedAtomicOrdering(PrevOrd, CurOrd);
PrevFence->getOperand(0).setImm(static_cast<int64_t>(Merged));
MI.eraseFromParent();
}
@@ -1731,7 +1756,7 @@ bool PISAPostLegalizerCombinerImpl::matchExtractSubvectorBuildVector(
assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
if (!MI.getOperand(2).isImm())
return false;
- auto *BV = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ llvm::MachineInstr *BV = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
if (!BV || BV->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
return false;
uint64_t Offset = MI.getOperand(2).getImm();
@@ -1765,24 +1790,24 @@ bool PISAPostLegalizerCombinerImpl::matchMergeAdjacentFences(
if (isSubgroupFence(MI))
return false;
- auto Order = static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
- auto BaseScopeNameA = getBaseScopeName(MI);
+ llvm::AtomicOrdering Order = static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
+ llvm::StringRef BaseScopeNameA = getBaseScopeName(MI);
if (BaseScopeNameA.empty())
return false;
- auto It = MI.getIterator();
+ MachineBasicBlock::iterator It = MI.getIterator();
const MachineBasicBlock &MBB = *MI.getParent();
while (It != MBB.begin()) {
--It;
if (It->getOpcode() == TargetOpcode::G_FENCE) {
- auto ItOrder = static_cast<AtomicOrdering>(It->getOperand(0).getImm());
+ llvm::AtomicOrdering ItOrder = static_cast<AtomicOrdering>(It->getOperand(0).getImm());
if (ItOrder != Order)
return false;
if (It->getOperand(1).getImm() == MI.getOperand(1).getImm())
return false;
- auto BaseScopeNameIt = getBaseScopeName(*It);
+ llvm::StringRef BaseScopeNameIt = getBaseScopeName(*It);
if (BaseScopeNameIt.empty() || BaseScopeNameIt != BaseScopeNameA)
return false;
@@ -1804,8 +1829,8 @@ void PISAPostLegalizerCombinerImpl::applyMergeAdjacentFences(
StringRef BaseScopeName = getBaseScopeName(*PrevFence);
assert(!BaseScopeName.empty() && "Expected a named sync scope");
std::string GenericName = (BaseScopeName + "-generic").str();
- auto &Ctx = MI.getMF()->getFunction().getContext();
- auto GenericID = Ctx.getOrInsertSyncScopeID(GenericName);
+ llvm::LLVMContext &Ctx = MI.getMF()->getFunction().getContext();
+ SyncScope::ID GenericID = Ctx.getOrInsertSyncScopeID(GenericName);
PrevFence->getOperand(1).setImm(static_cast<int64_t>(GenericID));
MI.eraseFromParent();
@@ -1901,7 +1926,8 @@ bool PISAPostLegalizerCombinerImpl::matchMinMaxIdentityFold(
MI.getOpcode() == TargetOpcode::G_SMAX);
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
- auto IsIdentity = [&](const APInt &Val) -> bool {
+ std::function<bool(const APInt &)> IsIdentity =
+ [&](const APInt &Val) -> bool {
switch (MI.getOpcode()) {
case TargetOpcode::G_UMIN:
return Val.isAllOnes();
@@ -1915,12 +1941,12 @@ bool PISAPostLegalizerCombinerImpl::matchMinMaxIdentityFold(
return false;
}
};
- if (auto C = getIConstantVRegValWithLookThrough(RHS, MRI))
+ if (std::optional<llvm::ValueAndVReg> C = getIConstantVRegValWithLookThrough(RHS, MRI))
if (IsIdentity(C->Value)) {
MatchInfo = LHS;
return true;
}
- if (auto C = getIConstantVRegValWithLookThrough(LHS, MRI))
+ if (std::optional<llvm::ValueAndVReg> C = getIConstantVRegValWithLookThrough(LHS, MRI))
if (IsIdentity(C->Value)) {
MatchInfo = RHS;
return true;
diff --git a/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp b/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
index 7a9cf7a4b3205..292652c0ed21d 100644
--- a/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
@@ -174,11 +174,11 @@ bool PISAPreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
}
void PISAPreLegalizerCombinerImpl::applyTruncatedLoad(MachineInstr &MI) const {
- auto *LoadMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
- auto *MMO = LoadMI->memoperands()[0];
- auto Dst = MI.getOperand(0);
- auto Addr = LoadMI->getOperand(1);
- auto *NewMMO = MI.getMF()->getMachineMemOperand(MMO, MMO->getOffset(),
+ llvm::MachineInstr *LoadMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ llvm::MachineMemOperand *MMO = LoadMI->memoperands()[0];
+ llvm::MachineOperand Dst = MI.getOperand(0);
+ llvm::MachineOperand Addr = LoadMI->getOperand(1);
+ llvm::MachineMemOperand *NewMMO = MI.getMF()->getMachineMemOperand(MMO, MMO->getOffset(),
MRI.getType(Dst.getReg()));
B.buildLoad(Dst, Addr, *NewMMO);
MI.eraseFromParent();
@@ -190,7 +190,7 @@ bool PISAPreLegalizerCombinerImpl::matchExpandNonPowerOf2LoadStore(
MI.getOpcode() == TargetOpcode::G_STORE);
GLoadStore &LS = cast<GLoadStore>(MI);
- auto Size = LS.getMemSizeInBits().getValue();
+ llvm::TypeSize Size = LS.getMemSizeInBits().getValue();
if (isPowerOf2_32(Size))
return false;
@@ -201,8 +201,8 @@ bool PISAPreLegalizerCombinerImpl::matchExpandNonPowerOf2LoadStore(
// Check if this is a load and only has zext uses that are handled by the
// extended load pattern - if yes, we want to handle it via said pattern.
if (MI.getOpcode() == TargetOpcode::G_LOAD) {
- for (auto &Use : MRI.use_operands(MI.getOperand(0).getReg())) {
- auto *Inst = Use.getParent();
+ for (llvm::MachineOperand &Use : MRI.use_operands(MI.getOperand(0).getReg())) {
+ llvm::MachineInstr *Inst = Use.getParent();
if (Inst->getOpcode() != TargetOpcode::G_ZEXT ||
!matchExtendedLoad(*Inst))
return true;
@@ -219,9 +219,9 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
GLoadStore &LS = cast<GLoadStore>(MI);
- auto PointerReg = LS.getPointerReg();
- auto ValueReg = LS.getOperand(0).getReg();
- auto &MMO = LS.getMMO();
+ llvm::Register PointerReg = LS.getPointerReg();
+ llvm::Register ValueReg = LS.getOperand(0).getReg();
+ llvm::MachineMemOperand &MMO = LS.getMMO();
/// The remaining size in Bits that still has to be loaded/stored
ssize_t Size = LS.getMemSizeInBits().getValue();
@@ -230,14 +230,14 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
Size = (Size + 7) & ~7;
unsigned Offset = 0;
- const auto SizeTy = LLT::integer(Size);
+ const llvm::LLT SizeTy = LLT::integer(Size);
// If the size was changed to the next power of 8 and we are storing a value,
// we need to modify the register's type as well.
// A similar check is needed for loads, but this is done at the end
if (MI.getOpcode() == TargetOpcode::G_STORE &&
SizeTy != MRI.getType(ValueReg)) {
- auto NewValueReg = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register NewValueReg = MRI.createGenericVirtualRegister(SizeTy);
B.buildZExt(NewValueReg, ValueReg);
ValueReg = NewValueReg;
}
@@ -245,24 +245,24 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
/// The register holding the loaded value at the end
Register LoadRes;
while (Size > 0) {
- auto OpSize = bit_floor(static_cast<size_t>(Size));
- const auto ShiftAmount = Offset * 8;
+ uint64_t OpSize = bit_floor(static_cast<size_t>(Size));
+ const unsigned ShiftAmount = Offset * 8;
const LLT OpTy = LLT::integer(OpSize);
- auto *NewMMO =
+ llvm::MachineMemOperand *NewMMO =
MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset() + Offset, OpTy);
/// Stores the (potentially modified) pointer register
- auto AddrReg = PointerReg;
+ llvm::Register AddrReg = PointerReg;
// We might need to change the store offset
if (Offset != 0) {
- auto NewAddr = MRI.cloneVirtualRegister(AddrReg);
+ llvm::Register NewAddr = MRI.cloneVirtualRegister(AddrReg);
// Get the pointer size from the pointer register type
const LLT PtrTy = MRI.getType(AddrReg);
const LLT IntTy = LLT::integer(PtrTy.getSizeInBits());
- auto Const = MRI.createGenericVirtualRegister(IntTy);
+ llvm::Register Const = MRI.createGenericVirtualRegister(IntTy);
B.buildConstant(Const, Offset);
B.buildPtrAdd(NewAddr, AddrReg, Const);
@@ -270,11 +270,11 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
}
if (MI.getOpcode() == TargetOpcode::G_STORE) {
- auto Res = ValueReg;
+ llvm::Register Res = ValueReg;
// If we're not storing from the start, we need to shift our value first
if (Offset != 0) {
- auto ShrRes = MRI.createGenericVirtualRegister(SizeTy);
- auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register ShrRes = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
B.buildConstant(ShiftConst, ShiftAmount);
B.buildLShr(ShrRes, ValueReg, ShiftConst);
Res = ShrRes;
@@ -282,7 +282,7 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
// Next, truncate it to the correct size, if needed
if (SizeTy != OpTy) {
- auto TruncRes = MRI.createGenericVirtualRegister(OpTy);
+ llvm::Register TruncRes = MRI.createGenericVirtualRegister(OpTy);
B.buildTrunc(TruncRes, Res);
Res = TruncRes;
}
@@ -291,22 +291,22 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
} else {
// When loading, do the same thing but basically in reverse
// First, load the value
- auto LoadedValReg = MRI.createGenericVirtualRegister(OpTy);
+ llvm::Register LoadedValReg = MRI.createGenericVirtualRegister(OpTy);
B.buildLoad(LoadedValReg, AddrReg, *NewMMO);
- auto Res = LoadedValReg;
+ llvm::Register Res = LoadedValReg;
// Extend it to the correct size, if needed
if (SizeTy != OpTy) {
- auto ZextRes = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register ZextRes = MRI.createGenericVirtualRegister(SizeTy);
B.buildZExt(ZextRes, LoadedValReg);
Res = ZextRes;
}
// If we're not loading the first Bytes, then we need to shift it
if (Offset != 0) {
- auto ShiftRes = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register ShiftRes = MRI.createGenericVirtualRegister(SizeTy);
- auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
B.buildConstant(ShiftConst, ShiftAmount);
B.buildShl(ShiftRes, Res, ShiftConst);
@@ -318,7 +318,7 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
B.buildConstant(LoadRes, 0);
}
- auto NewLoadRes = MRI.createGenericVirtualRegister(SizeTy);
+ llvm::Register NewLoadRes = MRI.createGenericVirtualRegister(SizeTy);
B.buildOr(NewLoadRes, LoadRes, Res);
LoadRes = NewLoadRes;
}
@@ -332,7 +332,7 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
// not a multiple of 8), we need to truncate it to the correct size again,
// in order not to break any following instructions
if (SizeTy != MRI.getType(ValueReg)) {
- auto TruncRes = MRI.createGenericVirtualRegister(MRI.getType(ValueReg));
+ llvm::Register TruncRes = MRI.createGenericVirtualRegister(MRI.getType(ValueReg));
B.buildTrunc(TruncRes, LoadRes);
LoadRes = TruncRes;
}
@@ -387,7 +387,7 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
MachineInstr *&SizeModificationOp) const {
GLoadStore &StoreInst = cast<GLoadStore>(MI);
- auto ValueReg = StoreInst.getOperand(0).getReg();
+ llvm::Register ValueReg = StoreInst.getOperand(0).getReg();
/// This has the integer size at the beginning, and each individual load
/// decreases the value by its load size. At the end, this must be zero.
@@ -468,7 +468,7 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
if (PtrAddInst->getOpcode() != TargetOpcode::G_PTR_ADD)
return false;
- auto LoadOffset =
+ std::optional<llvm::APInt> LoadOffset =
getIConstantVRegVal(PtrAddInst->getOperand(2).getReg(), MRI);
if (!LoadOffset.has_value() || LoadOffset.value() != LoadShift / 8)
return false;
@@ -487,14 +487,14 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
LoadOffset.value().getZExtValue())});
} else if (NextChainInst->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
NextChainInst->getOpcode() == TargetOpcode::G_LOAD) {
- const auto &LoadInst = cast<GLoadStore>(*NextChainInst);
+ const llvm::GLoadStore &LoadInst = cast<GLoadStore>(*NextChainInst);
const LLT LoadTy = LoadInst.getMMO().getType();
if (!LoadTy.isScalar())
return false;
// This is the last load instruction of the chain, so it must match the
// remaining value in ValueSize
- auto LoadSize = LoadTy.getScalarSizeInBits();
+ unsigned LoadSize = LoadTy.getScalarSizeInBits();
if (ValueSize % 8 == 0 && LoadSize != ValueSize)
return false;
@@ -510,7 +510,7 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
SizeModificationOp->getOpcode() == TargetOpcode::G_TRUNC)
return false;
- auto SizeAfterModificationOp =
+ unsigned SizeAfterModificationOp =
MRI.getType(SizeModificationOp->getOperand(0).getReg())
.getScalarSizeInBits();
@@ -526,7 +526,7 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
// same G_SEXT(G_LOAD) pattern, causing an infinite loop. When the
// source size is not byte-aligned, the apply inserts in-place
// shl/ashr to align it first, producing a different pattern.
- auto SextSrcSize =
+ unsigned SextSrcSize =
MRI.getType(SizeModificationOp->getOperand(1).getReg())
.getScalarSizeInBits();
if (LoadSize == SextSrcSize && SextSrcSize % 8 == 0)
@@ -556,13 +556,15 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
// multiple G_STOREs using the individually loaded values
GLoadStore &StoreInst = cast<GLoadStore>(MI);
- auto PointerReg = StoreInst.getPointerReg();
- auto &MMO = StoreInst.getMMO();
+ llvm::Register PointerReg = StoreInst.getPointerReg();
+ llvm::MachineMemOperand &MMO = StoreInst.getMMO();
unsigned StoreSizeInBytes = StoreInst.getMemSize().getValue();
- for (auto [Index, Pair] : enumerate(Loads)) {
- auto [LoadMI, Offset] = Pair;
+ for (size_t Index = 0; Index < Loads.size(); ++Index) {
+ std::pair<MachineInstr *, unsigned> &Pair = Loads[Index];
+ MachineInstr *LoadMI = Pair.first;
+ unsigned Offset = Pair.second;
Register Value = LoadMI->getOperand(0).getReg();
LLT LoadTy = cast<GLoadStore>(LoadMI)->getMMO().getType();
@@ -581,7 +583,7 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
// load was extended, so we need to sext/zext this value before
// storing
if (Index == 0 && SizeModificationOp != nullptr) {
- auto Opcode = SizeModificationOp->getOpcode();
+ unsigned Opcode = SizeModificationOp->getOpcode();
assert(Opcode == TargetOpcode::G_ZEXT ||
Opcode == TargetOpcode::G_SEXT ||
Opcode == TargetOpcode::G_SEXT_INREG);
@@ -599,22 +601,22 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
// First, do we have to sign extend this value in-place using
// shifts?
if (OriginalSize % 8 != 0) {
- auto ExtendBy = 8 - (OriginalSize % 8);
+ unsigned ExtendBy = 8 - (OriginalSize % 8);
- auto ConstReg = MRI.createGenericVirtualRegister(LLT::integer(32));
+ llvm::Register ConstReg = MRI.createGenericVirtualRegister(LLT::integer(32));
B.buildConstant(ConstReg, ExtendBy);
if (LoadTy.getSizeInBits() < OriginalSize + ExtendBy) {
LoadTy = LLT::integer(OriginalSize + ExtendBy);
- auto ZExtRes = MRI.createGenericVirtualRegister(LoadTy);
+ llvm::Register ZExtRes = MRI.createGenericVirtualRegister(LoadTy);
B.buildZExt(ZExtRes, Res);
Res = ZExtRes;
}
- auto ShlRes = MRI.cloneVirtualRegister(Res);
+ llvm::Register ShlRes = MRI.cloneVirtualRegister(Res);
B.buildShl(ShlRes, Res, ConstReg);
- auto AShrRes = MRI.cloneVirtualRegister(ShlRes);
+ llvm::Register AShrRes = MRI.cloneVirtualRegister(ShlRes);
B.buildAShr(AShrRes, ShlRes, ConstReg);
Res = AShrRes;
@@ -625,18 +627,18 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
// Second, do we still have to SEXT it further?
if (StoreSizeInBits > OriginalSize) {
LoadTy = LLT::integer(StoreSizeInBits);
- auto SextRes = MRI.createGenericVirtualRegister(LoadTy);
+ llvm::Register SextRes = MRI.createGenericVirtualRegister(LoadTy);
B.buildSExt(SextRes, Res);
Res = SextRes;
}
} else {
// G_ZEXT, simple
- auto NewSize = StoreSizeInBytes - Offset;
+ unsigned NewSize = StoreSizeInBytes - Offset;
assert(NewSize > LoadTy.getSizeInBytes());
LoadTy = LLT::integer(NewSize * 8);
- auto ExtRes = MRI.createGenericVirtualRegister(LoadTy);
+ llvm::Register ExtRes = MRI.createGenericVirtualRegister(LoadTy);
B.buildZExt(ExtRes, Res);
Res = ExtRes;
@@ -657,21 +659,21 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
B.buildTrunc(Res, Value);
}
- auto *NewMMO = MI.getMF()->getMachineMemOperand(
+ llvm::MachineMemOperand *NewMMO = MI.getMF()->getMachineMemOperand(
&MMO, MMO.getOffset() + Offset, LoadTy);
/// Stores the (potentially modified) pointer register
- auto AddrReg = PointerReg;
+ llvm::Register AddrReg = PointerReg;
// Add the offset to the pointer reg if the offset is not zero
if (Offset != 0) {
- auto NewPointerReg =
+ llvm::Register NewPointerReg =
MRI.createGenericVirtualRegister(MRI.getType(AddrReg));
// Get the pointer size from the pointer register type
const LLT PtrTy = MRI.getType(AddrReg);
const LLT IntTy = LLT::integer(PtrTy.getSizeInBits());
- auto CstReg = MRI.createGenericVirtualRegister(IntTy);
+ llvm::Register CstReg = MRI.createGenericVirtualRegister(IntTy);
B.buildConstant(CstReg, Offset);
B.buildPtrAdd(NewPointerReg, AddrReg, CstReg);
@@ -689,8 +691,8 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
// (s24) = G_TRUNC (s32)
// G_STORE (s24), ptr
bool PISAPreLegalizerCombinerImpl::matchTruncatedStore(MachineInstr &MI) const {
- auto &Store = cast<GStore>(MI);
- auto *TruncMI = getDefIgnoringCopies(Store.getValueReg(), MRI);
+ llvm::GStore &Store = cast<GStore>(MI);
+ llvm::MachineInstr *TruncMI = getDefIgnoringCopies(Store.getValueReg(), MRI);
if (!TruncMI)
return false;
if (TruncMI->getOpcode() != TargetOpcode::G_TRUNC)
@@ -699,17 +701,17 @@ bool PISAPreLegalizerCombinerImpl::matchTruncatedStore(MachineInstr &MI) const {
MRI.getType(TruncMI->getOperand(1).getReg()).getSizeInBits()))
return false;
- auto Size = Store.getMemSizeInBits().getValue();
- auto Align = Store.getMMO().getAlign().value();
+ llvm::TypeSize Size = Store.getMemSizeInBits().getValue();
+ uint64_t Align = Store.getMMO().getAlign().value();
if (isPowerOf2_32(Size))
return false;
- auto NextPow2 = NextPowerOf2(Size);
- auto TruncSize = MRI.getType(TruncMI->getOperand(1).getReg()).getSizeInBits();
+ uint64_t NextPow2 = NextPowerOf2(Size);
+ llvm::TypeSize TruncSize = MRI.getType(TruncMI->getOperand(1).getReg()).getSizeInBits();
if (TruncSize != NextPow2)
return false;
- auto Remainder = NextPow2 - Size;
+ uint64_t Remainder = NextPow2 - Size;
if (NextPow2 % Remainder != 0)
return false;
@@ -721,7 +723,7 @@ bool PISAPreLegalizerCombinerImpl::matchTruncatedStore(MachineInstr &MI) const {
if (Align % (Remainder / 8) != 0)
return false;
- auto VecSize = NextPow2 / Remainder;
+ uint64_t VecSize = NextPow2 / Remainder;
if (VecSize > 4)
return false;
@@ -731,27 +733,27 @@ bool PISAPreLegalizerCombinerImpl::matchTruncatedStore(MachineInstr &MI) const {
// Changes the type of a non-power-of-2 store to a vector of smaller elements,
// if it comes from a trunc instruction.
void PISAPreLegalizerCombinerImpl::applyTruncatedStore(MachineInstr &MI) const {
- auto &Store = cast<GStore>(MI);
+ llvm::GStore &Store = cast<GStore>(MI);
- auto *TruncMI = getDefIgnoringCopies(Store.getValueReg(), MRI);
+ llvm::MachineInstr *TruncMI = getDefIgnoringCopies(Store.getValueReg(), MRI);
assert(TruncMI && TruncMI->getOpcode() == TargetOpcode::G_TRUNC);
- auto TruncVal = TruncMI->getOperand(1);
- auto Size = Store.getMemSizeInBits().getValue();
- auto NextPow2 = NextPowerOf2(Size);
- auto Remainder = NextPow2 - Size;
- auto VecSize = NextPow2 / Remainder;
- auto VecSizeSmall = Size / Remainder;
- auto VecType = LLT::fixed_vector(VecSize, LLT::integer(Remainder));
- auto VecTypeSmall = LLT::fixed_vector(VecSizeSmall, LLT::integer(Remainder));
- auto TmpDst = MRI.createGenericVirtualRegister(VecType);
- auto TmpDstSmall = MRI.createGenericVirtualRegister(VecTypeSmall);
+ llvm::MachineOperand TruncVal = TruncMI->getOperand(1);
+ llvm::TypeSize Size = Store.getMemSizeInBits().getValue();
+ uint64_t NextPow2 = NextPowerOf2(Size);
+ uint64_t Remainder = NextPow2 - Size;
+ uint64_t VecSize = NextPow2 / Remainder;
+ uint64_t VecSizeSmall = Size / Remainder;
+ llvm::LLT VecType = LLT::fixed_vector(VecSize, LLT::integer(Remainder));
+ llvm::LLT VecTypeSmall = LLT::fixed_vector(VecSizeSmall, LLT::integer(Remainder));
+ llvm::Register TmpDst = MRI.createGenericVirtualRegister(VecType);
+ llvm::Register TmpDstSmall = MRI.createGenericVirtualRegister(VecTypeSmall);
B.buildBitcast(TmpDst, TruncVal);
B.buildShuffleVector(TmpDstSmall, TmpDst, B.buildUndef(VecType), {0, 1, 2});
- auto Addr = Store.getPointerReg();
- auto &MMO = Store.getMMO();
- auto *NewMMO =
+ llvm::Register Addr = Store.getPointerReg();
+ llvm::MachineMemOperand &MMO = Store.getMMO();
+ llvm::MachineMemOperand *NewMMO =
MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset(), VecTypeSmall);
B.buildStore(TmpDstSmall, Addr, *NewMMO);
MI.eraseFromParent();
@@ -762,20 +764,20 @@ void PISAPreLegalizerCombinerImpl::applyTruncatedStore(MachineInstr &MI) const {
// This saves us from using second load instruction and few arithmetic
// instructions (shl, and, or) to zero extend.
bool PISAPreLegalizerCombinerImpl::matchExtendedLoad(MachineInstr &MI) const {
- auto *DefMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ llvm::MachineInstr *DefMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
if (!DefMI)
return false;
if (DefMI->getOpcode() != TargetOpcode::G_LOAD)
return false;
GLoad &Load = cast<GLoad>(*DefMI);
- auto Size = Load.getMemSizeInBits().getValue();
- auto Align = Load.getMMO().getAlign().value();
+ llvm::TypeSize Size = Load.getMemSizeInBits().getValue();
+ uint64_t Align = Load.getMMO().getAlign().value();
if (isPowerOf2_32(Size))
return false;
- auto ZextSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ llvm::TypeSize ZextSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
if (!isPowerOf2_32(ZextSize))
return false;
@@ -793,21 +795,21 @@ void PISAPreLegalizerCombinerImpl::applyExtendedLoad(MachineInstr &MI) const {
cast<GLoad>(getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI));
assert(LoadMI);
- auto DestSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
- auto LoadSize = LoadMI->getMemSizeInBits().getValue();
+ llvm::TypeSize DestSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ llvm::TypeSize LoadSize = LoadMI->getMemSizeInBits().getValue();
APInt Mask = APInt::getLowBitsSet(DestSize, LoadSize);
- auto NewLoadSize = LLT::integer(DestSize);
- auto LoadDst = MRI.createGenericVirtualRegister(NewLoadSize);
+ llvm::LLT NewLoadSize = LLT::integer(DestSize);
+ llvm::Register LoadDst = MRI.createGenericVirtualRegister(NewLoadSize);
- auto Addr = LoadMI->getOperand(1);
- auto &MMO = LoadMI->getMMO();
- auto *NewMMO = MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset(),
+ llvm::MachineOperand Addr = LoadMI->getOperand(1);
+ llvm::MachineMemOperand &MMO = LoadMI->getMMO();
+ llvm::MachineMemOperand *NewMMO = MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset(),
MRI.getType(LoadDst));
B.buildLoad(LoadDst, Addr, *NewMMO);
- auto MaskReg = MRI.createGenericVirtualRegister(LLT::integer(DestSize));
+ llvm::Register MaskReg = MRI.createGenericVirtualRegister(LLT::integer(DestSize));
B.buildConstant(MaskReg, Mask.getZExtValue());
B.buildAnd(MI.getOperand(0).getReg(), LoadDst, MaskReg);
MI.eraseFromParent();
@@ -820,19 +822,20 @@ void PISAPreLegalizerCombinerImpl::applyExtendedLoad(MachineInstr &MI) const {
// => %lo = G_EXTRACT_VECTOR_ELT <2 x 16> %1, 0
// => %hi = G_EXTRACT_VECTOR_ELT <2 x 16> %1, 1
bool PISAPreLegalizerCombinerImpl::matchTruncatedShift(MachineInstr &MI) const {
- auto &TruncMI = MI;
- auto &LshMI = *getDefIgnoringCopies(TruncMI.getOperand(1).getReg(), MRI);
+ llvm::MachineInstr &TruncMI = MI;
+ llvm::MachineInstr &LshMI = *getDefIgnoringCopies(TruncMI.getOperand(1).getReg(), MRI);
if (LshMI.getOpcode() == TargetOpcode::G_LSHR) {
- auto WideTy = MRI.getType(TruncMI.getOperand(1).getReg());
- auto NarrowTy = MRI.getType(TruncMI.getOperand(0).getReg());
- auto WideSize = WideTy.getScalarSizeInBits();
- auto NarrowSize = NarrowTy.getScalarSizeInBits();
+ llvm::LLT WideTy = MRI.getType(TruncMI.getOperand(1).getReg());
+ llvm::LLT NarrowTy = MRI.getType(TruncMI.getOperand(0).getReg());
+ unsigned WideSize = WideTy.getScalarSizeInBits();
+ unsigned NarrowSize = NarrowTy.getScalarSizeInBits();
if (WideTy.isScalar() && NarrowTy.isScalar()) {
if (isPowerOf2_32(WideSize) && isPowerOf2_32(NarrowSize)) {
- auto ShiftReg = LshMI.getOperand(2).getReg();
- if (auto Shift = getIConstantVRegValWithLookThrough(ShiftReg, MRI)) {
+ llvm::Register ShiftReg = LshMI.getOperand(2).getReg();
+ if (std::optional<llvm::ValueAndVReg> Shift =
+ getIConstantVRegValWithLookThrough(ShiftReg, MRI)) {
if (Shift.has_value()) {
- auto ShiftVal = Shift->Value.getZExtValue();
+ uint64_t ShiftVal = Shift->Value.getZExtValue();
if ((NarrowSize + ShiftVal) == WideSize)
return true;
}
@@ -843,34 +846,34 @@ bool PISAPreLegalizerCombinerImpl::matchTruncatedShift(MachineInstr &MI) const {
return false;
}
void PISAPreLegalizerCombinerImpl::applyTruncatedShift(MachineInstr &MI) const {
- auto &TruncMI = MI;
- auto &LshMI = *getDefIgnoringCopies(TruncMI.getOperand(1).getReg(), MRI);
- auto WideTy = MRI.getType(TruncMI.getOperand(1).getReg());
- auto NarrowTy = MRI.getType(TruncMI.getOperand(0).getReg());
- auto WideSize = WideTy.getScalarSizeInBits();
- auto NarrowSize = NarrowTy.getScalarSizeInBits();
+ llvm::MachineInstr &TruncMI = MI;
+ llvm::MachineInstr &LshMI = *getDefIgnoringCopies(TruncMI.getOperand(1).getReg(), MRI);
+ llvm::LLT WideTy = MRI.getType(TruncMI.getOperand(1).getReg());
+ llvm::LLT NarrowTy = MRI.getType(TruncMI.getOperand(0).getReg());
+ unsigned WideSize = WideTy.getScalarSizeInBits();
+ unsigned NarrowSize = NarrowTy.getScalarSizeInBits();
- auto VecLen = WideSize / NarrowSize;
- auto VecTy = LLT::fixed_vector(VecLen, NarrowTy);
- auto VecReg = MRI.createGenericVirtualRegister(VecTy);
+ unsigned VecLen = WideSize / NarrowSize;
+ llvm::LLT VecTy = LLT::fixed_vector(VecLen, NarrowTy);
+ llvm::Register VecReg = MRI.createGenericVirtualRegister(VecTy);
// trunc to extract high part
- auto BitCast = B.buildBitcast(VecReg, LshMI.getOperand(1));
+ llvm::MachineInstrBuilder BitCast = B.buildBitcast(VecReg, LshMI.getOperand(1));
B.buildExtractVectorElementConstant(TruncMI.getOperand(0), VecReg,
VecLen - 1);
// find trunc to extract low part (if any)
- auto ShiftSrc = LshMI.getOperand(1).getReg();
- for (auto &UseMI : MRI.use_instructions(ShiftSrc)) {
+ llvm::Register ShiftSrc = LshMI.getOperand(1).getReg();
+ for (llvm::MachineInstr &UseMI : MRI.use_instructions(ShiftSrc)) {
if (UseMI.getOpcode() == TargetOpcode::G_TRUNC) {
- auto DstSize = MRI.getType(UseMI.getOperand(0).getReg()).getSizeInBits();
+ llvm::TypeSize DstSize = MRI.getType(UseMI.getOperand(0).getReg()).getSizeInBits();
if (DstSize == NarrowSize) {
assert(MDT && "machine dominator pass must be available");
if (!MDT->dominates(&UseMI, &TruncMI) &&
!MDT->dominates(&TruncMI, &UseMI))
continue; // can not optimize out low part
MachineIRBuilder MIB(UseMI);
- auto Lo = MIB.buildExtractVectorElementConstant(UseMI.getOperand(0),
+ llvm::MachineInstrBuilder Lo = MIB.buildExtractVectorElementConstant(UseMI.getOperand(0),
VecReg, 0);
if (MDT->dominates(&UseMI, &TruncMI))
BitCast->moveBefore(Lo);
@@ -888,20 +891,20 @@ void PISAPreLegalizerCombinerImpl::applyTruncatedShift(MachineInstr &MI) const {
// => D(32) = COPY ARG(32)
bool PISAPreLegalizerCombinerImpl::matchRedundantMovesPre(
MachineInstr &MI) const {
- auto &BitcastMI = MI;
- auto DstReg = BitcastMI.getOperand(0).getReg();
- auto SrcReg = BitcastMI.getOperand(1).getReg();
+ llvm::MachineInstr &BitcastMI = MI;
+ llvm::Register DstReg = BitcastMI.getOperand(0).getReg();
+ llvm::Register SrcReg = BitcastMI.getOperand(1).getReg();
if (MRI.getType(DstReg).isVector() || !MRI.getType(SrcReg).isVector())
return false;
- auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::MachineInstr &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
if (BuildVecMI.getOpcode() != TargetOpcode::G_BUILD_VECTOR)
return false;
unsigned Mask = 0;
Register VecReg = 0;
for (unsigned I = 1; I < BuildVecMI.getNumOperands(); I++) {
- auto &ExtractMI =
+ llvm::MachineInstr &ExtractMI =
*getDefIgnoringCopies(BuildVecMI.getOperand(I).getReg(), MRI);
if (ExtractMI.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
return false;
@@ -911,8 +914,8 @@ bool PISAPreLegalizerCombinerImpl::matchRedundantMovesPre(
// must extract indices from the same vector
return false;
}
- auto IndexReg = ExtractMI.getOperand(2).getReg();
- auto Index = getIConstantVRegValWithLookThrough(IndexReg, MRI);
+ llvm::Register IndexReg = ExtractMI.getOperand(2).getReg();
+ std::optional<llvm::ValueAndVReg> Index = getIConstantVRegValWithLookThrough(IndexReg, MRI);
if (!Index.has_value())
return false;
// indices must be in the same order
@@ -933,13 +936,13 @@ bool PISAPreLegalizerCombinerImpl::matchRedundantMovesPre(
}
void PISAPreLegalizerCombinerImpl::applyRedundantMovesPre(
MachineInstr &MI) const {
- auto &BitcastMI = MI;
- auto DstReg = BitcastMI.getOperand(0).getReg();
- auto SrcReg = BitcastMI.getOperand(1).getReg();
- auto &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
- auto &ExtractMI =
+ llvm::MachineInstr &BitcastMI = MI;
+ llvm::Register DstReg = BitcastMI.getOperand(0).getReg();
+ llvm::Register SrcReg = BitcastMI.getOperand(1).getReg();
+ llvm::MachineInstr &BuildVecMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::MachineInstr &ExtractMI =
*getDefIgnoringCopies(BuildVecMI.getOperand(1).getReg(), MRI);
- auto VecReg = ExtractMI.getOperand(1).getReg();
+ llvm::Register VecReg = ExtractMI.getOperand(1).getReg();
if (MRI.getType(DstReg) == MRI.getType(VecReg))
B.buildCopy(DstReg, VecReg);
@@ -956,7 +959,7 @@ static MachineInstr *getReciprocalDivisor(MachineInstr *MI,
// Check for 1.0 / x pattern. Avoid m_GFCstOrSplat because
// getFConstantVRegValWithLookThrough loses bfloat16 semantics.
Register Dividend = MI->getOperand(1).getReg();
- auto *DivDefMI = getDefIgnoringCopies(Dividend, MRI);
+ llvm::MachineInstr *DivDefMI = getDefIgnoringCopies(Dividend, MRI);
if (!DivDefMI || DivDefMI->getOpcode() != TargetOpcode::G_FCONSTANT)
return nullptr;
if (!DivDefMI->getOperand(1).getFPImm()->isExactlyValue(1.0))
@@ -964,7 +967,7 @@ static MachineInstr *getReciprocalDivisor(MachineInstr *MI,
return getDefIgnoringCopies(MI->getOperand(2).getReg(), MRI);
}
- auto *GI = dyn_cast<GIntrinsic>(MI);
+ llvm::GIntrinsic *GI = dyn_cast<GIntrinsic>(MI);
if (GI && GI->is(Intrinsic::pisa_frcp))
return getDefIgnoringCopies(MI->getOperand(2).getReg(), MRI);
@@ -980,12 +983,14 @@ static MachineInstr *getReciprocalDivisor(MachineInstr *MI,
bool PISAPreLegalizerCombinerImpl::matchRcpSqrtToRsqrt(
MachineInstr &MI,
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
- auto GetFrcpSrc = [=](MachineInstr *MI) -> MachineInstr * {
+ std::function<MachineInstr *(MachineInstr *)> GetFrcpSrc =
+ [=](MachineInstr *MI) -> MachineInstr * {
if (!MI || !MI->getFlag(MachineInstr::FmContract))
return nullptr;
return getReciprocalDivisor(MI, MRI);
};
- auto GetSqrtSrc = [=](MachineInstr *MI) -> MachineInstr * {
+ std::function<MachineInstr *(MachineInstr *)> GetSqrtSrc =
+ [=](MachineInstr *MI) -> MachineInstr * {
if (!MI || !MI->getFlag(MachineInstr::FmContract))
return nullptr;
if (MI->getOpcode() == TargetOpcode::G_FSQRT)
@@ -998,7 +1003,7 @@ bool PISAPreLegalizerCombinerImpl::matchRcpSqrtToRsqrt(
if (Divisor) {
// result of sqrt cannot be snan, so Intrinsic::fabs is replaced with
// Intrinsic::pisa_fabs. We won't get G_FABS here.
- auto *GI = dyn_cast<GIntrinsic>(Divisor);
+ llvm::GIntrinsic *GI = dyn_cast<GIntrinsic>(Divisor);
if (GI && GI->is(Intrinsic::pisa_fabs)) {
WrapInFAbs = true;
Divisor = getDefIgnoringCopies(GI->getOperand(2).getReg(), MRI);
@@ -1023,7 +1028,7 @@ bool PISAPreLegalizerCombinerImpl::matchRcpSqrtToRsqrt(
return;
}
// 1/|sqrt(x)| == |frsqrt(x)|
- auto Frsqrt = B.buildIntrinsic(Intrinsic::pisa_frsqrt, {DstTy})
+ llvm::MachineInstrBuilder Frsqrt = B.buildIntrinsic(Intrinsic::pisa_frsqrt, {DstTy})
.addUse(Src)
.setMIFlags(MI.getFlags());
B.buildIntrinsic(Intrinsic::pisa_fabs, {MI.getOperand(0)})
@@ -1045,10 +1050,10 @@ bool PISAPreLegalizerCombinerImpl::matchSubFloorToFrc(
if (IsPlainFSub) {
XReg = MI.getOperand(1).getReg();
FloorReg = MI.getOperand(2).getReg();
- } else if (auto *GI = dyn_cast<GIntrinsic>(&MI);
+ } else if (llvm::GIntrinsic *GI = dyn_cast<GIntrinsic>(&MI);
GI && GI->is(Intrinsic::pisa_fsub)) {
unsigned NumOps = MI.getNumOperands();
- auto Round = static_cast<RoundingMode>(MI.getOperand(NumOps - 2).getImm());
+ llvm::RoundingMode Round = static_cast<RoundingMode>(MI.getOperand(NumOps - 2).getImm());
int64_t Sat = MI.getOperand(NumOps - 1).getImm();
if (Round != RoundingMode::TowardZero || Sat)
return false;
@@ -1058,7 +1063,7 @@ bool PISAPreLegalizerCombinerImpl::matchSubFloorToFrc(
return false;
}
- auto BitWidth = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+ llvm::TypeSize BitWidth = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
if (BitWidth != 32)
return false;
@@ -1107,18 +1112,18 @@ bool PISAPreLegalizerCombinerImpl::matchLaneIdLeftShiftChain(
return false;
if (!mi_match(ZExt, MRI, m_GZExt(m_GShl(m_MInstr(Intr), m_ICst(ShiftImm2)))))
return false;
- auto *LaneId = dyn_cast<GIntrinsic>(Intr);
+ llvm::GIntrinsic *LaneId = dyn_cast<GIntrinsic>(Intr);
if (!LaneId || LaneId->getIntrinsicID() != Intrinsic::pisa_lane_id)
return false;
- auto LaneIdTy = MRI.getType(LaneId->getOperand(0).getReg());
+ llvm::LLT LaneIdTy = MRI.getType(LaneId->getOperand(0).getReg());
constexpr int MaxNumBitsInLaneId = 5;
if ((ShiftImm1 + ShiftImm2) >=
(static_cast<int>(LaneIdTy.getSizeInBits()) - MaxNumBitsInLaneId - 1))
return false;
MatchInfo = [=, &MI](MachineIRBuilder &B) {
- auto NewShlReg = MRI.createGenericVirtualRegister(LaneIdTy);
+ llvm::Register NewShlReg = MRI.createGenericVirtualRegister(LaneIdTy);
B.buildShl(NewShlReg, LaneId->getOperand(0),
B.buildConstant(LaneIdTy, ShiftImm1 + ShiftImm2), MI.getFlags());
B.buildZExt(MI.getOperand(0), NewShlReg, ZExt->getFlags());
@@ -1134,8 +1139,8 @@ bool PISAPreLegalizerCombinerImpl::matchZExtAndToAndZExt(
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
Register ZExtReg = MI.getOperand(1).getReg();
Register MaskReg = MI.getOperand(2).getReg();
- auto *ZExtMI = getDefIgnoringCopies(ZExtReg, MRI);
- auto Mask = getIConstantVRegValWithLookThrough(MaskReg, MRI);
+ llvm::MachineInstr *ZExtMI = getDefIgnoringCopies(ZExtReg, MRI);
+ std::optional<llvm::ValueAndVReg> Mask = getIConstantVRegValWithLookThrough(MaskReg, MRI);
if ((!ZExtMI || ZExtMI->getOpcode() != TargetOpcode::G_ZEXT ||
!Mask.has_value())) {
@@ -1159,8 +1164,8 @@ bool PISAPreLegalizerCombinerImpl::matchZExtAndToAndZExt(
APInt NarrowMask = Mask->Value.zextOrTrunc(SrcTy.getSizeInBits());
unsigned ZExtFlags = ZExtMI->getFlags();
MatchInfo = [=](MachineIRBuilder &B) {
- auto NarrowMaskReg = B.buildConstant(SrcTy, NarrowMask);
- auto NarrowAnd = B.buildAnd(SrcTy, SrcReg, NarrowMaskReg);
+ llvm::MachineInstrBuilder NarrowMaskReg = B.buildConstant(SrcTy, NarrowMask);
+ llvm::MachineInstrBuilder NarrowAnd = B.buildAnd(SrcTy, SrcReg, NarrowMaskReg);
B.buildZExt(DstReg, NarrowAnd, ZExtFlags);
};
return true;
@@ -1179,63 +1184,63 @@ bool PISAPreLegalizerCombinerImpl::matchZExtAndToAndZExt(
// - bits of s32 (typeof(BaseReg)) written to by above - return value
static uint64_t getCoveredBits(Register BaseReg, Register &SrcVecReg,
uint64_t *SrcVecIdx, MachineRegisterInfo &MRI) {
- auto &SrcMI = *getDefIgnoringCopies(BaseReg, MRI);
+ llvm::MachineInstr &SrcMI = *getDefIgnoringCopies(BaseReg, MRI);
switch (SrcMI.getOpcode()) {
case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
- auto VecReg = SrcMI.getOperand(1).getReg();
+ llvm::Register VecReg = SrcMI.getOperand(1).getReg();
if (MRI.getType(VecReg).getNumElements() > 64)
return 0; // not supported
- auto IdxReg = SrcMI.getOperand(2).getReg();
+ llvm::Register IdxReg = SrcMI.getOperand(2).getReg();
if (SrcVecReg && (SrcVecReg != VecReg))
return 0; // extracting from different source vector ?
SrcVecReg = VecReg;
- auto Index = getIConstantVRegValWithLookThrough(IdxReg, MRI);
+ std::optional<llvm::ValueAndVReg> Index = getIConstantVRegValWithLookThrough(IdxReg, MRI);
if (!Index.has_value())
return 0; // not a constant
*SrcVecIdx |= (1ull << Index->Value.getZExtValue());
return ~0;
} break;
case TargetOpcode::G_ZEXT: {
- auto SrcReg = SrcMI.getOperand(1).getReg();
- auto &ExtractMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::Register SrcReg = SrcMI.getOperand(1).getReg();
+ llvm::MachineInstr &ExtractMI = *getDefIgnoringCopies(SrcReg, MRI);
if (ExtractMI.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
return 0;
if (getCoveredBits(SrcReg, SrcVecReg, SrcVecIdx, MRI))
return (1ull << MRI.getType(SrcReg).getScalarSizeInBits()) - 1;
} break;
case TargetOpcode::G_SHL: {
- auto SrcReg = SrcMI.getOperand(1).getReg();
- auto ShiftReg = SrcMI.getOperand(2).getReg();
- auto &ExtMI = *getDefIgnoringCopies(SrcReg, MRI);
+ llvm::Register SrcReg = SrcMI.getOperand(1).getReg();
+ llvm::Register ShiftReg = SrcMI.getOperand(2).getReg();
+ llvm::MachineInstr &ExtMI = *getDefIgnoringCopies(SrcReg, MRI);
if (ExtMI.getOpcode() != TargetOpcode::G_ZEXT)
return 0;
- auto Shift = getIConstantVRegValWithLookThrough(ShiftReg, MRI);
+ std::optional<llvm::ValueAndVReg> Shift = getIConstantVRegValWithLookThrough(ShiftReg, MRI);
if (!Shift.has_value())
return 0; // not a constant
- auto ShiftValue = Shift->Value.getZExtValue();
+ uint64_t ShiftValue = Shift->Value.getZExtValue();
// make sure we are not swapping bits
- auto OldSrcVecIdx = *SrcVecIdx;
- auto Bits = getCoveredBits(SrcReg, SrcVecReg, SrcVecIdx, MRI);
- auto BitSet = llvm::countr_zero(*SrcVecIdx & ~OldSrcVecIdx);
- auto Offset = (BitSet * MRI.getType(SrcVecReg).getScalarSizeInBits()) %
+ uint64_t OldSrcVecIdx = *SrcVecIdx;
+ uint64_t Bits = getCoveredBits(SrcReg, SrcVecReg, SrcVecIdx, MRI);
+ int BitSet = llvm::countr_zero(*SrcVecIdx & ~OldSrcVecIdx);
+ uint64_t Offset = (BitSet * MRI.getType(SrcVecReg).getScalarSizeInBits()) %
MRI.getType(SrcReg).getSizeInBits();
if (ShiftValue != Offset)
return 0; // will not insert at proper offset
return Bits << ShiftValue;
} break;
case TargetOpcode::G_OR: {
- auto LHSReg = SrcMI.getOperand(1).getReg();
- auto RHSReg = SrcMI.getOperand(2).getReg();
- auto LHSOp = getDefIgnoringCopies(LHSReg, MRI)->getOpcode();
- auto RHSOp = getDefIgnoringCopies(RHSReg, MRI)->getOpcode();
+ llvm::Register LHSReg = SrcMI.getOperand(1).getReg();
+ llvm::Register RHSReg = SrcMI.getOperand(2).getReg();
+ unsigned LHSOp = getDefIgnoringCopies(LHSReg, MRI)->getOpcode();
+ unsigned RHSOp = getDefIgnoringCopies(RHSReg, MRI)->getOpcode();
if ((LHSOp != TargetOpcode::G_SHL) && (LHSOp != TargetOpcode::G_ZEXT) &&
(LHSOp != TargetOpcode::G_OR))
return 0;
if ((RHSOp != TargetOpcode::G_SHL) && (RHSOp != TargetOpcode::G_ZEXT) &&
(RHSOp != TargetOpcode::G_OR))
return 0;
- auto LHSBits = getCoveredBits(LHSReg, SrcVecReg, SrcVecIdx, MRI);
- auto RHSBits = getCoveredBits(RHSReg, SrcVecReg, SrcVecIdx, MRI);
+ uint64_t LHSBits = getCoveredBits(LHSReg, SrcVecReg, SrcVecIdx, MRI);
+ uint64_t RHSBits = getCoveredBits(RHSReg, SrcVecReg, SrcVecIdx, MRI);
if (LHSBits && RHSBits)
return LHSBits | RHSBits;
} break;
@@ -1247,28 +1252,28 @@ static uint64_t getCoveredBits(Register BaseReg, Register &SrcVecReg,
bool PISAPreLegalizerCombinerImpl::matchExtractInsertToBitcast(
MachineInstr &MI, Register &SaveReg) const {
- auto &BuildVectorMI = MI;
- auto DstReg = BuildVectorMI.getOperand(0).getReg();
- auto BuildVectorTy = MRI.getType(DstReg);
- auto NumElts = BuildVectorTy.getNumElements();
+ llvm::MachineInstr &BuildVectorMI = MI;
+ llvm::Register DstReg = BuildVectorMI.getOperand(0).getReg();
+ llvm::LLT BuildVectorTy = MRI.getType(DstReg);
+ uint16_t NumElts = BuildVectorTy.getNumElements();
Register SrcVecReg; // G_EXTRACT_VECTOR_ELT
uint64_t SrcVecIdx = 0; // index of G_EXTRACT_VECTOR_ELT
for (unsigned I = 0; I < NumElts; I++) {
- auto BuildVectorEltReg = BuildVectorMI.getOperand(I + 1).getReg();
- auto Bits = getCoveredBits(BuildVectorEltReg, SrcVecReg, &SrcVecIdx, MRI);
+ llvm::Register BuildVectorEltReg = BuildVectorMI.getOperand(I + 1).getReg();
+ uint64_t Bits = getCoveredBits(BuildVectorEltReg, SrcVecReg, &SrcVecIdx, MRI);
if (!Bits)
return false; // did not match
if (BuildVectorTy.getSizeInBits() != MRI.getType(SrcVecReg).getSizeInBits())
return false;
if (!((1ull << I) & SrcVecIdx))
return false; // indices are not in ascending order
- auto EltSize = BuildVectorTy.getScalarSizeInBits();
+ unsigned EltSize = BuildVectorTy.getScalarSizeInBits();
uint64_t Mask = (EltSize == 64) ? (uint64_t)-1ll : (1ull << EltSize) - 1;
if (Mask != Bits)
return false; // did not cover full element
}
- auto SrcVecTy = MRI.getType(SrcVecReg);
+ llvm::LLT SrcVecTy = MRI.getType(SrcVecReg);
uint64_t Mask = (1ull << SrcVecTy.getNumElements()) - 1;
if (Mask != SrcVecIdx)
return false; // did not extract all indices
@@ -1278,7 +1283,7 @@ bool PISAPreLegalizerCombinerImpl::matchExtractInsertToBitcast(
}
void PISAPreLegalizerCombinerImpl::applyExtractInsertToBitcast(
MachineInstr &MI, Register SrcVecReg) const {
- auto DstReg = MI.getOperand(0).getReg();
+ llvm::Register DstReg = MI.getOperand(0).getReg();
if (MRI.getType(DstReg) == MRI.getType(SrcVecReg))
B.buildCopy(DstReg, SrcVecReg);
else
@@ -1298,37 +1303,37 @@ void PISAPreLegalizerCombinerImpl::applyExtractInsertToBitcast(
// => %19:_(<4 x s8>) = G_BITCAST %1:_(s32)
bool PISAPreLegalizerCombinerImpl::matchExtractBuildVectorToBitcast(
MachineInstr &MI, Register &SaveReg) const {
- auto &BuildMI = MI;
- auto DstTy = MRI.getType(BuildMI.getOperand(0).getReg());
+ llvm::MachineInstr &BuildMI = MI;
+ llvm::LLT DstTy = MRI.getType(BuildMI.getOperand(0).getReg());
Register CastSrcReg; // G_BITCAST %1
for (unsigned I = 1; I < BuildMI.getNumOperands(); I++) {
- auto EltReg = BuildMI.getOperand(I).getReg();
- auto &ExtMI = *getDefIgnoringCopies(EltReg, MRI);
+ llvm::Register EltReg = BuildMI.getOperand(I).getReg();
+ llvm::MachineInstr &ExtMI = *getDefIgnoringCopies(EltReg, MRI);
if (ExtMI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) {
- auto NumReg = ExtMI.getOperand(2).getReg();
- auto NumValue = getIConstantVRegValWithLookThrough(NumReg, MRI);
+ llvm::Register NumReg = ExtMI.getOperand(2).getReg();
+ std::optional<llvm::ValueAndVReg> NumValue = getIConstantVRegValWithLookThrough(NumReg, MRI);
if (!NumValue.has_value() || (NumValue->Value != (I - 1)))
return false;
- auto &CastMI = *getDefIgnoringCopies(ExtMI.getOperand(1).getReg(), MRI);
+ llvm::MachineInstr &CastMI = *getDefIgnoringCopies(ExtMI.getOperand(1).getReg(), MRI);
if (CastMI.getOpcode() != TargetOpcode::G_BITCAST)
return false;
- auto CastReg = CastMI.getOperand(1).getReg();
+ llvm::Register CastReg = CastMI.getOperand(1).getReg();
if (CastSrcReg && (CastSrcReg != CastReg))
return false;
if (DstTy.getSizeInBits() != MRI.getType(CastReg).getSizeInBits())
return false;
CastSrcReg = CastReg;
} else if (ExtMI.getOpcode() == TargetOpcode::G_TRUNC) {
- auto &ShiftMI = *getDefIgnoringCopies(ExtMI.getOperand(1).getReg(), MRI);
+ llvm::MachineInstr &ShiftMI = *getDefIgnoringCopies(ExtMI.getOperand(1).getReg(), MRI);
if (ShiftMI.getOpcode() != TargetOpcode::G_LSHR)
return false;
- auto ShiftValue = getIConstantVRegValWithLookThrough(
+ std::optional<llvm::ValueAndVReg> ShiftValue = getIConstantVRegValWithLookThrough(
ShiftMI.getOperand(2).getReg(), MRI);
if (!ShiftValue.has_value() ||
(ShiftValue->Value != ((I - 1) * DstTy.getScalarSizeInBits())))
return false;
- auto ShiftReg = ShiftMI.getOperand(1).getReg();
+ llvm::Register ShiftReg = ShiftMI.getOperand(1).getReg();
if (CastSrcReg && (CastSrcReg != ShiftReg))
return false;
CastSrcReg = ShiftReg;
@@ -1340,7 +1345,7 @@ bool PISAPreLegalizerCombinerImpl::matchExtractBuildVectorToBitcast(
}
void PISAPreLegalizerCombinerImpl::applyExtractBuildVectorToBitcast(
MachineInstr &MI, Register CastSrcReg) const {
- auto DstReg = MI.getOperand(0).getReg();
+ llvm::Register DstReg = MI.getOperand(0).getReg();
B.buildBitcast(DstReg, CastSrcReg);
MI.eraseFromParent();
}
@@ -1380,11 +1385,11 @@ bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
MachineInstr &MI,
std::function<void(MachineIRBuilder &)> &MatchInfo) const {
- auto DstReg = MI.getOperand(0).getReg();
+ llvm::Register DstReg = MI.getOperand(0).getReg();
if (!MRI.hasOneUse(DstReg))
return false;
- auto DstTy = MRI.getType(DstReg);
+ llvm::LLT DstTy = MRI.getType(DstReg);
if (!DstTy.isScalar() || (DstTy.getSizeInBits() == 32))
return false;
@@ -1397,7 +1402,7 @@ bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
MachineInstr *VecCmpMI = nullptr;
APInt Mask;
- auto ReductionOpcode = MI.getOpcode();
+ unsigned ReductionOpcode = MI.getOpcode();
std::function<bool(MachineInstr *)> Match = [&](MachineInstr *MI) {
if (!MI)
@@ -1429,7 +1434,7 @@ bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
m_GFCmp(m_Pred(), m_Reg(), m_Reg())))) {
if (VecCmpMI)
return VecCmpMI == MI;
- auto VecType = MRI.getType(MI->getOperand(0).getReg());
+ llvm::LLT VecType = MRI.getType(MI->getOperand(0).getReg());
if (!VecType.isFixedVector())
return false;
// Found cmp producing vector of predicates.
@@ -1448,7 +1453,7 @@ bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
const LLT S32 = LLT::integer(32);
// Build reduction in s32.
- auto ExtendedVector = MRI.createGenericVirtualRegister(
+ llvm::Register ExtendedVector = MRI.createGenericVirtualRegister(
LLT::fixed_vector(Mask.getBitWidth(), S32));
B.buildSExt(ExtendedVector, VecCmpMI->getOperand(0).getReg());
@@ -1457,8 +1462,8 @@ bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
B.buildConstant(S32, 0));
for (unsigned I = 1; I < Mask.getBitWidth(); ++I) {
- auto SecondSrc = MRI.createGenericVirtualRegister(S32);
- auto Dst = MRI.createGenericVirtualRegister(S32);
+ llvm::Register SecondSrc = MRI.createGenericVirtualRegister(S32);
+ llvm::Register Dst = MRI.createGenericVirtualRegister(S32);
B.buildExtractVectorElement(SecondSrc, ExtendedVector,
B.buildConstant(S32, I));
B.buildInstr(ReductionOpcode, {Dst}, {Reduction, SecondSrc});
@@ -1466,10 +1471,10 @@ bool PISAPreLegalizerCombinerImpl::matchReducePredicates(
}
// Replace next use if possible.
- auto UseMI = MRI.use_instr_begin(DstReg);
+ llvm::MachineRegisterInfo::use_instr_iterator UseMI = MRI.use_instr_begin(DstReg);
switch (UseMI->getOpcode()) {
case TargetOpcode::G_ICMP: {
- auto Pred = (CmpInst::Predicate)UseMI->getOperand(1).getPredicate();
+ llvm::CmpInst::Predicate Pred = (CmpInst::Predicate)UseMI->getOperand(1).getPredicate();
if (mi_match(UseMI->getOperand(3).getReg(), MRI, m_SpecificICst(-1)))
Pred = CmpInst::getInversePredicate(Pred);
B.buildICmp(Pred, UseMI->getOperand(0).getReg(), Reduction,
@@ -1518,9 +1523,9 @@ bool PISAPreLegalizerCombinerImpl::matchCmpInt1(
if (!PrevMI)
return false;
- auto DstReg = ICmpMI.getOperand(0).getReg();
+ llvm::Register DstReg = ICmpMI.getOperand(0).getReg();
MatchInfo = [DstReg, PrevMI, IsTrue, this](MachineIRBuilder &B) {
- auto PrevDstReg = PrevMI->getOperand(0).getReg();
+ llvm::Register PrevDstReg = PrevMI->getOperand(0).getReg();
if (IsTrue) {
if (MRI.hasOneUse(PrevDstReg))
@@ -1578,7 +1583,7 @@ bool PISAPreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
MachineFunctionProperties::Property::FailedISel))
return false;
- auto &TPC = getAnalysis<TargetPassConfig>();
+ llvm::TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
const Function &F = MF.getFunction();
bool EnableOpt =
MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
@@ -1592,7 +1597,7 @@ bool PISAPreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
// Enable CSE.
GISelCSEAnalysisWrapper &Wrapper =
getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
- auto *CSEInfo = &Wrapper.get(TPC.getCSEConfig());
+ llvm::GISelCSEInfo *CSEInfo = &Wrapper.get(TPC.getCSEConfig());
const PISASubtarget &STI = MF.getSubtarget<PISASubtarget>();
PISAPreLegalizerCombinerImpl Impl(MF, CInfo, *KB, CSEInfo, RuleConfig, STI,
@@ -1629,8 +1634,10 @@ bool PISAPreLegalizerCombinerImpl::matchSelectTruncOneZero(
return false;
// True value must be 1, false value must be 0.
- auto TrueOpt = getIConstantVRegValWithLookThrough(Sel.getTrueReg(), MRI);
- auto FalseOpt = getIConstantVRegValWithLookThrough(Sel.getFalseReg(), MRI);
+ std::optional<llvm::ValueAndVReg> TrueOpt =
+ getIConstantVRegValWithLookThrough(Sel.getTrueReg(), MRI);
+ std::optional<llvm::ValueAndVReg> FalseOpt =
+ getIConstantVRegValWithLookThrough(Sel.getFalseReg(), MRI);
if (!TrueOpt || !FalseOpt)
return false;
if (!TrueOpt->Value.isOne() || !FalseOpt->Value.isZero())
@@ -1638,8 +1645,8 @@ bool PISAPreLegalizerCombinerImpl::matchSelectTruncOneZero(
Register DstReg = Sel.getReg(0);
MatchInfo = [=](MachineIRBuilder &B) {
- auto One = B.buildConstant(WideTy, 1);
- auto And = B.buildAnd(WideTy, Wide, One);
+ llvm::MachineInstrBuilder One = B.buildConstant(WideTy, 1);
+ llvm::MachineInstrBuilder And = B.buildAnd(WideTy, Wide, One);
B.buildTrunc(DstReg, And);
};
return true;
diff --git a/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp b/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
index fb8fa3f69ed17..9c4d765c1b481 100644
--- a/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
+++ b/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
@@ -34,7 +34,7 @@ PISARegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
SmallVector<const ValueMapping *, 8> OpdsMapping(MI.getNumOperands());
for (unsigned Idx = 0; Idx < MI.getNumOperands(); ++Idx) {
- auto &MO = MI.getOperand(Idx);
+ const MachineOperand &MO = MI.getOperand(Idx);
if (MO.isReg() && MO.getReg().isValid()) {
unsigned Size = getSizeInBits(MO.getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/PISA/PISASubtarget.cpp b/llvm/lib/Target/PISA/PISASubtarget.cpp
index bfc7c7390c2cc..3bcb384ca0c62 100644
--- a/llvm/lib/Target/PISA/PISASubtarget.cpp
+++ b/llvm/lib/Target/PISA/PISASubtarget.cpp
@@ -47,6 +47,6 @@ PISASubtarget &PISASubtarget::initSubtargetDependencies(StringRef CPU,
// Determine compatibility of instruction's PISA target, specified via
// "let Predicates = []", vs. platform's target, specified via -mcpu=
bool PISASubtarget::supportsPISATarget(StringRef Name) const {
- auto InstrPISATarget = PISA::getPISATargetInfo(Name);
+ PISA::PISATargetInfo InstrPISATarget = PISA::getPISATargetInfo(Name);
return isCompatiblePISATargetInfo(PISATarget, InstrPISATarget);
}
diff --git a/llvm/lib/Target/PISA/PISATargetMachine.cpp b/llvm/lib/Target/PISA/PISATargetMachine.cpp
index f4b3e163a3b8d..f3fef36031876 100644
--- a/llvm/lib/Target/PISA/PISATargetMachine.cpp
+++ b/llvm/lib/Target/PISA/PISATargetMachine.cpp
@@ -88,7 +88,7 @@ PISATargetMachine::getSubtargetImpl(const Function &F) const {
SmallString<128> Key(CPU);
Key.append(FS);
- auto &I = SubtargetMap[Key];
+ std::unique_ptr<PISASubtarget> &I = SubtargetMap[Key];
if (!I) {
I = std::make_unique<PISASubtarget>(TargetTriple, CPU.str(), FS.str(),
*this);
@@ -97,7 +97,7 @@ PISATargetMachine::getSubtargetImpl(const Function &F) const {
}
unsigned PISATargetMachine::getAssumedAddrSpace(const Value *V) const {
- const auto *Ld = dyn_cast<LoadInst>(V);
+ const LoadInst *Ld = dyn_cast<LoadInst>(V);
if (!Ld || Ld->getPointerOperand()->getType()->getPointerAddressSpace() !=
unsigned(PISAAS::AddressSpace::CONSTANT))
return ~0U;
@@ -106,7 +106,7 @@ unsigned PISATargetMachine::getAssumedAddrSpace(const Value *V) const {
std::pair<const Value *, unsigned>
PISATargetMachine::getPredicatedAddrSpace(const Value *V) const {
- auto *II = dyn_cast<IntrinsicInst>(V);
+ const IntrinsicInst *II = dyn_cast<IntrinsicInst>(V);
if (!II)
return std::make_pair(nullptr, -1);
diff --git a/llvm/lib/Target/PISA/PISATargetMachine.h b/llvm/lib/Target/PISA/PISATargetMachine.h
index 5e98087d2272f..0f6bcf8396d7a 100644
--- a/llvm/lib/Target/PISA/PISATargetMachine.h
+++ b/llvm/lib/Target/PISA/PISATargetMachine.h
@@ -55,10 +55,10 @@ class PISATargetMachine : public CodeGenTargetMachineImpl {
}
bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DstAS) const override {
- auto Sas = (SrcAS == (unsigned)PISAAS::AddressSpace::GLOBAL) ||
+ bool Sas = (SrcAS == (unsigned)PISAAS::AddressSpace::GLOBAL) ||
(SrcAS == (unsigned)PISAAS::AddressSpace::CONSTANT) ||
(SrcAS == (unsigned)PISAAS::AddressSpace::GENERIC);
- auto Das = (DstAS == (unsigned)PISAAS::AddressSpace::GLOBAL) ||
+ bool Das = (DstAS == (unsigned)PISAAS::AddressSpace::GLOBAL) ||
(DstAS == (unsigned)PISAAS::AddressSpace::CONSTANT) ||
(DstAS == (unsigned)PISAAS::AddressSpace::GENERIC);
return Sas && Das;
diff --git a/llvm/lib/TargetParser/PISATargetParser.cpp b/llvm/lib/TargetParser/PISATargetParser.cpp
index 8b84d7eecd3ef..ef5926c8b37a1 100644
--- a/llvm/lib/TargetParser/PISATargetParser.cpp
+++ b/llvm/lib/TargetParser/PISATargetParser.cpp
@@ -19,6 +19,6 @@ void llvm::PISA::fillFeatureMap(StringRef CPU, StringMap<bool> &Features) {
return;
if (std::optional<StringMap<bool>> Default = getCPUDefaultTargetFeatures(
Info.Name, BasicPISASubTypeKV, BasicPISAFeatureKV))
- for (const auto &KV : *Default)
+ for (const StringMapEntry<bool> &KV : *Default)
Features[KV.first()] = KV.second;
}
>From 13503187fbb814f5ff147cf348e7a28b8f2d328f Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 05:49:09 +0000
Subject: [PATCH 04/13] Change lowerReturn to use getSubtarget<PISASubtarget>()
---
llvm/lib/Target/PISA/PISACallLowering.cpp | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/PISA/PISACallLowering.cpp b/llvm/lib/Target/PISA/PISACallLowering.cpp
index 1868290306b12..c7f7f0fde4261 100644
--- a/llvm/lib/Target/PISA/PISACallLowering.cpp
+++ b/llvm/lib/Target/PISA/PISACallLowering.cpp
@@ -35,7 +35,7 @@ bool PISACallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
return false;
if (Val) {
const DataLayout &DL = MIRBuilder.getDataLayout();
- const TargetSubtargetInfo &STI = MIRBuilder.getMF().getSubtarget();
+ const PISASubtarget &STI = MIRBuilder.getMF().getSubtarget<PISASubtarget>();
unsigned Op = 0;
Type *Ty = Val->getType();
Register VReg = VRegs[0];
@@ -384,8 +384,8 @@ unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
Type *ArgType) const {
MachineRegisterInfo *MRI = MIRBuilder.getMRI();
MachineFunction &MF = MIRBuilder.getMF();
- const PISARegisterInfo *TRI = static_cast<const PISARegisterInfo *>(
- MF.getSubtarget().getRegisterInfo());
+ const PISARegisterInfo *TRI =
+ MF.getSubtarget<PISASubtarget>().getRegisterInfo();
bool IsKernel = (F.getCallingConv() == CallingConv::PISA_KERNEL);
unsigned Op = 0;
@@ -540,8 +540,9 @@ bool PISACallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
}
MachineInstrBuilder MIB;
- const PISARegisterInfo *TRI = static_cast<const PISARegisterInfo *>(
- MIRBuilder.getMF().getSubtarget().getRegisterInfo());
+ const PISARegisterInfo *TRI = MIRBuilder.getMF()
+ .getSubtarget<PISASubtarget>()
+ .getRegisterInfo();
if (IsIndirectCall) {
Register CalleeReg = Info.Callee.getReg();
>From 32bc8be0dc93cacef438dfbdfb61213c3c9e8289 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 06:13:36 +0000
Subject: [PATCH 05/13] Use -mtriple=pisa and remove -verify-machineinstrs in
legalizer-unmerge-vectors.mir
---
.../GlobalISel/build-vector-with-constants-trunc-double.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir | 2 +-
.../test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir | 2 +-
.../CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir | 2 +-
.../GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir | 2 +-
.../PISA/GlobalISel/combine-local-id-range-trunc-zext.mir | 2 +-
.../CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir | 2 +-
.../CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir | 2 +-
.../test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir | 2 +-
.../PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir | 2 +-
.../PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir | 2 +-
.../CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir | 2 +-
.../GlobalISel/legalizer-extract-vector-elt-illegal-size.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir | 2 +-
.../CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir | 2 +-
.../GlobalISel/legalizer-load-non-power-of-2-overfetch.mir | 2 +-
.../CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir | 2 +-
.../PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir | 2 +-
.../GlobalISel/legalizer-load-store-vec-of-large-ints.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir | 2 +-
.../PISA/GlobalISel/legalizer-store-long-alignment.mir | 2 +-
.../CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir | 2 +-
.../CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir | 2 +-
.../CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir | 2 +-
.../GlobalISel/postlegalizer-extract-subvector-partial.mir | 2 +-
.../PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir | 4 ++--
.../PISA/GlobalISel/prelegalizer-reduce-predicates.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir | 2 +-
llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir | 2 +-
48 files changed, 49 insertions(+), 49 deletions(-)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir b/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
index 3c2e0376c6fda..63acc4760ca34 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Regression test to make sure truncating double into i8 doesn't try to produce an 8 bit float.
--- |
target triple = "pisa"
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir
index b0dec696aeede..8ef616474255c 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-abs-iredsmax.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
--- |
define i32 @combine_abs_max_i32(i32 %data, i32 %mask, i32 %passthru) {
%a = call i32 @llvm.abs.i32(i32 %data, i1 false)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir
index 534a6a345f4fd..35b36c9d094b7 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-cmp-and-all-ones.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck -check-prefix=COMBINER %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck -check-prefix=COMBINER %s
--- |
target triple = "pisa"
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir
index 08a8dd687113a..7c11941aedfd0 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-extract-build-vector.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
--- |
define void @test_extract_build_vector(ptr %addr, ptr %addr2) {
%rv = load <4 x i32>, ptr %addr, align 32
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir
index b5434203547c2..2e574bdb9d937 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-fadd-constant.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
---
name: test_fadd_constant
legalized: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
index 0658e26af5512..babbdafe38d6a 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# This test exercises the `NoVectors` early-break inside
# PISA::getDefIgnoringBitcasts(). The combine
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir
index 959aaa318bf7f..a69655060b00b 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-local-id-range-trunc-zext.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
--- |
; Test that computeKnownBitsForTargetInstr correctly propagates range
; attribute information from pisa.local.id intrinsics.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir
index 572f6c3ab288d..e4c6643e4e1c7 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-p2i-to-i2p-addrspace.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Test that p2i_to_i2p_fixed combine rule correctly handles pointer address
# spaces. A COPY between pointers of different address spaces is illegal.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
index 694e978edc5f5..8cbb7e7fce8b8 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Tests for PISAPreLegalizerCombinerImpl::matchSelectTruncOneZero.
#
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir
index d492c5fd6c653..cf9e31078c2cb 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-trunc-bool-trunc.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck -check-prefix=COMBINER %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck -check-prefix=COMBINER %s
--- |
define i8 @test_trunc_and_trunc(i32 %a) {
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
index b5abbdcfdb747..b861151d39e81 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Exercises PISAPreLegalizerCombinerImpl::applyTruncatedShift. The combine
# rule trunc_shift_to_vector_extract turns:
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir b/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
index 86d7ce2b0bbc9..8154cd3396beb 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Tests for PISAPostLegalizerCombinerImpl::matchFixIllegalShiftAmt /
# applyFixIllegalShiftAmt.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir
index adc32984beca6..c4e3464c66549 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalize-load-store-subbyte-scalar.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Sub-byte scalar G_STORE paths unreachable from .ll: the prelegalizer combiner
# rewrites every sub-byte store value into a single G_TRUNC, so only hand-written
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir
index 8543f42709ca5..397fa9cb0f5bb 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-16-32-vec.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
---
name: test_2xi16_or
body: |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir
index 411243b1a87f6..0b0cc7e69228f 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-concat-vectors-clamp-elts.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
# Check that G_CONCAT_VECTORS of vectors larger than 4 elts are legalized.
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir
index e8fa9c84f0adb..c139938772652 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-constant.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck %s
---
name: test_constant_s32
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir
index 71079bbb50b91..1fb2564aacc39 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-subvector.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck %s
---
name: extract_subvector_s64
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
index 81147be2a538c..6ff8503af3087 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Check that i32 vectors with an illegal number of elements are extended to the nearest legal size before extracting the element using the G_EXTRACT_VECTOR_ELT instruction.
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir
index b51cc3ccf27b0..5cf0490821921 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-fldexp.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
---
name: test_afn_bfloat
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir
index eef864f65c4ca..222767252dcba 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-freeze-i96.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Verify that G_FREEZE is legalized successfully into two i64 parts.
# NOTE: The intermediate widening to i128 is an internal step, and does not appear in the final output.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir
index bcaf76d745111..5bb28538d0708 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-frem.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
---
name: test_frem_afn
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir
index 4f6217576f36d..af616abbb9fbb 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-i2f.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
---
name: test_nonpow2_uitofp
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir
index 6d310a0e5c7bb..5db4dec81669d 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-icmp.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - -verify-machineinstrs | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
---
name: test_s1
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir
index 5b7bb46ca3308..ede400e27d413 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-implicit-def.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck %s
---
name: test_implicit_def_s1
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir
index c25457854eda1..5aa145938ff91 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-insert-subvector.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck %s
---
name: insert_subvector_s64
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir
index 67fda24269841..78d28560cfaa0 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-non-power-of-2-overfetch.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
# Check that loads/stores of vectors of <n x i32> elements with n
# being [5,7] elements is resolved to a load of <8 x i32> for smaller
# alignments. However, when alignment is > 8 bytes for slm and constant
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir
index 5a29c576890e8..5986846bdc938 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-i1-vec.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck %s
---
name: test_load_128
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir
index dea8bb21f3cc5..d3f30d1f229a3 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-non-standard-vec.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
# Check that loads/stores of vectors of non-2^n elements are resolved to legal
# instructions. As of creating this test, they are split apart into multiple
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir
index 168f7907827c8..623de797c736c 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-load-store-vec-of-large-ints.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -march=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck -check-prefix=LEGAL %s
# Check that loads/stores of vectors of 32n-bit width elements wider than 64 bits are resolved to vectors of 32-bit elements.
# Check that if (alignment < element bit width) scalarization is performed instead.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
index e9b14060413a3..52aab7a14ffd9 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Test that legalizeGMulh correctly handles carry propagation in the
# schoolbook 64-bit multiply-high expansion.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir
index 0aaafc45bec2a..703d52d0df301 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Verify G_UMULH lowering on non-power-of-2 type.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir
index fac84023771fa..e08e0cd49f506 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s5.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Check that i5 phi operand is extended to i16.
--- |
target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir
index 682d8b5428f92..187f18b991529 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-s96.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Check that i96 phi operand doesn't exit with UnableToLegalize.
--- |
target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir
index 9c8dcbd2e9b2d..d1651c264769f 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-scalarize.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
--- |
target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-n16:32-G1"
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir
index 11a6fa78ed52b..82e92a5803cc4 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-phi-widen.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Check that i24 operands in phi inst are extended to i32.
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir
index 66b742daeeb5d..b9849f3dbcaf4 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-select-widen.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Check that i40 operands in select inst are extended to i64.
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
index ade58cf506a0a..6301686e40120 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
# Check that alignment too large to store in 32-bit value isn't truncated.
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir
index 79df8faa0f77e..cb7f9e4869fe2 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-udiv128-sequence.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Checks that sequence produced by Expand IR instructions for udiv128 is legalized:
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir
index e95a5b0a44db9..18562155aa599 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-unmerge-vectors.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer %s -o - | FileCheck %s
# Verify UNMERGE lowering case where both SrcTy and DstTy are vectors with over 4 elements.
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
index b491bd641184e..347d43c878e9c 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Ensure that G_ZEXT with large destination types is handled without causing a crash.
--- |
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir b/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir
index 6ffd5feac885d..d9e933983a183 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/mir-print-bfloat.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
---
name: test_bfloat
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
index 3f43b136219ca..678276d43751f 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Tests the compare_select PostLegalizerCombiner rule:
# %cond = G_ICMP intpred(ne), %x(s32), 0
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir
index 2c8f60aee0b89..3226ea2cb42e9 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-extract-subvector-partial.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# G_EXTRACT_SUBVECTOR fed by a chain of G_INSERT_SUBVECTOR into IMPLICIT_DEF,
# where the extracted window fully contains the inserts. The postlegalizer
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
index e0ab946e993ca..8ecbda48bef10 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Test that the PostLegalizer does not infinite-loop on shl(or x, const), const.
#
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir b/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir
index e194f452990c4..f2665036d82f2 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postrapseudo-i1.mir
@@ -1,5 +1,5 @@
-# RUN: llc -march=pisa -run-pass=postrapseudos -verify-machineinstrs %s -o - | FileCheck -check-prefix=PSEUDO %s
-# RUN-CRASHES: llc -march=pisa -start-before=postrapseudos -verify-machineinstrs %s -o - | FileCheck -check-prefix=ASM %s
+# RUN: llc -mtriple=pisa -run-pass=postrapseudos -verify-machineinstrs %s -o - | FileCheck -check-prefix=PSEUDO %s
+# RUN-CRASHES: llc -mtriple=pisa -start-before=postrapseudos %s -o - | FileCheck -check-prefix=ASM %s
--- |
define i32 @test_copy_pred(i32 %arg) {
ret i32 %arg
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
index 456a006bded3b..99b4fae45cbc6 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --validate-debuginfo False --version 6
-# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# The reduce_predicates PreLegalizerCombiner rule rewrites an AND/OR/XOR
# reduction over the per-lane results of a vector compare into an explicit
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
index a8d3e9bd863b7..c18dd86a662aa 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
@@ -1,4 +1,4 @@
-# RUN: llc -march=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
--- |
define i64 @zext_and_mask_rhs(i32 %x) {
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir b/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir
index c383a57135cfa..88b2fdaa9e540 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/retain-extractvec.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -march=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
---
name: retain_extract_subvector
>From 32e9faedb15b748832d51ce862cea1805675edc0 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 06:24:53 +0000
Subject: [PATCH 06/13] Obtain PISARegisterInfo through PISASubtarget
---
llvm/lib/Target/PISA/PISARegisterBankInfo.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp b/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
index 9c4d765c1b481..daa2937ab2dda 100644
--- a/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
+++ b/llvm/lib/Target/PISA/PISARegisterBankInfo.cpp
@@ -8,6 +8,7 @@
#include "PISARegisterBankInfo.h"
#include "PISARegisterInfo.h"
+#include "PISASubtarget.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterBank.h"
@@ -29,7 +30,7 @@ PISARegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
const MachineFunction &MF = *MI.getParent()->getParent();
const MachineRegisterInfo &MRI = MF.getRegInfo();
- const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+ const PISARegisterInfo *TRI = MF.getSubtarget<PISASubtarget>().getRegisterInfo();
SmallVector<const ValueMapping *, 8> OpdsMapping(MI.getNumOperands());
>From 4e7636f8a4d3acdd6afde5db447d87e866da0837 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 06:35:01 +0000
Subject: [PATCH 07/13] Use {} instead of std::make_pair
---
.../lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp | 2 +-
llvm/lib/Target/PISA/PISAISelLowering.cpp | 2 +-
llvm/lib/Target/PISA/PISALegalizerInfo.cpp | 5 ++---
llvm/lib/Target/PISA/PISARegisterInfo.cpp | 7 +++----
llvm/lib/Target/PISA/PISATargetMachine.cpp | 13 +++++--------
5 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp b/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp
index acfaa8c58847f..5075a96e9a917 100644
--- a/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp
+++ b/llvm/lib/Target/PISA/MCTargetDesc/PISARegEncoder.cpp
@@ -177,7 +177,7 @@ RegEncoder::decodeVirtualRegister(MCRegister Reg) {
unsigned BankBits = (Reg >> NumRegBits) & ((1U << NumBankBits) - 1);
const char *Prefix = getPrefixFromBank(getRegBank(BankBits));
unsigned Num = Reg & ((1U << NumRegBits) - 1);
- return std::make_pair(Prefix, Num);
+ return {Prefix, Num};
}
bool RegEncoder::isVirtualRegNo(unsigned RegNo) {
diff --git a/llvm/lib/Target/PISA/PISAISelLowering.cpp b/llvm/lib/Target/PISA/PISAISelLowering.cpp
index 51f00605d3969..47d785c3d9b77 100644
--- a/llvm/lib/Target/PISA/PISAISelLowering.cpp
+++ b/llvm/lib/Target/PISA/PISAISelLowering.cpp
@@ -472,7 +472,7 @@ PISATargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
}
if (RC)
- return std::make_pair(0u, RC);
+ return {0u, RC};
}
return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
diff --git a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
index ae72145211323..29196c9c2b635 100644
--- a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
+++ b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
@@ -421,8 +421,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
llvm::LLT EltTy = Query.Types[0];
uint16_t NumElts = EltTy.getNumElements();
uint64_t NewNumElts = PowerOf2Ceil(NumElts) / 2;
- return std::make_pair(
- 0, LLT::fixed_vector(NewNumElts, EltTy.getScalarType()));
+ return {0, LLT::fixed_vector(NewNumElts, EltTy.getScalarType())};
})
// split up vectors of non-standard size elements
.fewerElementsIf(
@@ -433,7 +432,7 @@ PISALegalizerInfo::PISALegalizerInfo(const PISASubtarget &ST) {
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
llvm::LLT EltTy = Query.Types[0];
- return std::make_pair(0, EltTy.getScalarType());
+ return {0, EltTy.getScalarType()};
})
// cast non-^2 scalars to vectors of i8
.bitcastIf(
diff --git a/llvm/lib/Target/PISA/PISARegisterInfo.cpp b/llvm/lib/Target/PISA/PISARegisterInfo.cpp
index 01eb0ca574677..f5015c80453d5 100644
--- a/llvm/lib/Target/PISA/PISARegisterInfo.cpp
+++ b/llvm/lib/Target/PISA/PISARegisterInfo.cpp
@@ -99,10 +99,9 @@ PISARegisterInfo::PISARegisterInfo() : PISAGenRegisterInfo(PISA::DummyReg) {
RCD->NumElements = RC->LaneMask.getNumLanes();
RCD->ScalarBitSize = getScalarBitSize(RC, RCD->NumElements);
}
- std::pair<unsigned, unsigned> Key =
- std::make_pair(RCD->NumElements, RCD->ScalarBitSize);
+ std::pair<unsigned, unsigned> Key{RCD->NumElements, RCD->ScalarBitSize};
if (!VecRegClassMap[Key])
- VecRegClassMap[std::make_pair(RCD->NumElements, RCD->ScalarBitSize)] = RC;
+ VecRegClassMap[Key] = RC;
RegClassMap[RC] = std::move(RCD);
}
@@ -266,7 +265,7 @@ PISARegisterInfo::getBitSizeFromRegClass(const TargetRegisterClass *RC) const {
const TargetRegisterClass *
PISARegisterInfo::getVectorRegClass(unsigned NumElts, unsigned BitSize) const {
- std::pair<unsigned, unsigned> P = std::make_pair(NumElts, BitSize);
+ std::pair<unsigned, unsigned> P{NumElts, BitSize};
DenseMap<std::pair<unsigned, unsigned>,
const TargetRegisterClass *>::const_iterator I =
VecRegClassMap.find(P);
diff --git a/llvm/lib/Target/PISA/PISATargetMachine.cpp b/llvm/lib/Target/PISA/PISATargetMachine.cpp
index f3fef36031876..0c10924a18076 100644
--- a/llvm/lib/Target/PISA/PISATargetMachine.cpp
+++ b/llvm/lib/Target/PISA/PISATargetMachine.cpp
@@ -108,22 +108,19 @@ std::pair<const Value *, unsigned>
PISATargetMachine::getPredicatedAddrSpace(const Value *V) const {
const IntrinsicInst *II = dyn_cast<IntrinsicInst>(V);
if (!II)
- return std::make_pair(nullptr, -1);
+ return {nullptr, ~0U};
switch (II->getIntrinsicID()) {
case Intrinsic::pisa_isaddr_private:
- return std::make_pair(II->getArgOperand(0),
- unsigned(PISAAS::AddressSpace::PRIVATE));
+ return {II->getArgOperand(0), unsigned(PISAAS::AddressSpace::PRIVATE)};
case Intrinsic::pisa_isaddr_global:
- return std::make_pair(II->getArgOperand(0),
- unsigned(PISAAS::AddressSpace::GLOBAL));
+ return {II->getArgOperand(0), unsigned(PISAAS::AddressSpace::GLOBAL)};
case Intrinsic::pisa_isaddr_shared:
- return std::make_pair(II->getArgOperand(0),
- unsigned(PISAAS::AddressSpace::SHARED));
+ return {II->getArgOperand(0), unsigned(PISAAS::AddressSpace::SHARED)};
default:
break;
}
- return std::make_pair(nullptr, -1);
+ return {nullptr, ~0U};
}
TargetTransformInfo
>From d2df6a053e110a30a8506f55fdc74813ea28acd9 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 06:42:52 +0000
Subject: [PATCH 08/13] Remove IR sections from MIR tests
---
.../build-vector-with-constants-trunc-double.mir | 12 +-----------
.../combine-getdef-ignoring-bitcasts-novectors.mir | 9 ---------
.../GlobalISel/combine-select-trunc-one-zero.mir | 9 ---------
.../PISA/GlobalISel/combine-truncated-shift.mir | 5 -----
.../PISA/GlobalISel/fix-illegal-shift-amt.mir | 5 -----
.../legalizer-extract-vector-elt-illegal-size.mir | 11 -----------
.../CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir | 9 ---------
.../GlobalISel/legalizer-store-long-alignment.mir | 7 -------
.../CodeGen/PISA/GlobalISel/legalizer-zext-big.mir | 6 ------
.../GlobalISel/postlegalizer-compare-select.mir | 13 -------------
.../postlegalizer-no-commute-shift-loop.mir | 7 -------
.../GlobalISel/prelegalizer-reduce-predicates.mir | 7 -------
.../PISA/GlobalISel/prelegalizer-zext-and.mir | 9 ---------
13 files changed, 1 insertion(+), 108 deletions(-)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir b/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
index 63acc4760ca34..85510b53160cb 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/build-vector-with-constants-trunc-double.mir
@@ -1,22 +1,12 @@
# RUN: llc -mtriple=pisa -run-pass=pisa-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
# Regression test to make sure truncating double into i8 doesn't try to produce an 8 bit float.
---- |
- target triple = "pisa"
-
- define pisa_kernel void @test(i8 %id) {
- %b = bitcast <1 x double> zeroinitializer to <8 x i8>
- %elem = extractelement <8 x i8> %b, i8 %id
- store i8 %elem, ptr addrspace(1) null, align 1
- ret void
- }
-...
---
name: test
legalized: true
tracksRegLiveness: true
isSSA: true
body: |
- bb.1 (%ir-block.0):
+ bb.1:
; CHECK-LABEL: {{^}}name: test
; CHECK: [[CONST:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[CONST]](i64)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
index babbdafe38d6a..62cce2c9c3fa5 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-getdef-ignoring-bitcasts-novectors.mir
@@ -17,15 +17,6 @@
# The shifts_of_constants combine should fold the hinted shift into one constant.
# The body-level CHECK lines below verify the folded G_CONSTANT result.
---- |
- define <2 x i16> @build_vector_with_vector_bitcast_in_chain(i16 %a, i16 %b) {
- ret <2 x i16> zeroinitializer
- }
-
- define i16 @shift_of_constants_via_assert_sext_hints(i16 %unused) {
- ret i16 0
- }
-...
---
name: build_vector_with_vector_bitcast_in_chain
legalized: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
index 8cbb7e7fce8b8..8c26184c523fb 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-select-trunc-one-zero.mir
@@ -6,15 +6,6 @@
# select(trunc(wide -> i1), N 1, N 0) => trunc(and(wide, 1), N)
# to avoid introducing an illegal i1 compare when lowering bool-to-int.
---- |
- define i16 @test_select_trunc_one_zero_match(i32 %wide) { ret i16 0 }
- define i16 @test_no_fold_cond_from_icmp(i32 %a) { ret i16 0 }
- define i16 @test_no_fold_wrong_true_val(i32 %wide) { ret i16 0 }
- define i16 @test_no_fold_wrong_false_val(i32 %wide) { ret i16 0 }
- define i32 @test_no_fold_dst_geq_wide(i16 %wide) { ret i32 0 }
- define i16 @test_no_fold_nonconstant_true(i32 %wide, i16 %t) { ret i16 0 }
-...
-
# ============================================================================
# Positive test: condition is i1 from G_TRUNC, true=1, false=0, DstTy < WideTy.
# Expected transform: select(trunc(i32 to i1), i16 1, i16 0) => trunc(i32 & 1, i16)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir b/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
index b861151d39e81..8d1e293c2b199 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/combine-truncated-shift.mir
@@ -18,11 +18,6 @@
# report for this file; both the lo and hi extracts must appear in the
# output.
---- |
- define i16 @lo_and_hi_truncs(i32 %x) {
- ret i16 0
- }
-...
---
name: lo_and_hi_truncs
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir b/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
index 8154cd3396beb..e1061b3edb9b6 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/fix-illegal-shift-amt.mir
@@ -9,11 +9,6 @@
# - inserting G_TRUNC if the amount type is wider than i32 (e.g. i64)
# - inserting G_ZEXT if the amount type is narrower than i32 (e.g. i16)
---- |
- define i64 @test_shl_i64_i64_amt(i64 %val, i64 %amt) { ret i64 0 }
- define i32 @test_ashr_i32_i16_amt(i32 %val, i16 %amt) { ret i32 0 }
-...
-
# ============================================================================
# Trunc path: shift amount is i64 (>32 bits) -> G_TRUNC inserted to i32.
# Covers applyFixIllegalShiftAmt line 1673: MIB.buildTrunc(I32, ShAmtReg)
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
index 6ff8503af3087..d027e8a3e547a 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-extract-vector-elt-illegal-size.mir
@@ -1,17 +1,6 @@
# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Check that i32 vectors with an illegal number of elements are extended to the nearest legal size before extracting the element using the G_EXTRACT_VECTOR_ELT instruction.
---- |
- define pisa_kernel i32 @test_v10_32b(<10 x i32> %vec) {
- %elem = extractelement <10 x i32> %vec, i32 5
- ret i32 %elem
- }
-
- define pisa_kernel i32 @test_v20_32b(<20 x i32> %vec) {
- %elem = extractelement <20 x i32> %vec, i32 10
- ret i32 %elem
- }
-...
---
name: test_v10_32b
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
index 52aab7a14ffd9..01b03ac02864e 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-mulh-i64.mir
@@ -9,15 +9,6 @@
# upper 64 bits. A buggy implementation that shifts each cross-term
# independently (P_hh + (P_lh>>32) + (P_hl>>32)) drops this carry.
---- |
- define i64 @test_umulh_i64(i64 %a, i64 %b) {
- ret i64 0
- }
- define i64 @test_smulh_i64(i64 %a, i64 %b) {
- ret i64 0
- }
-...
-
# --- G_UMULH i64 ---
# Verify that the legalized sequence includes carry-aware cross-term
# accumulation: the adds feeding the final result must include the
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
index 6301686e40120..71de757dcced9 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-store-long-alignment.mir
@@ -1,13 +1,6 @@
# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck -check-prefix=LEGAL %s
# Check that alignment too large to store in 32-bit value isn't truncated.
---- |
- define void @test_concat_clamp_elts(ptr %addr) {
- %vec.expand = shufflevector <6 x i16> undef, <6 x i16> undef, <12 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
- store <12 x i16> %vec.expand, ptr addrspace(1) null, align 4294967296
- ret void
- }
-...
---
name: test_concat_clamp_elts
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
index 347d43c878e9c..a9e34678f21c5 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/legalizer-zext-big.mir
@@ -2,12 +2,6 @@
# RUN: llc -mtriple=pisa -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
# Ensure that G_ZEXT with large destination types is handled without causing a crash.
---- |
- define i128 @test_zext_big(i16 %arg) {
- %ret = zext i16 %arg to i128
- ret i128 %ret
- }
-...
---
name: test_zext_big
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
index 678276d43751f..9435854c9acc8 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-compare-select.mir
@@ -6,19 +6,6 @@
# =>
# %res = G_PISA_SELECT %x(s32), %a(s32), %b(s32)
---- |
- define i32 @compare_select_ne(i32 %x, i32 %a, i32 %b) {
- %cond = icmp ne i32 %x, 0
- %res = select i1 %cond, i32 %a, i32 %b
- ret i32 %res
- }
-
- define i32 @compare_select_eq(i32 %x, i32 %a, i32 %b) {
- %cond = icmp eq i32 %x, 0
- %res = select i1 %cond, i32 %a, i32 %b
- ret i32 %res
- }
-...
---
name: compare_select_ne
legalized: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
index 8ecbda48bef10..f8539385b679c 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/postlegalizer-no-commute-shift-loop.mir
@@ -19,13 +19,6 @@
# commute_shift is therefore excluded from the PostLegalizer
# (pisa_post_removed_combines).
---- |
- define i32 @shl_or_const(i32 %x) {
- %or = or i32 %x, 4
- %shl = shl i32 %or, 2
- ret i32 %shl
- }
-...
---
name: shl_or_const
legalized: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
index 99b4fae45cbc6..89268b8eef36d 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-reduce-predicates.mir
@@ -11,13 +11,6 @@
# inverting the predicate for the -1 case (eq -> ne) and leaving it unchanged
# for the 0 case.
---- |
- define i32 @reduce_and(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
- define i32 @reduce_or(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
- define i32 @reduce_xor(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
- define i32 @reduce_and_use_icmp_neg1(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
- define i32 @reduce_and_use_icmp_zero(<2 x i32> %a, <2 x i32> %b) { ret i32 0 }
-...
---
name: reduce_and
tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
index c18dd86a662aa..ef1b974235fff 100644
--- a/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
+++ b/llvm/test/CodeGen/PISA/GlobalISel/prelegalizer-zext-and.mir
@@ -1,14 +1,5 @@
# RUN: llc -mtriple=pisa -run-pass=pisa-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
---- |
- define i64 @zext_and_mask_rhs(i32 %x) {
- ret i64 0
- }
- define i64 @zext_and_mask_lhs(i32 %x) {
- ret i64 0
- }
-...
-
# --- canonical: G_AND (G_ZEXT x), C ---
---
name: zext_and_mask_rhs
>From 1899433a181f28652d791252fbbff79111c50174 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 06:48:55 +0000
Subject: [PATCH 09/13] Use LLT::scalarOrVector
---
llvm/lib/Target/PISA/PISACallLowering.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/PISA/PISACallLowering.cpp b/llvm/lib/Target/PISA/PISACallLowering.cpp
index c7f7f0fde4261..0242f42892b2b 100644
--- a/llvm/lib/Target/PISA/PISACallLowering.cpp
+++ b/llvm/lib/Target/PISA/PISACallLowering.cpp
@@ -279,7 +279,7 @@ bool PISACallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
EltRegClass = &PISA::Reg32bRegClass;
EltLLT = LLT::integer(32);
TargetReg = MRI->createGenericVirtualRegister(
- LLT::vector(ElementCount::getFixed(TotalSize / 32), I32));
+ LLT::scalarOrVector(ElementCount::getFixed(TotalSize / 32), I32));
NumElts = TotalSize / 32;
}
>From ff7cdcab176a6792f80a9dde8755e7c7580abe68 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 06:54:39 +0000
Subject: [PATCH 10/13] Replace assert with reportFatalUsageError
---
llvm/lib/Target/PISA/PISACallLowering.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/PISA/PISACallLowering.cpp b/llvm/lib/Target/PISA/PISACallLowering.cpp
index 0242f42892b2b..1358e08ef8ef7 100644
--- a/llvm/lib/Target/PISA/PISACallLowering.cpp
+++ b/llvm/lib/Target/PISA/PISACallLowering.cpp
@@ -17,6 +17,7 @@
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/IR/Metadata.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ModRef.h"
using namespace llvm;
@@ -436,7 +437,7 @@ unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
Op = ParamScalar[IsKernel][3];
break;
default:
- assert(false && "Bit size for call arg not supported");
+ reportFatalUsageError("Bit size for call arg not supported");
}
} else if (ArgType->isPointerTy()) {
if (BitSize == 64) {
@@ -514,7 +515,7 @@ unsigned PISACallLowering::getLoadParamOpcode(MachineIRBuilder &MIRBuilder,
Op = ParamVector[IsKernel][3][NumElts - 2];
break;
default:
- assert(false && "Bit size for call arg not supported");
+ reportFatalUsageError("Bit size for call arg not supported");
}
assert(Op && "argument type is not supported");
} else {
>From e9ad3567fc88fa1ae2590f81df823e1956a2bfc0 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 07:03:05 +0000
Subject: [PATCH 11/13] Rename legalizeGLoad to legalizeGLoadStore (it handles
both loads and stores)
---
llvm/lib/Target/PISA/PISALegalizerInfo.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
index 29196c9c2b635..3078800c0a05f 100644
--- a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
+++ b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
@@ -1488,8 +1488,8 @@ static bool legalizeGExtload(MachineInstr &MI, MachineIRBuilder &B) {
return true;
}
-static bool legalizeGLoad(MachineInstr &MI, MachineIRBuilder &B,
- LegalizerHelper &Helper) {
+static bool legalizeGLoadStore(MachineInstr &MI, MachineIRBuilder &B,
+ LegalizerHelper &Helper) {
llvm::MachineRegisterInfo &MRI = *B.getMRI();
GISelChangeObserver &Observer = Helper.Observer;
llvm::MachineOperand &ValMO = MI.getOperand(0);
@@ -3479,7 +3479,7 @@ bool PISALegalizerInfo::legalizeCustom(
return legalizeGItofp(MI, B);
case TargetOpcode::G_STORE:
case TargetOpcode::G_LOAD:
- return legalizeGLoad(MI, B, Helper);
+ return legalizeGLoadStore(MI, B, Helper);
case TargetOpcode::G_SEXTLOAD:
case TargetOpcode::G_ZEXTLOAD:
return legalizeGExtload(MI, B);
>From 9f5888734904002f793a4f60879550f9bad55667 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 07:11:38 +0000
Subject: [PATCH 12/13] Replace llvm_unreachable with reportFatalUsageError
---
llvm/lib/Target/PISA/PISALegalizerInfo.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
index 3078800c0a05f..f68a83471960a 100644
--- a/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
+++ b/llvm/lib/Target/PISA/PISALegalizerInfo.cpp
@@ -19,6 +19,7 @@
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/IR/IntrinsicsPISA.h"
#include "llvm/IR/PISAIntrinsicUtils.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/PISAAddrSpace.h"
using namespace llvm;
@@ -2164,7 +2165,7 @@ static bool legalizeGFpow(MachineInstr &MI, MachineIRBuilder &B) {
// can only do approximation of pow()
bool AllowApprox = MI.getFlag(MachineInstr::FmAfn);
if (!AllowApprox)
- llvm_unreachable("not implemented (fpow)");
+ reportFatalUsageError("G_FPOW without afn is not supported on PISA");
llvm::Register LogReg = MRI.createGenericVirtualRegister(DstTy);
llvm::Register MulReg = MRI.createGenericVirtualRegister(DstTy);
>From e8c46bb8c17c83b8ed6cd71f6024a6122df13193 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Tue, 22 Sep 2026 07:13:46 +0000
Subject: [PATCH 13/13] Replace /// with //
---
llvm/lib/Target/PISA/PISACombine.td | 2 +-
.../Target/PISA/PISAPostLegalizerCombiner.cpp | 2 +-
.../Target/PISA/PISAPreLegalizerCombiner.cpp | 22 +++++++++----------
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/PISA/PISACombine.td b/llvm/lib/Target/PISA/PISACombine.td
index 2b7939e661d88..5e27e05001228 100644
--- a/llvm/lib/Target/PISA/PISACombine.td
+++ b/llvm/lib/Target/PISA/PISACombine.td
@@ -826,7 +826,7 @@ def reduce_predicates : GICombineRule<
// %28:_(s8) = G_TRUNC %2:reg32b(s32)
// %31:_(s32) = G_LSHR %2:reg32b, %23:_(s32)
// %32:_(s8) = G_TRUNC %31:_(s32)
-///%149:_(<4 x s8>) = G_BUILD_VECTOR %17:_(s8), %25:_(s8), %28:_(s8), %32:_(s8)
+// %149:_(<4 x s8>) = G_BUILD_VECTOR %17:_(s8), %25:_(s8), %28:_(s8), %32:_(s8)
// %141:_(s32) = G_BITCAST %149:_(<4 x s8>)
// => %A = G_AND %1, 0xFFFF
// => %B = G_SHL %2, 16
diff --git a/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp b/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
index a3f66f9f42e8f..44e684baf8380 100644
--- a/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/PISA/PISAPostLegalizerCombiner.cpp
@@ -1064,7 +1064,7 @@ void PISAPostLegalizerCombinerImpl::applyAndSelect(MachineInstr &MI,
// %28:_(s8) = G_TRUNC %2:reg32b(s32)
// %31:_(s32) = G_LSHR %2:reg32b, %23:_(s32)
// %32:_(s8) = G_TRUNC %31:_(s32)
-///%149:_(<4 x s8>) = G_BUILD_VECTOR %17:_(s8), %25:_(s8), %28:_(s8), %32:_(s8)
+// %149:_(<4 x s8>) = G_BUILD_VECTOR %17:_(s8), %25:_(s8), %28:_(s8), %32:_(s8)
// %141:_(s32) = G_BITCAST %149:_(<4 x s8>)
// => %A = G_AND %1, 0xFFFF
// => %B = G_SHL %2, 16
diff --git a/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp b/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
index 292652c0ed21d..000bb301a5dc8 100644
--- a/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/PISA/PISAPreLegalizerCombiner.cpp
@@ -223,7 +223,7 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
llvm::Register ValueReg = LS.getOperand(0).getReg();
llvm::MachineMemOperand &MMO = LS.getMMO();
- /// The remaining size in Bits that still has to be loaded/stored
+ // The remaining size in Bits that still has to be loaded/stored
ssize_t Size = LS.getMemSizeInBits().getValue();
// Support sizes that are not a multiple of 8 by "promoting" them to the next
// multiple of 8. If the size is already a multiple of 8, it is not modified
@@ -242,7 +242,7 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
ValueReg = NewValueReg;
}
- /// The register holding the loaded value at the end
+ // The register holding the loaded value at the end
Register LoadRes;
while (Size > 0) {
uint64_t OpSize = bit_floor(static_cast<size_t>(Size));
@@ -253,7 +253,7 @@ void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
llvm::MachineMemOperand *NewMMO =
MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset() + Offset, OpTy);
- /// Stores the (potentially modified) pointer register
+ // Stores the (potentially modified) pointer register
llvm::Register AddrReg = PointerReg;
// We might need to change the store offset
if (Offset != 0) {
@@ -389,14 +389,14 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
llvm::Register ValueReg = StoreInst.getOperand(0).getReg();
- /// This has the integer size at the beginning, and each individual load
- /// decreases the value by its load size. At the end, this must be zero.
- /// NB: We use the size of the store here, promoted to the next multiple
- /// of 8. This size might be modified later if we find a G_*EXT/G_TRUNC.
+ // This has the integer size at the beginning, and each individual load
+ // decreases the value by its load size. At the end, this must be zero.
+ // NB: We use the size of the store here, promoted to the next multiple
+ // of 8. This size might be modified later if we find a G_*EXT/G_TRUNC.
unsigned ValueSize = StoreInst.getMemSize().getValue() * 8;
- /// This stores the size of the last load instruction in Bytes, s.t. we can
- /// verify that the current load is larger than the previous one
+ // This stores the size of the last load instruction in Bytes, s.t. we can
+ // verify that the current load is larger than the previous one
unsigned LastSize = 0;
SizeModificationOp = nullptr;
@@ -411,7 +411,7 @@ bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
NextChainInst->getOpcode() == TargetOpcode::G_ZEXT ||
NextChainInst->getOpcode() == TargetOpcode::G_SEXT ||
NextChainInst->getOpcode() == TargetOpcode::G_SEXT_INREG) {
- /// The "actual" size that was used during loading
+ // The "actual" size that was used during loading
unsigned LoadSize = MRI.getType(NextChainInst->getOperand(1).getReg())
.getScalarSizeInBits();
@@ -662,7 +662,7 @@ void PISAPreLegalizerCombinerImpl::applySimplifyNonPowerOf2LoadStoreChain(
llvm::MachineMemOperand *NewMMO = MI.getMF()->getMachineMemOperand(
&MMO, MMO.getOffset() + Offset, LoadTy);
- /// Stores the (potentially modified) pointer register
+ // Stores the (potentially modified) pointer register
llvm::Register AddrReg = PointerReg;
// Add the offset to the pointer reg if the offset is not zero
More information about the llvm-branch-commits
mailing list