[llvm-branch-commits] [llvm] [6/7][PISA] Add PISA codegen pipeline and lowering passes (PR #214559)
Michal Paszkowski via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Aug 6 22:59:24 PDT 2026
https://github.com/michalpaszkowski updated https://github.com/llvm/llvm-project/pull/214559
>From bd8a2d16f618a3daa9287c79458ccbc541151e19 Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Mon, 3 Aug 2026 04:11:27 -0700
Subject: [PATCH 1/3] Add PISA codegen pipeline and lowering passes
Add the IR- and MIR-level passes that make up the PISA code generation
pipeline and wire them into PISAPassConfig in PISATargetMachine:
- Intrinsic handling: PISAEmitIntrinsics, PISAExpandIntrinsics,
PISAReplaceIntrinsics
- Call/ABI lowering: PISALegalizeCalls, PISAKernelByValArgsLowering
- IR preparation: PISAPropagateNullPointers, PISALayout, PISAConstProp
- Legalization/verification: PISALegalizePredicates, PISAVerifyTypes,
PISAVerifier
- Post-selection: PISAScopeSelector, PISACacheHintSelector,
PISALegalizeSubregAccess, PISAOptimizeSubregAccess,
PISAOptimizeRedundantCopies, PISAMarkConvergentNoMerge,
PISAInsertLifetimeStart
PISAPassConfig now overrides the GlobalISel and pre/post-regalloc hooks
to run these passes, disables the machine passes that assume physical
registers, and runs the load/store vectorizer and atomic expansion in
the IR stage.
AsmPrinter is added in a follow-up change, so this commit is tested with
MIR-level -run-pass/-stop-after tests.
---
llvm/lib/Target/PISA/CMakeLists.txt | 22 +
llvm/lib/Target/PISA/PISA.h | 8 +-
.../lib/Target/PISA/PISACacheHintSelector.cpp | 104 ++
llvm/lib/Target/PISA/PISAConstProp.cpp | 58 +
llvm/lib/Target/PISA/PISAConstProp.h | 31 +
llvm/lib/Target/PISA/PISAEmitIntrinsics.cpp | 565 ++++++++++
llvm/lib/Target/PISA/PISAExpandIntrinsics.cpp | 106 ++
.../Target/PISA/PISAInsertLifetimeStart.cpp | 751 +++++++++++++
.../PISA/PISAKernelByValArgsLowering.cpp | 294 ++++++
.../Target/PISA/PISAKernelByValArgsLowering.h | 40 +
llvm/lib/Target/PISA/PISALayout.cpp | 581 ++++++++++
llvm/lib/Target/PISA/PISALegalizeCalls.cpp | 505 +++++++++
.../Target/PISA/PISALegalizePredicates.cpp | 994 ++++++++++++++++++
.../Target/PISA/PISALegalizeSubregAccess.cpp | 435 ++++++++
.../Target/PISA/PISAMarkConvergentNoMerge.cpp | 64 ++
.../PISA/PISAOptimizeRedundantCopies.cpp | 171 +++
.../Target/PISA/PISAOptimizeSubregAccess.cpp | 195 ++++
.../Target/PISA/PISAPropagateNullPointers.cpp | 171 +++
.../lib/Target/PISA/PISAReplaceIntrinsics.cpp | 99 ++
llvm/lib/Target/PISA/PISAScopeSelector.cpp | 112 ++
llvm/lib/Target/PISA/PISATargetMachine.cpp | 208 +++-
llvm/lib/Target/PISA/PISAVerifier.cpp | 301 ++++++
llvm/lib/Target/PISA/PISAVerifyTypes.cpp | 87 ++
.../CodeGen/PISA/mark-convergent-no-merge.mir | 56 +
.../test/CodeGen/PISA/pisa-scope-selector.mir | 105 ++
llvm/test/CodeGen/PISA/propagate-null.ll | 173 +++
llvm/test/CodeGen/PISA/verify-types.mir | 35 +
27 files changed, 6211 insertions(+), 60 deletions(-)
create mode 100644 llvm/lib/Target/PISA/PISACacheHintSelector.cpp
create mode 100644 llvm/lib/Target/PISA/PISAConstProp.cpp
create mode 100644 llvm/lib/Target/PISA/PISAConstProp.h
create mode 100644 llvm/lib/Target/PISA/PISAEmitIntrinsics.cpp
create mode 100644 llvm/lib/Target/PISA/PISAExpandIntrinsics.cpp
create mode 100644 llvm/lib/Target/PISA/PISAInsertLifetimeStart.cpp
create mode 100644 llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp
create mode 100644 llvm/lib/Target/PISA/PISAKernelByValArgsLowering.h
create mode 100644 llvm/lib/Target/PISA/PISALayout.cpp
create mode 100644 llvm/lib/Target/PISA/PISALegalizeCalls.cpp
create mode 100644 llvm/lib/Target/PISA/PISALegalizePredicates.cpp
create mode 100644 llvm/lib/Target/PISA/PISALegalizeSubregAccess.cpp
create mode 100644 llvm/lib/Target/PISA/PISAMarkConvergentNoMerge.cpp
create mode 100644 llvm/lib/Target/PISA/PISAOptimizeRedundantCopies.cpp
create mode 100644 llvm/lib/Target/PISA/PISAOptimizeSubregAccess.cpp
create mode 100644 llvm/lib/Target/PISA/PISAPropagateNullPointers.cpp
create mode 100644 llvm/lib/Target/PISA/PISAReplaceIntrinsics.cpp
create mode 100644 llvm/lib/Target/PISA/PISAScopeSelector.cpp
create mode 100644 llvm/lib/Target/PISA/PISAVerifier.cpp
create mode 100644 llvm/lib/Target/PISA/PISAVerifyTypes.cpp
create mode 100644 llvm/test/CodeGen/PISA/mark-convergent-no-merge.mir
create mode 100644 llvm/test/CodeGen/PISA/pisa-scope-selector.mir
create mode 100644 llvm/test/CodeGen/PISA/propagate-null.ll
create mode 100644 llvm/test/CodeGen/PISA/verify-types.mir
diff --git a/llvm/lib/Target/PISA/CMakeLists.txt b/llvm/lib/Target/PISA/CMakeLists.txt
index a51c8cfafc728..a790f0abe2312 100644
--- a/llvm/lib/Target/PISA/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/CMakeLists.txt
@@ -18,30 +18,51 @@ tablegen(LLVM PISAGenPostLegalizeGICombiner.inc -gen-global-isel-combiner
add_public_tablegen_target(PISACommonTableGen)
add_llvm_target(PISACodeGen
+ PISACacheHintSelector.cpp
PISACallLowering.cpp
+ PISAConstProp.cpp
+ PISAEmitIntrinsics.cpp
+ PISAExpandIntrinsics.cpp
PISAISelLowering.cpp
+ PISAInsertLifetimeStart.cpp
PISAInstrInfo.cpp
PISAInstructionSelector.cpp
+ PISAKernelByValArgsLowering.cpp
+ PISALegalizeCalls.cpp
+ PISALegalizePredicates.cpp
+ PISALegalizeSubregAccess.cpp
PISALegalizerInfo.cpp
PISAMCInstLower.cpp
PISAMachineFunctionInfo.cpp
+ PISAMarkConvergentNoMerge.cpp
+ PISAOptimizeRedundantCopies.cpp
+ PISAOptimizeSubregAccess.cpp
PISAPostLegalizerCombiner.cpp
PISAPreLegalizerCombiner.cpp
+ PISAPropagateNullPointers.cpp
PISARegManager.cpp
PISARegisterBankInfo.cpp
PISARegisterInfo.cpp
+ PISAReplaceIntrinsics.cpp
+ PISAScopeSelector.cpp
PISASubtarget.cpp
PISATargetMachine.cpp
PISAUtils.cpp
+ PISAVerifier.cpp
+ PISAVerifyTypes.cpp
+ PISALayout.cpp
LINK_COMPONENTS
Analysis
AsmPrinter
+ BitReader
CodeGen
CodeGenTypes
Core
Demangle
GlobalISel
+ IPO
+ Linker
MC
Passes
Scalar
@@ -52,6 +73,7 @@ add_llvm_target(PISACodeGen
TransformUtils
PISADesc
PISAInfo
+ Vectorize
ADD_TO_COMPONENT
PISA
diff --git a/llvm/lib/Target/PISA/PISA.h b/llvm/lib/Target/PISA/PISA.h
index 375208bf28350..78916267e4708 100644
--- a/llvm/lib/Target/PISA/PISA.h
+++ b/llvm/lib/Target/PISA/PISA.h
@@ -37,7 +37,7 @@ FunctionPass *createPISAPreLegalizerCombiner();
FunctionPass *createPISAPostLegalizerCombiner();
FunctionPass *createPISAReplaceIntrinsicsPass();
MachineFunctionPass *createPISALegalizePredicatesPass();
-MachineFunctionPass *createCacheHintSelectorPass();
+MachineFunctionPass *createPISACacheHintSelectorPass();
MachineFunctionPass *createPISAScopeSelectorPass();
InstructionSelector *
@@ -45,20 +45,16 @@ createPISAInstructionSelector(const PISATargetMachine &TM,
const PISASubtarget &Subtarget,
const RegisterBankInfo &RBI);
-MachineFunctionPass *
-createPISAMachineFunctionPrinterPass(const std::string &Banner,
- unsigned Counter);
FunctionPass *createPISALayoutPass();
MachineFunctionPass *createPISAVerifyTypesPass();
-void initializeCacheHintSelectorPass(PassRegistry &);
+void initializePISACacheHintSelectorPass(PassRegistry &);
void initializePISAEmitIntrinsicsPass(PassRegistry &);
void initializePISAExpandIntrinsicsPass(PassRegistry &);
void initializePISAInsertLifetimeStartPass(PassRegistry &);
void initializePISAKernelByValArgsLoweringLegacyPass(PassRegistry &);
void initializePISALegalizeCallsPass(PassRegistry &);
void initializePISALegalizeSubregAccessPass(PassRegistry &);
-void initializePISAMachineFunctionPrinterPass(PassRegistry &);
void initializePISAMarkConvergentNoMergePass(PassRegistry &);
void initializePISAOptimizeRedundantCopiesPass(PassRegistry &);
void initializePISAOptimizeSubregAccessPass(PassRegistry &);
diff --git a/llvm/lib/Target/PISA/PISACacheHintSelector.cpp b/llvm/lib/Target/PISA/PISACacheHintSelector.cpp
new file mode 100644
index 0000000000000..859ecc68160c4
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISACacheHintSelector.cpp
@@ -0,0 +1,104 @@
+//===-- PISACacheHintSelector.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// cachehint (.cc) support
+// - LLVM IR specifies cache hints via the 'pisa.cache.ctrl' MMRA tag
+// (carried on the instruction's !mmra metadata)
+// - that value is encoded within MI's target-specific flags by
+// PISA TargetLowering::getTargetMMOFlags()
+// - PISA instructions define $cachehint input used in instruction printing
+// - this pass extract target-specific flags from MI and encodes $cachehint
+
+#include "MCTargetDesc/PISAInstPrinter.h"
+#include "PISA.h"
+
+#define GET_INSTRINFO_OPERAND_ENUM
+#include "PISAGenInstrInfo.inc"
+
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+#define DEBUG_TYPE "pisa-cache-hint-selector"
+#define DEBUG_NAME "PISA Cache Hint Selector"
+
+using namespace llvm;
+using namespace llvm::PISA;
+
+namespace {
+class PISACacheHintSelector : public llvm::MachineFunctionPass {
+public:
+ static char ID;
+
+ PISACacheHintSelector() : MachineFunctionPass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+ bool runOnMachineFunction(MachineFunction &MF) override;
+};
+} // namespace
+
+char PISACacheHintSelector::ID = 0;
+INITIALIZE_PASS(PISACacheHintSelector, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+MachineFunctionPass *llvm::createPISACacheHintSelectorPass() {
+ return new PISACacheHintSelector();
+}
+
+void PISACacheHintSelector::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+static unsigned getLoadCacheHintsFromFlags(MachineMemOperand::Flags F) {
+ static_assert(MachineMemOperand::MOTargetFlag1 == 1u << 6,
+ "Unexpected flag value");
+
+ if (F & MachineMemOperand::MONonTemporal)
+ return LoadCacheControl_L1UC_L2UC_L3UC;
+
+ return (static_cast<unsigned>(F) >> 6) & 0xFU;
+}
+
+bool PISACacheHintSelector::runOnMachineFunction(MachineFunction &MF) {
+ LLVM_DEBUG(dbgs() << "Running PISA Cache Hint Selector\n");
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ if (MI.memoperands_empty())
+ continue;
+
+ LLVM_DEBUG(dbgs() << "MI: " << MI);
+
+ const int CacheHintIdx =
+ PISA::getNamedOperandIdx(MI.getOpcode(), PISA::OpName::cachehint);
+ if (CacheHintIdx == -1)
+ continue;
+
+ auto *MMO = *MI.memoperands_begin();
+ if (!(MMO->isLoad() || MMO->isStore()))
+ continue;
+
+ PISAAS::AddressSpace AS =
+ static_cast<PISAAS::AddressSpace>(MMO->getAddrSpace());
+ // Shared address space exists in local memory only and
+ // cache hints make no sense for it.
+ if (AS == PISAAS::AddressSpace::SHARED)
+ continue;
+
+ // If the cache hint operand is already set, skip it.
+ if (MI.getOperand(CacheHintIdx).getImm() != 0)
+ continue;
+
+ unsigned int CacheHint = getLoadCacheHintsFromFlags(MMO->getFlags());
+ if (CacheHint != 1 && (MMO->isStore() || CacheHint != 15))
+ MI.getOperand(CacheHintIdx).setImm(CacheHint);
+
+ LLVM_DEBUG(dbgs() << "Updated MI: " << MI);
+ }
+ }
+
+ return false;
+}
diff --git a/llvm/lib/Target/PISA/PISAConstProp.cpp b/llvm/lib/Target/PISA/PISAConstProp.cpp
new file mode 100644
index 0000000000000..35c7b7b0c680a
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAConstProp.cpp
@@ -0,0 +1,58 @@
+//===-- PISAConstProp.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAConstProp.h"
+#include "llvm/IR/Constants.h"
+
+#include <cmath>
+
+namespace llvm {
+namespace PISA {
+namespace ConstProp {
+
+Constant *foldFrcp(ConstantFP *C0) {
+ auto APF = C0->getValueAPF();
+ double C0value = C0->getType()->isFloatTy()
+ ? static_cast<double>(APF.convertToFloat())
+ : APF.convertToDouble();
+ if (C0->isNaN())
+ return ConstantFP::getQNaN(C0->getType(), C0->isNegative());
+ if (!C0value)
+ return ConstantFP::getInfinity(C0->getType(), C0->isNegative());
+
+ return ConstantFP::get(C0->getType(), 1. / C0value);
+}
+
+Constant *foldFrsqrt(ConstantFP *C0) {
+ auto APF = C0->getValueAPF();
+ double C0value = C0->getType()->isFloatTy()
+ ? static_cast<double>(APF.convertToFloat())
+ : APF.convertToDouble();
+ if (C0->isNaN() || C0value < 0)
+ return ConstantFP::getQNaN(C0->getType(), C0->isNegative());
+ if (!C0value)
+ return ConstantFP::getInfinity(C0->getType());
+
+ return ConstantFP::get(C0->getType(), sqrt(1. / C0value));
+}
+
+Constant *foldFtanh(ConstantFP *C0) {
+ auto APF = C0->getValueAPF();
+ double C0value = C0->getType()->isFloatTy()
+ ? static_cast<double>(APF.convertToFloat())
+ : APF.convertToDouble();
+ if (C0->isNaN())
+ return ConstantFP::getQNaN(C0->getType(), C0->isNegative());
+
+ const double Th = tanh(C0value);
+ return ConstantFP::get(C0->getType(), Th);
+}
+
+} // namespace ConstProp
+} // namespace PISA
+} // namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAConstProp.h b/llvm/lib/Target/PISA/PISAConstProp.h
new file mode 100644
index 0000000000000..e978732874a0e
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAConstProp.h
@@ -0,0 +1,31 @@
+//===-- PISAConstProp.h ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISACONSTPROP_H
+#define LLVM_LIB_TARGET_PISA_PISACONSTPROP_H
+
+#include "llvm/IR/Constants.h"
+
+namespace llvm {
+namespace PISA {
+///
+/// Namespace implementing constant folding/propagation (PISA own additional
+/// functionality to common LLVM constant folding/propagation).
+///
+namespace ConstProp {
+// Frcp (1/x)
+Constant *foldFrcp(ConstantFP *C0);
+// Frsqrt (1/sqrt(x))
+Constant *foldFrsqrt(ConstantFP *C0);
+// Hyperbolic tangent
+Constant *foldFtanh(ConstantFP *C0);
+} // namespace ConstProp
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISACONSTPROP_H
diff --git a/llvm/lib/Target/PISA/PISAEmitIntrinsics.cpp b/llvm/lib/Target/PISA/PISAEmitIntrinsics.cpp
new file mode 100644
index 0000000000000..8ee055a35be63
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAEmitIntrinsics.cpp
@@ -0,0 +1,565 @@
+//===-- PISAEmitIntrinsics.cpp - emit PISA intrinsics ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+
+#define DEBUG_TYPE "pisa-emit-intrinsics"
+#define DEBUG_NAME "PISA emit intrinsics"
+
+using namespace llvm;
+
+namespace llvm {
+void initializePISAEmitIntrinsicsPass(PassRegistry &);
+} // namespace llvm
+
+// Helper that answers two related questions about the SNaN/QNaN payload of
+// a value, both used to decide when @llvm.fabs is safe to lower to the
+// PISA hardware fabs (which quiets SNaN inputs):
+//
+// * cannotBeSNaN(V) - def-side walk: is V's bit pattern provably never a
+// signaling NaN? True for IEEE arithmetic results (always produce QNaN),
+// integer-to-FP conversions, nnan-flagged producers, non-signaling
+// constants, nofpclass(snan) arguments / call results, and sign-only
+// recursion through fabs / fneg / copysign.
+//
+// * allUsesAreSNaNInsensitive(V) - use-side walk: do every transitive
+// consumer of V treat an SNaN payload the same as a QNaN payload?
+// True when all uses reach fcmp (boolean result), maxnum / minnum /
+// maximumnum / minimumnum (which quiet SNaN internally per IEEE
+// 754-2019), or further sign-only / lane-shuffle ops feeding the same
+// set.
+//
+// Either condition alone is sufficient to rewrite @llvm.fabs to
+// @llvm.pisa.fabs. The two queries are conceptually dual (def-side vs
+// use-side) and share the same recursion infrastructure, depth limit and
+// cache, so they live in a single class. Per-Value caching is important:
+// fabs calls in straight-line code often share common subexpressions, and
+// extractelement / lane-shuffle chains naturally revisit the same value.
+//
+// Other operations (arithmetic-as-a-consumer, bitcast-to-int, store, ret,
+// opaque calls, llvm.maximum / llvm.minimum, etc.) are conservatively
+// rejected. IEEE arithmetic does in fact quiet SNaN at the output, but
+// keeping the use-side rule narrow protects the optimisation under
+// constrained FP environments and integer-bit observers.
+class SNaNPayloadAnalysis {
+ static constexpr unsigned MaxDepth = 8;
+
+ // One cache entry per Value, holding the result of either or both queries
+ // once they've been computed. std::optional makes the "not yet computed"
+ // state explicit, so a Value asked only the def-side question never
+ // populates the use-side answer (and vice versa).
+ struct CachedResult {
+ std::optional<bool> CannotBeSNaN;
+ std::optional<bool> AllUsesInsensitive;
+ };
+ DenseMap<const Value *, CachedResult> Cache;
+
+ // --- Def-side computation -------------------------------------------------
+ bool computeCannotBeSNaN(const Value *V, unsigned Depth) {
+ if (Depth > MaxDepth)
+ return false;
+
+ // Constants: check the bit pattern directly.
+ if (const auto *CFP = dyn_cast<ConstantFP>(V))
+ return !CFP->getValueAPF().isSignaling();
+
+ if (const auto *C = dyn_cast<Constant>(V)) {
+ if (isa<UndefValue>(C) || isa<PoisonValue>(C))
+ return true;
+ if (auto *VTy = dyn_cast<FixedVectorType>(C->getType())) {
+ for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {
+ const Constant *Elt = C->getAggregateElement(I);
+ // getAggregateElement may return null for constants it can't split
+ // (e.g. some ConstantExpr forms). Be conservative.
+ if (!Elt || !computeCannotBeSNaN(Elt, Depth + 1))
+ return false;
+ }
+ return true;
+ }
+ // Other constants (ConstantExpr, etc.); be conservative.
+ return false;
+ }
+
+ // Function arguments / call results with a `nofpclass(snan)` attribute.
+ if (const auto *Arg = dyn_cast<Argument>(V))
+ return (Arg->getNoFPClass() & fcSNan) == fcSNan;
+ if (const auto *CB = dyn_cast<CallBase>(V))
+ if ((CB->getRetNoFPClass() & fcSNan) == fcSNan)
+ return true;
+
+ // nnan implies no NaN at all, hence no signaling NaN.
+ if (const auto *FPOp = dyn_cast<FPMathOperator>(V))
+ if (FPOp->hasNoNaNs())
+ return true;
+
+ const auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ return false;
+
+ switch (I->getOpcode()) {
+ // IEEE 754 arithmetic ops produce QNaN, never SNaN.
+ case Instruction::FAdd:
+ case Instruction::FSub:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ // PISA frem is legalized to fdiv+fma+selects, all IEEE arithmetic that
+ // produces QNaN whenever it produces a NaN.
+ return true;
+ // Integer-to-FP conversions can never produce a NaN.
+ case Instruction::SIToFP:
+ case Instruction::UIToFP:
+ return true;
+ // FNeg only flips the sign bit; SNaN-ness is preserved.
+ case Instruction::FNeg:
+ return cannotBeSNaN(I->getOperand(0), Depth + 1);
+ case Instruction::Select:
+ return cannotBeSNaN(I->getOperand(1), Depth + 1) &&
+ cannotBeSNaN(I->getOperand(2), Depth + 1);
+ case Instruction::ExtractElement:
+ return cannotBeSNaN(I->getOperand(0), Depth + 1);
+ case Instruction::InsertElement:
+ return cannotBeSNaN(I->getOperand(0), Depth + 1) &&
+ cannotBeSNaN(I->getOperand(1), Depth + 1);
+ case Instruction::Call: {
+ const auto *II = dyn_cast<IntrinsicInst>(I);
+ if (!II)
+ return false;
+ switch (II->getIntrinsicID()) {
+ // Generic LLVM math intrinsics produce QNaN per IEEE 754.
+ case Intrinsic::fma:
+ case Intrinsic::fmuladd:
+ case Intrinsic::sqrt:
+ case Intrinsic::sin:
+ case Intrinsic::cos:
+ case Intrinsic::tan:
+ case Intrinsic::exp:
+ case Intrinsic::exp2:
+ case Intrinsic::log:
+ case Intrinsic::log2:
+ case Intrinsic::log10:
+ case Intrinsic::pow:
+ case Intrinsic::powi:
+ case Intrinsic::minnum:
+ case Intrinsic::maxnum:
+ case Intrinsic::minimum:
+ case Intrinsic::maximum:
+ case Intrinsic::canonicalize:
+ case Intrinsic::pisa_fabs:
+ case Intrinsic::pisa_fadd:
+ case Intrinsic::pisa_fsub:
+ case Intrinsic::pisa_fmul:
+ case Intrinsic::pisa_fma:
+ case Intrinsic::pisa_fdiv_rnd:
+ case Intrinsic::pisa_pow_rnd:
+ case Intrinsic::pisa_fsqrt_rnd:
+ case Intrinsic::pisa_frnd_rnd:
+ case Intrinsic::pisa_frcp:
+ case Intrinsic::pisa_frcp_rnd:
+ case Intrinsic::pisa_frsqrt:
+ case Intrinsic::pisa_sin_rnd:
+ case Intrinsic::pisa_cos_rnd:
+ case Intrinsic::pisa_tanh_rnd:
+ case Intrinsic::pisa_exp_rnd:
+ case Intrinsic::pisa_exp2_rnd:
+ case Intrinsic::pisa_log_rnd:
+ case Intrinsic::pisa_log2_rnd:
+ case Intrinsic::pisa_log10_rnd:
+ case Intrinsic::pisa_fmin_sat:
+ case Intrinsic::pisa_fmax_sat:
+ case Intrinsic::pisa_ftrunc:
+ case Intrinsic::pisa_sitofp:
+ case Intrinsic::pisa_uitofp:
+ return true;
+ // IR fabs / copysign preserve the magnitude operand's NaN payload, so
+ // SNaN-ness is preserved; recurse into the source.
+ case Intrinsic::fabs:
+ case Intrinsic::copysign:
+ return cannotBeSNaN(II->getArgOperand(0), Depth + 1);
+ default:
+ return false;
+ }
+ }
+ default:
+ return false;
+ }
+ }
+
+ // --- Use-side computation -------------------------------------------------
+ bool computeAllUsesInsensitive(const Value *V, unsigned Depth) {
+ if (Depth > MaxDepth)
+ return false;
+ if (V->use_empty())
+ // No observable users at all - vacuously insensitive.
+ return true;
+ for (const User *U : V->users()) {
+ const auto *I = dyn_cast<Instruction>(U);
+ if (!I || !isUserInsensitive(*I, Depth + 1))
+ return false;
+ }
+ return true;
+ }
+
+ bool isUserInsensitive(const Instruction &I, unsigned Depth) {
+ switch (I.getOpcode()) {
+ case Instruction::FCmp:
+ return true;
+ case Instruction::ExtractElement:
+ case Instruction::InsertElement:
+ case Instruction::ShuffleVector:
+ return allUsesAreSNaNInsensitive(&I, Depth);
+ case Instruction::Call: {
+ const auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ return false;
+ switch (II->getIntrinsicID()) {
+ // Per IEEE 754-2019 maxnum/minnum treat SNaN as missing data, quieting
+ // it internally. The distinction in the result is gone.
+ case Intrinsic::maxnum:
+ case Intrinsic::minnum:
+ case Intrinsic::maximumnum:
+ case Intrinsic::minimumnum:
+ return true;
+ // Constrained maxnum/minnum and non-signaling fcmp: the data result
+ // is insensitive to SNaN vs QNaN, but under fpexcept.strict /
+ // fpexcept.maytrap the exception flags differ (IEEE 754 mandates
+ // FE_INVALID for SNaN operands). Only safe when exceptions are
+ // ignored.
+ case Intrinsic::experimental_constrained_maxnum:
+ case Intrinsic::experimental_constrained_minnum:
+ case Intrinsic::experimental_constrained_fcmp: {
+ const auto *CFP = cast<ConstrainedFPIntrinsic>(II);
+ auto EB = CFP->getExceptionBehavior();
+ return EB && *EB == fp::ebIgnore;
+ }
+ // Sign-only operations are bit-preserving on the payload, so the
+ // distinction propagates through; recurse to the consumers.
+ case Intrinsic::fabs:
+ case Intrinsic::pisa_fabs:
+ case Intrinsic::copysign:
+ return allUsesAreSNaNInsensitive(II, Depth);
+ // experimental.constrained.fcmps - the signaling comparison signals
+ // FE_INVALID on *any* NaN (SNaN or QNaN), so quieting makes no
+ // difference to exception behavior.
+ case Intrinsic::experimental_constrained_fcmps:
+ return true;
+ default:
+ return false;
+ }
+ }
+ default:
+ return false;
+ }
+ }
+
+public:
+ // Query whether V is provably never a signaling NaN. Result is cached.
+ bool cannotBeSNaN(const Value *V, unsigned Depth = 0) {
+ auto &Slot = Cache[V];
+ if (Slot.CannotBeSNaN)
+ return *Slot.CannotBeSNaN;
+ bool Result = computeCannotBeSNaN(V, Depth);
+ // Re-fetch: recursion may have inserted intermediate entries and
+ // invalidated the earlier reference.
+ Cache[V].CannotBeSNaN = Result;
+ return Result;
+ }
+
+ // Query whether all transitive users of V are SNaN-payload-insensitive.
+ // Result is cached.
+ bool allUsesAreSNaNInsensitive(const Value *V, unsigned Depth = 0) {
+ auto &Slot = Cache[V];
+ if (Slot.AllUsesInsensitive)
+ return *Slot.AllUsesInsensitive;
+ bool Result = computeAllUsesInsensitive(V, Depth);
+ // Re-fetch: recursion may have inserted intermediate entries and
+ // invalidated the earlier reference.
+ Cache[V].AllUsesInsensitive = Result;
+ return Result;
+ }
+};
+
+namespace {
+class PISAEmitIntrinsics : public FunctionPass,
+ public InstVisitor<PISAEmitIntrinsics> {
+
+ IRBuilder<> *IRB = nullptr;
+ bool Changed = false;
+ SNaNPayloadAnalysis SNaNAnalysis;
+
+public:
+ static char ID;
+ PISAEmitIntrinsics() : FunctionPass(ID) {
+ initializePISAEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
+ }
+
+ bool runOnFunction(Function &F) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ }
+
+ void visitIntrinsicInst(IntrinsicInst &I);
+};
+} // namespace
+
+char PISAEmitIntrinsics::ID = 0;
+INITIALIZE_PASS(PISAEmitIntrinsics, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+// given a metadata node, return CmpInst::Predicate representation
+static CmpInst::Predicate getPredicateFromName(StringRef Name) {
+ CmpInst::Predicate Pred;
+ Pred = StringSwitch<CmpInst::Predicate>(Name)
+ .Case("false", CmpInst::FCMP_FALSE)
+ .Case("oeq", CmpInst::FCMP_OEQ)
+ .Case("ogt", CmpInst::FCMP_OGT)
+ .Case("oge", CmpInst::FCMP_OGE)
+ .Case("olt", CmpInst::FCMP_OLT)
+ .Case("ole", CmpInst::FCMP_OLE)
+ .Case("one", CmpInst::FCMP_ONE)
+ .Case("ord", CmpInst::FCMP_ORD)
+ .Case("uno", CmpInst::FCMP_UNO)
+ .Case("ueq", CmpInst::FCMP_UEQ)
+ .Case("ugt", CmpInst::FCMP_UGT)
+ .Case("uge", CmpInst::FCMP_UGE)
+ .Case("ult", CmpInst::FCMP_ULT)
+ .Case("ule", CmpInst::FCMP_ULE)
+ .Case("une", CmpInst::FCMP_UNE)
+ .Case("true", CmpInst::FCMP_TRUE)
+ .Default(CmpInst::BAD_FCMP_PREDICATE);
+ return Pred;
+}
+
+// replace one intrinsic with another, stripping some args
+static void replaceIntrinsicWith(IntrinsicInst *II, Intrinsic::ID IID,
+ unsigned IgnoreLast) {
+ IRBuilder<> IRB(II);
+
+ SmallVector<Value *, 4> Args;
+ // NOLINTNEXTLINE(llvm-qualified-auto)
+ for (auto It = II->arg_begin(); It != (II->arg_end() - IgnoreLast); ++It)
+ Args.push_back(*It);
+ auto *NewI = IRB.CreateIntrinsic(II->getType(), IID, Args,
+ isa<FPMathOperator>(II) ? II : nullptr);
+ II->replaceAllUsesWith(NewI);
+ II->eraseFromParent();
+}
+
+// replace metadata with equivalent numerical representation
+static void replaceRoundingModeMD(IntrinsicInst *II, Intrinsic::ID IID,
+ bool IsConstrained) {
+ IRBuilder<> IRB(II);
+ auto *ImmTy = Type::getInt8Ty(II->getContext());
+
+ // constrained have roundmode, exception on the end
+ // non-constrained only have roundmode on the end
+ auto IgnoreLast = IsConstrained ? 2 : 1;
+ auto *MD =
+ cast<MetadataAsValue>(II->getArgOperand(II->arg_size() - IgnoreLast))
+ ->getMetadata();
+ auto RoundMode = convertStrToRoundingMode(cast<MDString>(MD)->getString());
+ // PISA has no runtime rounding-mode control. Map round.dynamic to the
+ // IEEE 754 default (round-to-nearest-even).
+ if (!RoundMode || *RoundMode == RoundingMode::Dynamic)
+ RoundMode = RoundingMode::NearestTiesToEven;
+ Constant *RoundVal = ConstantInt::get(ImmTy, (unsigned)*RoundMode);
+
+ SmallVector<Value *, 4> Args;
+ // NOLINTNEXTLINE(llvm-qualified-auto)
+ for (auto It = II->arg_begin(); It != (II->arg_end() - IgnoreLast); ++It)
+ Args.push_back(*It);
+ Args.push_back(RoundVal);
+ switch (IID) {
+ default:
+ break;
+ case Intrinsic::pisa_fadd:
+ case Intrinsic::pisa_fsub:
+ case Intrinsic::pisa_fmul:
+ case Intrinsic::pisa_fma:
+ case Intrinsic::pisa_ftrunc:
+ case Intrinsic::pisa_uitofp:
+ case Intrinsic::pisa_sitofp:
+ Args.push_back(ConstantInt::getFalse(II->getContext())); // saturation
+ break;
+ }
+ auto *NewI = IRB.CreateIntrinsic(II->getType(), IID, Args,
+ isa<FPMathOperator>(II) ? II : nullptr);
+ II->replaceAllUsesWith(NewI);
+ II->eraseFromParent();
+}
+
+void PISAEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
+ IRB->SetInsertPoint(&I);
+ auto II = I.getIntrinsicID();
+ switch (II) {
+// intrinsics with rounding mode (e.g. llvm.experimental.constrained*) are
+// mapped into PISA equivalents, with rounding mode MD being mapped to an
+// equivalent immediate values. ISel maps these to proper instructions.
+#define REPLACE_ROUNDMODE(from, to, constrained) \
+ case from: { \
+ replaceRoundingModeMD(&I, to, constrained); \
+ Changed = true; \
+ break; \
+ }
+ REPLACE_ROUNDMODE(Intrinsic::pisa_fptoui_md, Intrinsic::pisa_fptoui_rnd,
+ false)
+ REPLACE_ROUNDMODE(Intrinsic::pisa_fptosi_md, Intrinsic::pisa_fptosi_rnd,
+ false)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fadd,
+ Intrinsic::pisa_fadd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fsub,
+ Intrinsic::pisa_fsub, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fmul,
+ Intrinsic::pisa_fmul, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fdiv,
+ Intrinsic::pisa_fdiv_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_uitofp,
+ Intrinsic::pisa_uitofp, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_sitofp,
+ Intrinsic::pisa_sitofp, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_sqrt,
+ Intrinsic::pisa_fsqrt_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fma,
+ Intrinsic::pisa_fma, true)
+ // TODO: check if we need to split into mul+add
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fmuladd,
+ Intrinsic::pisa_fma, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_fptrunc,
+ Intrinsic::pisa_ftrunc, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_rint,
+ Intrinsic::pisa_frnd_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_log2,
+ Intrinsic::pisa_log2_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_sin,
+ Intrinsic::pisa_sin_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_cos,
+ Intrinsic::pisa_cos_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_tanh,
+ Intrinsic::pisa_tanh_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_exp,
+ Intrinsic::pisa_exp_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_exp2,
+ Intrinsic::pisa_exp2_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_log,
+ Intrinsic::pisa_log_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_log10,
+ Intrinsic::pisa_log10_rnd, true)
+ REPLACE_ROUNDMODE(Intrinsic::experimental_constrained_pow,
+ Intrinsic::pisa_pow_rnd, true)
+#undef REPLACE_ROUNDMODE
+#define REPLACE_INTRINSIC(from, to, strip) \
+ case from: { \
+ replaceIntrinsicWith(&I, to, strip); \
+ Changed = true; \
+ break; \
+ }
+ REPLACE_INTRINSIC(Intrinsic::experimental_constrained_minnum,
+ Intrinsic::minnum, 1)
+ REPLACE_INTRINSIC(Intrinsic::experimental_constrained_maxnum,
+ Intrinsic::maxnum, 1)
+ REPLACE_INTRINSIC(Intrinsic::experimental_constrained_floor,
+ Intrinsic::floor, 1)
+ REPLACE_INTRINSIC(Intrinsic::experimental_constrained_ceil, Intrinsic::ceil,
+ 1)
+ REPLACE_INTRINSIC(Intrinsic::experimental_constrained_round,
+ Intrinsic::round, 1)
+ REPLACE_INTRINSIC(Intrinsic::experimental_constrained_trunc,
+ Intrinsic::trunc, 1)
+#undef REPLACE_INTRINSIC
+ case Intrinsic::experimental_constrained_fptosi: {
+ // exceptions are not supported
+ auto *NewI = IRB->CreateFPToSI(I.getOperand(0), I.getType());
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ Changed = true;
+ } break;
+ case Intrinsic::experimental_constrained_fptoui: {
+ // exceptions are not supported
+ auto *NewI = IRB->CreateFPToUI(I.getOperand(0), I.getType());
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ Changed = true;
+ } break;
+ case Intrinsic::experimental_constrained_fpext: {
+ // exceptions are not supported
+ auto *NewI = IRB->CreateFPExt(I.getOperand(0), I.getType());
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ Changed = true;
+ } break;
+ case Intrinsic::experimental_constrained_fcmp: {
+ // exceptions are not supported
+ auto *MD = cast<MetadataAsValue>(I.getOperand(2))->getMetadata();
+ auto Predicate = getPredicateFromName(cast<MDString>(MD)->getString());
+ auto *NewI = IRB->CreateFCmp(Predicate, I.getOperand(0), I.getOperand(1));
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ Changed = true;
+ } break;
+ case Intrinsic::experimental_constrained_fcmps: {
+ // exceptions are not supported
+ auto *MD = cast<MetadataAsValue>(I.getOperand(2))->getMetadata();
+ auto Predicate = getPredicateFromName(cast<MDString>(MD)->getString());
+ auto *NewI = IRB->CreateFCmpS(Predicate, I.getOperand(0), I.getOperand(1));
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ Changed = true;
+ } break;
+ case Intrinsic::experimental_constrained_frem: {
+ // https://llvm.org/docs/LangRef.html#llvm-experimental-constrained-frem-intrinsic
+ // ... rounding mode argument has no effect ...
+ auto *NewI = IRB->CreateFRem(I.getOperand(0), I.getOperand(1));
+ if (auto *CastedNewI = dyn_cast<Instruction>(NewI))
+ CastedNewI->setFastMathFlags(I.getFastMathFlags());
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ Changed = true;
+ } break;
+ case Intrinsic::fabs: {
+ // PISA fabs and IEEE fabs differ only on signaling NaN inputs: PISA fabs
+ // quiets the NaN, while IEEE fabs preserves the payload. The cheaper
+ // PISA fabs is safe whenever the SNaN-vs-QNaN distinction cannot affect
+ // any observable result at this call site. Two sufficient conditions:
+ //
+ // * cannotBeSNaN(&I) - the fabs result is provably never an SNaN.
+ // For a fabs call this folds in the nnan flag on the call itself
+ // (FPMathOperator::hasNoNaNs) and recurses into the operand's
+ // def chain (IEEE arithmetic, sitofp, nnan producers, ...).
+ // * allUsesAreSNaNInsensitive(&I) - no consumer of the fabs result
+ // observes the SNaN payload (uses bottom out in maxnum/minnum,
+ // fcmp, or lane-shuffle/sign-only chains into the same).
+ if (SNaNAnalysis.cannotBeSNaN(&I) ||
+ SNaNAnalysis.allUsesAreSNaNInsensitive(&I)) {
+ replaceIntrinsicWith(&I, Intrinsic::pisa_fabs, 0);
+ Changed = true;
+ }
+ } break;
+ default:
+ break;
+ }
+}
+
+bool PISAEmitIntrinsics::runOnFunction(Function &Func) {
+ SNaNAnalysis = SNaNPayloadAnalysis();
+ IRBuilder<> LocalIRB(Func.getContext());
+
+ IRB = &LocalIRB;
+ Changed = false;
+
+ visit(Func);
+
+ return Changed;
+}
+
+FunctionPass *llvm::createPISAEmitIntrinsicsPass() {
+ return new PISAEmitIntrinsics();
+}
diff --git a/llvm/lib/Target/PISA/PISAExpandIntrinsics.cpp b/llvm/lib/Target/PISA/PISAExpandIntrinsics.cpp
new file mode 100644
index 0000000000000..1b6572e349e4b
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAExpandIntrinsics.cpp
@@ -0,0 +1,106 @@
+//===-- PISAExpandIntrinsics.cpp - modify function signatures -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISASubtarget.h"
+#include "PISATargetMachine.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
+
+#define DEBUG_TYPE "pisa-expand-intrinsics"
+#define DEBUG_NAME "PISA expand intrinsics"
+
+using namespace llvm;
+
+namespace {
+
+class PISAExpandIntrinsics : public FunctionPass {
+public:
+ static char ID;
+
+ PISAExpandIntrinsics() : FunctionPass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetPassConfig>();
+ AU.addRequired<TargetTransformInfoWrapperPass>();
+ }
+ bool runOnFunction(Function &F) override;
+};
+
+} // namespace
+
+char PISAExpandIntrinsics::ID = 0;
+
+INITIALIZE_PASS_BEGIN(PISAExpandIntrinsics, DEBUG_TYPE, DEBUG_NAME, false,
+ false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_END(PISAExpandIntrinsics, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+bool PISAExpandIntrinsics::runOnFunction(Function &F) {
+ auto &TPC = getAnalysis<TargetPassConfig>();
+ auto &TM = TPC.getTM<TargetMachine>();
+ const auto *ST = TM.getSubtargetImpl(F);
+ const auto *TLI = ST->getTargetLowering();
+
+ SmallVector<MemIntrinsic *> MemIntrs;
+ for (auto &I : instructions(F)) {
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ continue;
+ if (II->getIntrinsicID() == Intrinsic::memset ||
+ II->getIntrinsicID() == Intrinsic::memcpy ||
+ II->getIntrinsicID() == Intrinsic::memmove) {
+ MemIntrinsic *MI = cast<MemIntrinsic>(II);
+ uint64_t Len = ~0, Limit = 0;
+ if (ConstantInt *LenCI = dyn_cast<ConstantInt>(MI->getLength()))
+ Len = LenCI->getZExtValue();
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::memset:
+ Limit = TLI->getMaxStoresPerMemset(F.hasOptSize());
+ break;
+ case Intrinsic::memcpy:
+ Limit = TLI->getMaxStoresPerMemcpy(F.hasOptSize());
+ break;
+ case Intrinsic::memmove:
+ Limit = TLI->getMaxStoresPerMemmove(F.hasOptSize());
+ break;
+ }
+ if (Len > Limit)
+ MemIntrs.push_back(MI);
+ }
+ }
+
+ bool Changed = false;
+
+ // Expand llvm.mem* intrinsics to a loop
+ const TargetTransformInfo &TTI =
+ getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+ for (MemIntrinsic *MemCall : MemIntrs) {
+ if (auto *Memcpy = dyn_cast<MemCpyInst>(MemCall))
+ expandMemCpyAsLoop(Memcpy, TTI);
+ else if (auto *Memmove = dyn_cast<MemMoveInst>(MemCall))
+ expandMemMoveAsLoop(Memmove, TTI);
+ else if (auto *Memset = dyn_cast<MemSetInst>(MemCall))
+ expandMemSetAsLoop(Memset);
+ Changed = true;
+ MemCall->eraseFromParent();
+ }
+
+ return Changed;
+}
+
+FunctionPass *llvm::createPISAExpandIntrinsicsPass() {
+ return new PISAExpandIntrinsics();
+}
diff --git a/llvm/lib/Target/PISA/PISAInsertLifetimeStart.cpp b/llvm/lib/Target/PISA/PISAInsertLifetimeStart.cpp
new file mode 100644
index 0000000000000..a6290af9a9225
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAInsertLifetimeStart.cpp
@@ -0,0 +1,751 @@
+//=== PISAInsertLifetimeStart.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Inserts "lifetime.start %R;" markers at loop headers.
+//
+// What the marker means
+// ---------------------
+// A "lifetime.start %R;" at a loop header H asserts that the value register R
+// holds on entry to the loop from the back-edge (the value carried from the
+// previous iteration) is never used. In other words R is NOT loop-carried: each
+// iteration produces a fresh R before reading it, so no value flows from one
+// iteration to the next through R. Without the marker R looks alive across
+// iterations and must be conservatively preserved; with it, R may be treated as
+// undefined on entry to the loop.
+//
+// The pattern we detect
+// ---------------------
+// We look for a register R with:
+// - an undefined seed before the loop (an IMPLICIT_DEF, so R starts the loop
+// with no real value), and
+// - inside the loop, a real definition D that always runs before any use U of
+// R in the same iteration.
+// When D always precedes U, every U reads the value D just produced this
+// iteration, never the value carried across the back-edge -- so marking R is
+// sound.
+//
+// Why this needs more than dominance
+// ----------------------------------
+// D need not dominate U. A common shape guards the definition block and the use
+// block with the SAME predicate, so the loop header can branch around the
+// definition block:
+//
+// H: goto.cond p -> M ; p true skips the definition
+// D: R = ... ; runs only when p is false
+// M: goto.cond p -> L ; the same p skips the use
+// U: ... = R ; runs only when p is false
+// L: latch
+//
+// Here the path H -> M -> U skips D, so D does not dominate U; but that path
+// needs p both true (to skip D) and false (to reach U), which is impossible.
+// So whenever U runs, D ran first. We capture this by comparing the predicate
+// guards of D and U: U is covered by D when D runs before U and every predicate
+// guarding D also guards U (so U running implies D ran). Values may reach U
+// through pure repacks (copy / insert / extract / mov); we follow R forward
+// through those to its real uses.
+//
+// Detection flow (per register R, per loop L with header H)
+// ---------------------------------------------------------
+// 1. R has an IMPLICIT_DEF seed.
+// 2. R is live across the back-edge (live-in to H, live-out of a latch).
+// 3. R's value does not escape the loop: neither R nor a value repacked from
+// it is observed after the loop (its carried value must not leak out).
+// 4. The value reaching H from outside L is the IMPLICIT_DEF seed, not a real
+// value (an IMPLICIT_DEF dominates H, no real def dominates H, L has a
+// unique preheader).
+// 5. Following R forward through repack ops, every real use U is inside L and
+// is covered by a real def D of R that runs before U (by dominance, or by
+// a shared predicate guard with D ordered before U).
+// If all hold, emit the marker at H.
+//
+// Pass placement and liveness
+// ---------------------------
+// Runs in addPreEmitPass(), after register coalescing, so the marked vreg name
+// matches the emitted name; gated to -O != none. LiveVariables cannot run here
+// (the MIR is non-SSA at this slot), so liveness is recomputed by a small
+// backward block-level dataflow. MachineLoopInfo / MachineDominatorTree /
+// MachinePostDominatorTree are pure-CFG and valid here.
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISAInstrInfo.h"
+#include "PISARegisterInfo.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+// Generated register-class -> lifetime.start opcode lookup.
+namespace llvm {
+namespace PISA {
+struct LifetimeStartEntry {
+ // Since upstream commit 92f01b267efe ([TableGen] Use StringTable for
+ // searchable tables), string columns in SearchableTables are emitted as
+ // StringTable offsets (unsigned) rather than const char* pointers.
+ unsigned RegClassName;
+ unsigned Opcode;
+};
+
+#define GET_LifetimeStartTable_DECL
+#define GET_LifetimeStartTable_IMPL
+#include "PISAGenSearchableTables.inc"
+
+} // namespace PISA
+} // namespace llvm
+
+#define DEBUG_TYPE "pisa-insert-lifetime-start"
+
+STATISTIC(NumLifetimeMarkers, "Number of lifetime.start markers inserted");
+
+// NB: the option name must differ from the pass registration name
+// ("pisa-insert-lifetime-start", used by -run-pass / -start-after /
+// -print-after and the DEBUG_TYPE). The legacy PassNameParser registers every
+// pass's arg name as a CLI literal, so a cl::opt of the same name aborts every
+// tool at startup ("registered more than once"). Hence the "-enable" suffix.
+static cl::opt<bool> LifetimeStartOpt(
+ "pisa-insert-lifetime-start-enable",
+ cl::desc("Insert lifetime.start liveness markers at loop headers"),
+ cl::init(true), cl::Hidden);
+
+namespace {
+
+// Per-block liveness as a BitVector indexed by Register::virtReg2Index (only
+// virtual registers are tracked).
+using LiveMap = DenseMap<const MachineBasicBlock *, BitVector>;
+
+// A predicated branch (predgoto) "goto.cond <negate><cond>, <label>". The block
+// ending in it has two CFG successors: the label target (taken when cond equals
+// !negate) and the fall-through (taken when cond equals negate). PredCtrl
+// records what is needed to attribute each outgoing edge to a (predicate,
+// polarity) pair.
+struct PredCtrl {
+ MachineBasicBlock *Block; // block containing predgoto (source block)
+ MachineBasicBlock *Label; // the explicit branch target operand
+ Register Pred; // the predicate register (cond)
+ bool Negate; // the negate flag (operand 0)
+};
+
+// One packed (predicate vreg, polarity) guard key, used for set membership.
+using PredKey = uint64_t;
+
+// Pack (predicate vreg, polarity) into one key for set membership.
+inline PredKey predKey(Register P, bool Pol) {
+ return (static_cast<PredKey>(P.id()) << 1) | (Pol ? 1u : 0u);
+}
+
+class PISAInsertLifetimeStart : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISAInsertLifetimeStart();
+
+ StringRef getPassName() const override {
+ return "PISA Insert lifetime.start";
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+private:
+ MachineRegisterInfo *MRI = nullptr;
+ const TargetInstrInfo *TII = nullptr;
+ MachineDominatorTree *MDT = nullptr;
+ MachinePostDominatorTree *MPDT = nullptr;
+ // Every predicated branch in the function. Control-dependence is a whole-CFG
+ // property, so this is gathered once and reused for all candidates.
+ SmallVector<PredCtrl, 16> PredCtrls;
+
+ void computeBlockLiveness(MachineFunction &MF, LiveMap &LiveIn,
+ LiveMap &LiveOut) const;
+
+ bool isBackEdgeLive(Register R, const MachineLoop *L, const LiveMap &LiveIn,
+ const LiveMap &LiveOut) const;
+
+ bool isLiveOutOfLoop(Register R, const MachineLoop *L,
+ const LiveMap &LiveIn) const;
+
+ bool implicitDefSeedReachesHeader(const MachineRegisterInfo &MRI, Register R,
+ const MachineLoop *L,
+ const MachineDominatorTree &MDT) const;
+
+ SmallVector<std::pair<const MachineBasicBlock *, PredKey>, 8>
+ directCtrlDeps(const MachineBasicBlock *B) const;
+
+ DenseSet<PredKey> transitiveCtrlSet(const MachineBasicBlock *B) const;
+
+ bool isRealFullDef(const MachineInstr &MI, Register R) const;
+
+ bool defRunsBeforeUse(const MachineInstr &D, const MachineInstr &U) const;
+
+ bool reachesWithinIteration(const MachineBasicBlock *From,
+ const MachineBasicBlock *To,
+ const MachineLoop *L) const;
+
+ bool hasTransitiveControlDependence(Register R, const MachineLoop *L) const;
+
+ // Returns true iff any marker was inserted.
+ bool run(MachineFunction &MF, MachineLoopInfo &MLI);
+};
+
+} // namespace
+
+// Pick the typed marker variant for R's register class; 0 if none (then the
+// caller skips R). The register-class -> opcode mapping is generated from
+// VTs.LifetimeTypes (see LifetimeStartTable in PISAInstrInfo.td).
+static unsigned pickLifetimeStartOpcode(const TargetRegisterInfo &TRI,
+ const TargetRegisterClass *RC) {
+ const auto *Entry =
+ PISA::lookupLifetimeStartByRegClass(TRI.getRegClassName(RC));
+ return Entry ? Entry->Opcode : 0;
+}
+
+// True if MI only repacks its operand bits -- copy / undef / insert / extract /
+// mov -- so the value flows through it without being observed.
+static bool isForwardingOpcode(const MachineInstr &MI,
+ const TargetInstrInfo &TII) {
+ if (MI.isCopy() || MI.isImplicitDef())
+ return true;
+ StringRef Name = TII.getName(MI.getOpcode());
+ return Name.starts_with("insert_") || Name.starts_with("extract_") ||
+ Name.starts_with("mov_");
+}
+
+void PISAInsertLifetimeStart::getAnalysisUsage(AnalysisUsage &AU) const {
+ // Both are pure-CFG / SSA-independent and valid at addPreEmitPass.
+ // LiveVariables is deliberately NOT required: it rejects the non-SSA
+ // MIR present at this slot (see file header) -- liveness is recomputed
+ // manually.
+ AU.addRequired<MachineLoopInfoWrapperPass>();
+ AU.addRequired<MachineDominatorTreeWrapperPass>();
+ // Post-dominators back the control-dependence test.
+ AU.addRequired<MachinePostDominatorTreeWrapperPass>();
+ AU.setPreservesCFG(); // we only ever insert marker MIs
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+PISAInsertLifetimeStart::PISAInsertLifetimeStart() : MachineFunctionPass(ID) {
+ initializePISAInsertLifetimeStartPass(*PassRegistry::getPassRegistry());
+}
+
+// Manual block-level liveness over virtual registers (NOT LiveVariables --
+// invalid at this slot, see file header). Fills LiveIn/LiveOut, each
+// BitVector sized to MRI.getNumVirtRegs() and indexed by
+// Register::virtReg2Index.
+void PISAInsertLifetimeStart::computeBlockLiveness(MachineFunction &MF,
+ LiveMap &LiveIn,
+ LiveMap &LiveOut) const {
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const unsigned NumV = MRI.getNumVirtRegs();
+
+ // Local transfer functions, computed once per block:
+ // Def[B] = registers fully defined somewhere in B.
+ // UpUse[B] = upward-exposed uses (read in B before any def in B).
+ LiveMap Def, UpUse;
+ for (MachineBasicBlock &MBB : MF) {
+ BitVector DefB(NumV), UseB(NumV), DefSoFar(NumV);
+ for (MachineInstr &MI : MBB) {
+ if (MI.isDebugInstr())
+ continue;
+ // Uses first: a use reads the value defined by an EARLIER instruction, so
+ // it is upward-exposed unless this block already (fully) defined it.
+ for (const MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.getReg().isVirtual() || !MO.readsReg())
+ continue;
+ unsigned I = MO.getReg().virtRegIndex();
+ if (!DefSoFar.test(I))
+ UseB.set(I);
+ }
+ // Then defs. A full def (subreg 0) kills upward liveness; a sub-register
+ // def preserves the other bits, so it does not kill (and its read of the
+ // remaining bits was already counted as a use above via readsReg()).
+ for (const MachineOperand &MO : MI.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.getReg().isVirtual())
+ continue;
+ if (MO.getSubReg() != PISA::NoSubRegister)
+ continue; // partial def: not a kill
+ unsigned I = MO.getReg().virtRegIndex();
+ DefB.set(I);
+ DefSoFar.set(I);
+ }
+ }
+ Def[&MBB] = std::move(DefB);
+ UpUse[&MBB] = std::move(UseB);
+ LiveIn[&MBB] = BitVector(NumV);
+ LiveOut[&MBB] = BitVector(NumV);
+ }
+
+ // Backward iterative dataflow to fixpoint:
+ // LiveOut[B] = U_{S in succ(B)} LiveIn[S]
+ // LiveIn[B] = UpUse[B] U (LiveOut[B] \ Def[B])
+ bool Changed = true;
+ while (Changed) {
+ Changed = false;
+ // Reverse layout order converges faster for mostly-forward CFGs.
+ for (MachineBasicBlock &MBB : reverse(MF)) {
+ BitVector Out(NumV);
+ for (const MachineBasicBlock *Succ : MBB.successors())
+ Out |= LiveIn[Succ];
+
+ BitVector In = Out;
+ In.reset(Def[&MBB]); // Out & ~Def
+ In |= UpUse[&MBB];
+
+ if (Out != LiveOut[&MBB]) {
+ LiveOut[&MBB] = std::move(Out);
+ Changed = true;
+ }
+ if (In != LiveIn[&MBB]) {
+ LiveIn[&MBB] = std::move(In);
+ Changed = true;
+ }
+ }
+ }
+}
+
+// True iff R is live across L's back-edge: live-in to the header and live-out
+// of at least one latch.
+bool PISAInsertLifetimeStart::isBackEdgeLive(Register R, const MachineLoop *L,
+ const LiveMap &LiveIn,
+ const LiveMap &LiveOut) const {
+ const unsigned I = R.virtRegIndex();
+ const MachineBasicBlock *H = L->getHeader();
+ auto HIt = LiveIn.find(H);
+ if (HIt == LiveIn.end() || !HIt->second.test(I))
+ return false; // not live-in to the header
+
+ SmallVector<MachineBasicBlock *, 4> Latches;
+ L->getLoopLatches(Latches);
+ for (const MachineBasicBlock *Latch : Latches) {
+ auto LIt = LiveOut.find(Latch);
+ if (LIt != LiveOut.end() && LIt->second.test(I))
+ return true; // live across the latch -> header back-edge
+ }
+ return false;
+}
+
+// True if R's value is observed after the loop (R live-in to a loop-exit
+// block); such a register must not be marked because its carried value
+// escapes.
+bool PISAInsertLifetimeStart::isLiveOutOfLoop(Register R, const MachineLoop *L,
+ const LiveMap &LiveIn) const {
+ const unsigned I = R.virtRegIndex();
+ SmallVector<MachineBasicBlock *, 4> ExitBlocks;
+ L->getExitBlocks(ExitBlocks);
+ for (const MachineBasicBlock *S : ExitBlocks) {
+ auto It = LiveIn.find(S);
+ if (It != LiveIn.end() && It->second.test(I))
+ return true; // value escapes the loop -> observed after the loop
+ }
+ return false;
+}
+
+// True iff it is SOUND to assert R dead at L's header: the value of R that
+// reaches the header from OUTSIDE the loop is the IMPLICIT_DEF undef seed,
+// not a real (live) value. This uses a dominance approximation; it also
+// subsumes the verifier requirement that the inserted header USE be dominated
+// by a def.
+//
+// Requires ALL of:
+// (1) an IMPLICIT_DEF def of R STRICTLY dominates the header -- the undef
+// seed (a def in H itself sits at/after the insertion point at this
+// post-PHIElim slot, so it does not dominate the marker use);
+// (2) NO non-IMPLICIT_DEF def of R dominates the header -- a real
+// dominating def (e.g. a preheader load that seeds a loop-carried
+// reduction) is the live value entering the loop, so "dead at header"
+// would delete a live seed and miscompile; and
+// (3) the loop has a UNIQUE preheader, so the single loop-entry path makes
+// the closest dominating def the reaching def at the header. Without
+// it, a real def on one of several entry edges need not dominate the
+// header, so (2) could pass while a live value still reaches it.
+//
+// Defs INSIDE the loop body (the loop-carried redefinition the marker is
+// entitled to discard) do not dominate the header and are correctly ignored.
+bool PISAInsertLifetimeStart::implicitDefSeedReachesHeader(
+ const MachineRegisterInfo &MRI, Register R, const MachineLoop *L,
+ const MachineDominatorTree &MDT) const {
+ const MachineBasicBlock *H = L->getHeader();
+
+ // (3) Unique preheader -> a single loop-entry path into the header, so the
+ // closest def dominating H is exactly the value reaching it from outside.
+ if (!L->getLoopPreheader())
+ return false;
+
+ bool ImplicitDefDominates = false;
+ for (const MachineInstr &DefMI : MRI.def_instructions(R)) {
+ const MachineBasicBlock *DefBB = DefMI.getParent();
+ // A def in H itself sits at or after the insertion point (this slot is
+ // post-PHIElim / NoPHIs, so getFirstNonPHI() is the block top), so it does
+ // not dominate the inserted use; only STRICT dominators of H carry a value
+ // into the loop along the entry path.
+ if (DefBB == H || !MDT.dominates(DefBB, H))
+ continue;
+ if (DefMI.isImplicitDef())
+ ImplicitDefDominates = true; // (1) the undef seed dominates the header
+ else
+ return false; // (2) a real value reaches the header -> not dead on entry
+ }
+ return ImplicitDefDominates;
+}
+
+// Collect direct (immediate) control dependences of block B: which predicated
+// branches directly decide whether B runs, together with the predicate
+// polarity that forces B to execute. B is control-dependent on edge (A -> C)
+// iff B post-dominates C and B does not (reflexively) post-dominate A.
+SmallVector<std::pair<const MachineBasicBlock *, PredKey>, 8>
+PISAInsertLifetimeStart::directCtrlDeps(const MachineBasicBlock *B) const {
+ SmallVector<std::pair<const MachineBasicBlock *, PredKey>, 8> Deps;
+ for (const PredCtrl &PC : PredCtrls) {
+ if (MPDT->dominates(B, PC.Block))
+ continue; // B post-dominates the branch site -> not control-dependent
+ for (MachineBasicBlock *C : PC.Block->successors()) {
+ if (!MPDT->dominates(B, C))
+ continue;
+ bool ToLabel = (C == PC.Label);
+ // Edge to label is taken when cond == !negate; fall-through when
+ // ==negate.
+ bool Pol = ToLabel ? !PC.Negate : PC.Negate;
+ Deps.push_back({PC.Block, predKey(PC.Pred, Pol)});
+ }
+ }
+ return Deps;
+}
+
+// Transitive guard set of block B: every (pred,pol) that must hold for B to
+// run, following the control-dependence chain. A block inherits the guards of
+// each branch site it is control-dependent on. The Seen set makes the closure
+// terminate even with cyclic control dependence (loops).
+DenseSet<PredKey>
+PISAInsertLifetimeStart::transitiveCtrlSet(const MachineBasicBlock *B) const {
+ DenseSet<PredKey> S;
+ SmallVector<const MachineBasicBlock *, 8> Work{B};
+ DenseSet<const MachineBasicBlock *> Seen{B};
+ while (!Work.empty()) {
+ const MachineBasicBlock *X = Work.pop_back_val();
+ for (auto [Site, Key] : directCtrlDeps(X)) {
+ S.insert(Key);
+ if (Seen.insert(Site).second)
+ Work.push_back(Site);
+ }
+ }
+ return S;
+}
+
+// A real (non-undef, non-debug) full (subreg-0) def of R. A repack op that
+// fully defines R counts; what matters for soundness is ordering, not the
+// opcode.
+bool PISAInsertLifetimeStart::isRealFullDef(const MachineInstr &MI,
+ Register R) const {
+ if (MI.isImplicitDef() || MI.isDebugInstr())
+ return false;
+ for (const MachineOperand &MO : MI.operands())
+ if (MO.isReg() && MO.isDef() && MO.getReg() == R &&
+ MO.getSubReg() == PISA::NoSubRegister)
+ return true;
+ return false;
+}
+
+// True iff D is executed before U within an iteration: D's block dominates
+// U's block, or (same block) D is the earlier instruction. With a shared
+// guard (or on its own) this proves R holds a same-iteration value where U
+// observes it.
+bool PISAInsertLifetimeStart::defRunsBeforeUse(const MachineInstr &D,
+ const MachineInstr &U) const {
+ const MachineBasicBlock *DB = D.getParent(), *UB = U.getParent();
+ if (DB != UB)
+ return MDT->dominates(DB, UB);
+ for (const MachineInstr &MI : *DB) {
+ if (&MI == &D)
+ return true;
+ if (&MI == &U)
+ return false;
+ }
+ return false;
+}
+
+// True if block To is reachable from block From along intra-loop edges
+// without passing through L's header -- i.e. To runs after From in the
+// iteration (the header is the only re-entry point, so excluding it cuts
+// every back-edge of L). Used by the cover below to reject a def that,
+// although it shares the observation's guard, runs after the observation (so
+// the observation still sees the back-edge value).
+bool PISAInsertLifetimeStart::reachesWithinIteration(
+ const MachineBasicBlock *From, const MachineBasicBlock *To,
+ const MachineLoop *L) const {
+ const MachineBasicBlock *H = L->getHeader();
+ SmallVector<const MachineBasicBlock *, 8> Work;
+ DenseSet<const MachineBasicBlock *> Seen;
+ for (const MachineBasicBlock *S : From->successors())
+ if (L->contains(S) && S != H && Seen.insert(S).second)
+ Work.push_back(S);
+ while (!Work.empty()) {
+ const MachineBasicBlock *B = Work.pop_back_val();
+ if (B == To)
+ return true;
+ for (const MachineBasicBlock *S : B->successors())
+ if (L->contains(S) && S != H && Seen.insert(S).second)
+ Work.push_back(S);
+ }
+ return false;
+}
+
+// Per-use cover test. R's value reaches its real observations through
+// forwarding ops (copy/insert/extract/mov) that only repack it, so we follow
+// R forward through them and require that every real use U of a value derived
+// from R is covered: a real def D of R runs before U and D's guards are
+// implied by U's guards, so U always sees this iteration's value rather than
+// the back-edge value. D ordered before U is checked by dominance or, for a
+// shared guard, by D not running after U within the iteration
+// (reachesWithinIteration) -- without that ordering U could read the
+// back-edge value before D rewrites it. Guards are compared with the
+// transitive guard set, so a deep observation links back to the predicate
+// guarding the def even when intervening blocks test other predicates.
+//
+// A register with no real def (pass-through), or an observation no def
+// covers, is rejected. A use OUTSIDE the loop L is always rejected: a value
+// derived from R that is read after the loop is the carried value escaping.
+// This catches escapes through a different register than R itself, which the
+// isLiveOutOfLoop(R) gate -- keyed on R alone -- cannot see.
+bool PISAInsertLifetimeStart::hasTransitiveControlDependence(
+ Register R, const MachineLoop *L) const {
+ SmallVector<const MachineInstr *, 4> RDefs;
+ for (const MachineInstr &MI : MRI->def_instructions(R))
+ if (isRealFullDef(MI, R))
+ RDefs.push_back(&MI);
+ if (RDefs.empty())
+ return false; // pass-through: nothing redefines R in the loop
+
+ SmallVector<Register, 8> Work{R};
+ DenseSet<Register> Visited;
+ bool SawObservation = false;
+ while (!Work.empty()) {
+ Register V = Work.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue; // already processed
+ for (const MachineInstr &U : MRI->use_instructions(V)) {
+ if (U.isDebugInstr())
+ continue;
+ if (isForwardingOpcode(U, *TII)) {
+ // Repack: follow the produced virtual reg(s); it observes nothing.
+ for (const MachineOperand &MO : U.operands())
+ if (MO.isReg() && MO.isDef() && MO.getReg().isVirtual())
+ Work.push_back(MO.getReg());
+ continue;
+ }
+ // Real observation of a value derived from R.
+ SawObservation = true;
+ if (!L->contains(U.getParent()))
+ return false; // observed after the loop -> escaping back-edge value
+ DenseSet<PredKey> CU = transitiveCtrlSet(U.getParent());
+ bool Covered = false;
+ for (const MachineInstr *D : RDefs) {
+ // A def cannot cover its OWN read: a read-modify-write of R (e.g. an
+ // accumulator `R = iadd R, x`) reads the incoming/back-edge value
+ // before this instruction's def, so it must stay an uncovered
+ // observation.
+ if (D == &U)
+ continue;
+ if (defRunsBeforeUse(*D, U)) { // dominated -> fresh, predicate-free
+ Covered = true;
+ break;
+ }
+ // Guard-containment cover: D and U live in different blocks. Only
+ // sound if D is NOT after U within the iteration -- otherwise U reads
+ // the back-edge value before D rewrites it. (Same-block D-after-U was
+ // already settled false by defRunsBeforeUse, and
+ // reachesWithinIteration is block-granular, so skip it there.)
+ const MachineBasicBlock *DB = D->getParent(), *UB = U.getParent();
+ if (DB == UB || reachesWithinIteration(UB, DB, L))
+ continue; // D runs after U -> cannot cover this observation
+ DenseSet<PredKey> CD = transitiveCtrlSet(DB);
+ if (CD.empty())
+ continue; // unconditional D is covered only via the dominance
+ // branch
+ // Containment: every guard of D also guards U, so U running implies D
+ // ran the same iteration.
+ bool Implies = true;
+ for (PredKey K : CD) {
+ // predKey identifies a guard by (predicate vreg, polarity) only. That
+ // is a value identity ONLY when the predicate vreg has a single
+ // reaching def: this MIR is non-SSA (post-PHIElim / coalescing), so a
+ // predicate vreg may be redefined and hold different values at
+ // different branch sites. If D's guard predicate is multiply defined,
+ // a matching key in CU can come from a branch testing a DIFFERENT
+ // value, so "U ran => D ran" no longer follows. Refuse the cover.
+ Register P = Register(static_cast<unsigned>(K >> 1));
+ if (!MRI->hasOneDef(P)) {
+ Implies = false;
+ break;
+ }
+ if (!CU.contains(K)) {
+ Implies = false;
+ break;
+ }
+ }
+ if (Implies)
+ Covered = true;
+ if (Covered)
+ break;
+ }
+ if (!Covered)
+ return false; // observed outside R's def guard -> back-edge value
+ // live
+ }
+ }
+ // SawObservation stays false only for an unobservable register -> reject.
+ return SawObservation;
+}
+
+bool PISAInsertLifetimeStart::run(MachineFunction &MF, MachineLoopInfo &MLI) {
+ const unsigned NumV = MRI->getNumVirtRegs();
+ if (NumV == 0 || MLI.empty())
+ return false;
+
+ // Candidate signal: a vreg with an IMPLICIT_DEF def (undef-on-entry seed).
+ BitVector HasImplicitDef(NumV);
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (MI.isImplicitDef()) {
+ Register R = MI.getOperand(0).getReg();
+ if (R.isVirtual())
+ HasImplicitDef.set(R.virtRegIndex());
+ }
+ if (HasImplicitDef.none())
+ return false;
+
+ // Collect every predicated branch in the function. Control-dependence is a
+ // whole-CFG property, so this is gathered once and reused for all candidates.
+ PredCtrls.clear();
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (MI.getOpcode() == PISA::predgoto) {
+ Register P = MI.getOperand(1).getReg();
+ if (!P.isVirtual())
+ continue;
+ PredCtrls.push_back({&MBB, MI.getOperand(2).getMBB(), P,
+ MI.getOperand(0).getImm() != 0});
+ }
+ if (PredCtrls.empty())
+ return false;
+
+ // Visit loops innermost-first, dedup globally.
+ SmallVector<MachineLoop *, 8> Loops;
+ for (MachineLoop *TopL : MLI) {
+ SmallVector<MachineLoop *, 8> WL{TopL};
+ while (!WL.empty()) {
+ MachineLoop *L = WL.pop_back_val();
+ Loops.push_back(L);
+ WL.append(L->begin(), L->end());
+ }
+ }
+ llvm::stable_sort(Loops, [](const MachineLoop *A, const MachineLoop *B) {
+ return A->getLoopDepth() > B->getLoopDepth();
+ });
+
+ LiveMap LiveIn, LiveOut;
+ computeBlockLiveness(MF, LiveIn, LiveOut);
+
+ BitVector Marked(NumV);
+ bool Changed = false;
+
+ for (MachineLoop *L : Loops) {
+ MachineBasicBlock *H = L->getHeader();
+ MachineBasicBlock::iterator InsertPt = H->getFirstNonPHI();
+ for (unsigned I : HasImplicitDef.set_bits()) {
+ if (Marked.test(I))
+ continue; // already processed
+ Register R = Register::index2VirtReg(I);
+ if (!isBackEdgeLive(R, L, LiveIn, LiveOut))
+ continue;
+ if (isLiveOutOfLoop(R, L, LiveIn)) {
+ // R escapes the loop: its exit value is observed after the loop and is
+ // (or depends on) the carried value, so "dead at the header" is
+ // unsound.
+ LLVM_DEBUG(dbgs() << "[" DEBUG_TYPE "] skip " << printReg(R, nullptr)
+ << ": live-out of the loop (escapes to a post-loop "
+ "use)\n");
+ continue;
+ }
+ if (!implicitDefSeedReachesHeader(*MRI, R, L, *MDT)) {
+ // Either the header marker would be a use not dominated by any def (R's
+ // only seed/def is inside the loop body), or a REAL def reaches the
+ // header from outside the loop -- i.e. a live value enters the loop and
+ // "dead at header" would be a miscompile.
+ LLVM_DEBUG(dbgs() << "[" DEBUG_TYPE "] skip " << printReg(R, nullptr)
+ << ": IMPLICIT_DEF seed does not reach the loop "
+ "header (real def reaches it, or no dominating "
+ "seed / no unique preheader)\n");
+ continue;
+ }
+ if (!hasTransitiveControlDependence(R, L)) {
+ LLVM_DEBUG(dbgs() << "[" DEBUG_TYPE "] skip " << printReg(R, nullptr)
+ << ": an observation is not covered by a guarded "
+ "preceding def (transitive)\n");
+ continue;
+ }
+ unsigned Opc = pickLifetimeStartOpcode(*MRI->getTargetRegisterInfo(),
+ MRI->getRegClass(R));
+ if (!Opc) {
+ LLVM_DEBUG(dbgs() << "[" DEBUG_TYPE "] skip " << printReg(R, nullptr)
+ << ": no lifetime.start variant for its reg class\n");
+ continue;
+ }
+ BuildMI(*H, InsertPt, DebugLoc(), TII->get(Opc)).addDef(R);
+ Marked.set(I);
+ ++NumLifetimeMarkers;
+ Changed = true;
+ LLVM_DEBUG(dbgs() << "[" DEBUG_TYPE "] mark " << printReg(R, nullptr)
+ << " at " << printMBBReference(*H) << " (loop depth "
+ << L->getLoopDepth() << ")\n");
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << "[" DEBUG_TYPE "] " << MF.getName() << ": "
+ << Marked.count() << " marker(s) inserted\n");
+ return Changed;
+}
+
+bool PISAInsertLifetimeStart::runOnMachineFunction(MachineFunction &MF) {
+ if (!LifetimeStartOpt)
+ return false; // default: insert nothing
+
+ MRI = &MF.getRegInfo();
+ TII = MF.getSubtarget().getInstrInfo();
+ MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+ MPDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
+ MachineLoopInfo &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+ return run(MF, MLI);
+}
+
+char PISAInsertLifetimeStart::ID = 0;
+INITIALIZE_PASS_BEGIN(PISAInsertLifetimeStart, DEBUG_TYPE,
+ "PISA insert lifetime.start", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(PISAInsertLifetimeStart, DEBUG_TYPE,
+ "PISA insert lifetime.start", false, false)
+
+namespace llvm {
+FunctionPass *createPISAInsertLifetimeStart() {
+ return new PISAInsertLifetimeStart();
+}
+} // namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp b/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp
new file mode 100644
index 0000000000000..03b1d6141001a
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp
@@ -0,0 +1,294 @@
+//===-- PISAKernelByValArgsLowering.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISAKernelByValArgsLowering.h"
+
+#include "PISA.h"
+#include "PISADefines.h"
+
+#include <llvm/IR/BasicBlock.h>
+#include <llvm/IR/CallingConv.h>
+#include <llvm/IR/Function.h>
+#include <llvm/IR/IRBuilder.h>
+#include <llvm/IR/IntrinsicInst.h>
+#include <llvm/IR/Module.h>
+#include <llvm/Support/Debug.h>
+#include <llvm/Support/PISAAddrSpace.h>
+
+#define DEBUG_TYPE "pisa-kernel-byval-args-lowering"
+#define DEBUG_NAME "PISA kernel byval args lowering (Legacy)"
+
+using namespace llvm;
+using namespace llvm::PISA;
+using namespace llvm::PISAAS;
+
+namespace {
+class PISAKernelByValArgsLowering {
+public:
+ explicit PISAKernelByValArgsLowering(Function &F) : F(F), Ctx(F.getContext()) {
+ assert(F.getCallingConv() == CallingConv::PISA_KERNEL &&
+ "Expected PISA_KERNEL calling convention");
+ assert(F.getReturnType()->isVoidTy() && "Expected void return type");
+ }
+
+ bool run();
+
+private:
+ static constexpr auto ByRefAddressSpace =
+ static_cast<unsigned>(AddressSpace::CONSTANT);
+
+ AttributeList getNewAttributes(const AttributeList &Attrs) const;
+ void transferArgumentUses(Function &NewF);
+
+ Function &F;
+ LLVMContext &Ctx;
+};
+
+bool PISAKernelByValArgsLowering::run() {
+ if (none_of(F.args(), [](auto &Arg) { return Arg.hasByValAttr(); }))
+ return false;
+
+ LLVM_DEBUG(dbgs() << "Lowering kernel aggregate arguments for " << F.getName()
+ << "\n");
+
+ AttributeList NewAttrs = getNewAttributes(F.getAttributes());
+
+ SmallVector<Type *> NewArgTypes;
+ transform(F.args(), std::back_inserter(NewArgTypes),
+ [this](auto &Arg) -> Type * {
+ if (Arg.hasByValAttr())
+ return PointerType::get(Ctx, ByRefAddressSpace);
+ return Arg.getType();
+ });
+
+ auto *NewFTy = FunctionType::get(Type::getVoidTy(Ctx), NewArgTypes, false);
+ auto *NewF = Function::Create(NewFTy, F.getLinkage());
+ NewF->takeName(&F);
+ NewF->setAttributes(NewAttrs);
+ NewF->setCallingConv(CallingConv::PISA_KERNEL);
+
+ F.getParent()->getFunctionList().insert(F.getIterator(), NewF);
+
+ // Transfer debug info to the new function
+ auto *DISubprog = F.getSubprogram();
+ NewF->setSubprogram(DISubprog);
+ F.setSubprogram(nullptr);
+
+ // Splice the body of the old function into the new function.
+ NewF->splice(NewF->begin(), &F);
+
+ transferArgumentUses(*NewF);
+
+ F.replaceAllUsesWith(NewF);
+
+ NewF->copyMetadata(&F, 0);
+ return true;
+}
+
+AttributeList
+PISAKernelByValArgsLowering::getNewAttributes(const AttributeList &Attrs) const {
+ AttributeList NewAttrs;
+
+ // Copy function attributes.
+ if (auto FunctionAttrs = Attrs.getFnAttrs(); FunctionAttrs.hasAttributes()) {
+ AttrBuilder AB(Ctx, FunctionAttrs);
+ NewAttrs = NewAttrs.addFnAttributes(Ctx, AB);
+ }
+
+ for (const auto &Arg : F.args()) {
+ const auto Index = Arg.getArgNo();
+ auto ArgAttrs = Attrs.getParamAttrs(Index);
+ AttrBuilder AB(Ctx, ArgAttrs);
+
+ if (ArgAttrs.hasAttribute(Attribute::ByVal)) {
+ // Replace byval with byref
+ assert(cast<PointerType>(Arg.getType())->getAddressSpace() ==
+ static_cast<unsigned>(AddressSpace::PRIVATE) &&
+ "Expected private address space");
+
+ auto *ByValTy = ArgAttrs.getByValType();
+
+ AB.removeAttribute(Attribute::ByVal).addByRefAttr(ByValTy);
+ NewAttrs = NewAttrs.addParamAttributes(Ctx, Index, AB);
+ } else {
+ // Copy the argument as is.
+ NewAttrs = NewAttrs.addParamAttributes(Ctx, Index, AB);
+ }
+ }
+
+ return NewAttrs;
+}
+
+void PISAKernelByValArgsLowering::transferArgumentUses(Function &NewF) {
+ IRBuilder<> Builder(Ctx);
+
+ for (auto [OldArg, NewArg] : zip_equal(F.args(), NewF.args())) {
+ NewArg.takeName(&OldArg);
+
+ if (!OldArg.hasByValAttr()) {
+ OldArg.replaceAllUsesWith(&NewArg);
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << "Convert byval argument <" << OldArg << "> to byref <"
+ << NewArg << ">\n");
+
+ SmallVector<Value *, 8> Stack;
+ DenseMap<Value *, Value *> Map;
+ SmallVector<Instruction *, 8> ToDelete;
+ Stack.push_back(&OldArg);
+ Map[&OldArg] = &NewArg;
+
+ // Traverse all users of the argument and create GEPs and loads with the
+ // new address space. A GEP/memop chain can later be combined into a single
+ // ld.param PISA instruction.
+ while (!Stack.empty()) {
+ auto *V = Stack.back();
+ Stack.pop_back();
+ for (auto *U : V->users()) {
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
+ assert(GEP->getPointerOperand() == V);
+ Stack.push_back(GEP);
+ Builder.SetInsertPoint(GEP->getIterator());
+ auto *NewGEP = Builder.CreateGEP(
+ GEP->getSourceElementType(), Map[V],
+ SmallVector<Value *, 4>(GEP->indices()),
+ GEP->getName() + ".byref", GEP->getNoWrapFlags());
+ LLVM_DEBUG(dbgs() << "Created getelementptr: " << *NewGEP << "\n");
+ Map[GEP] = NewGEP;
+ } else if (auto *Load = dyn_cast<LoadInst>(U)) {
+ Builder.SetInsertPoint(Load->getIterator());
+ auto *NewLoad = Builder.CreateAlignedLoad(
+ Load->getType(), Map[Load->getPointerOperand()], Load->getAlign(),
+ Load->isVolatile(), Load->getName() + ".byref");
+ LLVM_DEBUG(dbgs() << "Created load: " << *NewLoad << "\n");
+ Load->replaceAllUsesWith(NewLoad);
+ ToDelete.push_back(Load);
+ } else if (auto *MemCpy = dyn_cast<MemCpyInst>(U)) {
+ if (MemCpy && MemCpy->getSource() == V) {
+ Builder.SetInsertPoint(MemCpy);
+ auto *NewMemCpy = Builder.CreateMemCpy(
+ MemCpy->getDest(), MemCpy->getDestAlign(),
+ Map[MemCpy->getSource()], MemCpy->getSourceAlign(),
+ MemCpy->getLength(), MemCpy->isVolatile());
+ LLVM_DEBUG(dbgs() << "Created memcpy: " << *NewMemCpy << "\n");
+ (void)NewMemCpy;
+ ToDelete.push_back(MemCpy);
+ } else {
+ LLVM_DEBUG(dbgs() << "Only memcpy using byval argument as a source "
+ "can be combined into a ld.param: "
+ << *MemCpy << "\n");
+ }
+ } else {
+ LLVM_DEBUG(
+ dbgs()
+ << "Byval argument's user can't be combined into a ld.param: "
+ << *U << "\n");
+ }
+ }
+ }
+
+ // Perform cleanup: erase replaced memory operations and the chain of GEPs
+ // between them and the argument.
+ for (auto *Inst : ToDelete) {
+ Value *Ptr = nullptr;
+ if (auto *Load = dyn_cast<LoadInst>(Inst)) {
+ Ptr = Load->getPointerOperand();
+ } else if (auto *MemCpy = dyn_cast<MemCpyInst>(Inst)) {
+ Ptr = MemCpy->getSource();
+ }
+ assert(Ptr);
+ Inst->eraseFromParent();
+ while (auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {
+ Ptr = GEP->getPointerOperand();
+ if (GEP->getNumUses() > 0)
+ break;
+ GEP->eraseFromParent();
+ }
+ }
+
+ // If all of the argument's users were processed earlier, nothing more needs
+ // to be done here.
+ if (OldArg.getNumUses() == 0)
+ continue;
+
+ // Otherwise, create a copy in the private address space and replace any
+ // remaining users.
+ Builder.SetInsertPoint(NewF.getEntryBlock().getFirstInsertionPt());
+ unsigned AS = static_cast<unsigned>(AddressSpace::PRIVATE);
+ auto *ByValTy = OldArg.getAttribute(Attribute::ByVal).getValueAsType();
+ auto *Alloca = Builder.CreateAlloca(ByValTy, AS, nullptr,
+ OldArg.getName() + ".private");
+ LLVM_DEBUG(dbgs() << "Created alloca: " << *Alloca << "\n");
+ const auto &DL = F.getParent()->getDataLayout();
+ auto *MemCpy = Builder.CreateMemCpy(Alloca, Alloca->getAlign(), &NewArg,
+ NewArg.getPointerAlignment(DL),
+ DL.getTypeStoreSize(ByValTy));
+ LLVM_DEBUG(dbgs() << "Created memcpy: " << *MemCpy << "\n");
+ (void)MemCpy;
+ OldArg.replaceAllUsesWith(Alloca);
+ }
+}
+
+class PISAKernelByValArgsLoweringLegacy : public ModulePass {
+public:
+ static char ID;
+
+ PISAKernelByValArgsLoweringLegacy() : ModulePass(ID) {}
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ bool runOnModule(Module &M) override {
+ bool Changed = false;
+ for (auto FI = M.begin(), FE = M.end(); FI != FE;) {
+ Function &F = *FI++;
+ if (F.isDeclaration() || F.getCallingConv() != CallingConv::PISA_KERNEL)
+ continue;
+ PISAKernelByValArgsLowering KBVAL(F);
+ if (KBVAL.run()) {
+ F.eraseFromParent();
+ Changed = true;
+ }
+ }
+ return Changed;
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ ModulePass::getAnalysisUsage(AU);
+ }
+};
+} // namespace
+
+char PISAKernelByValArgsLoweringLegacy::ID = 0;
+INITIALIZE_PASS(PISAKernelByValArgsLoweringLegacy, DEBUG_TYPE, DEBUG_NAME,
+ false, false)
+
+ModulePass *llvm::createPISAKernelByValArgsLoweringLegacyPass() {
+ return new PISAKernelByValArgsLoweringLegacy();
+}
+
+PreservedAnalyses KernelByValArgsLoweringPass::run(Module &M,
+ ModuleAnalysisManager &) {
+ SmallVector<Function *> ToErase;
+
+ for (auto &F : M) {
+ if (F.isDeclaration() || F.getCallingConv() != CallingConv::PISA_KERNEL)
+ continue;
+
+ if (PISAKernelByValArgsLowering LKA(F); LKA.run())
+ ToErase.push_back(&F);
+ }
+
+ if (ToErase.empty())
+ return PreservedAnalyses::all();
+
+ for (auto *F : ToErase)
+ F->eraseFromParent();
+
+ return PreservedAnalyses::none();
+}
diff --git a/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.h b/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.h
new file mode 100644
index 0000000000000..80854888ae97e
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.h
@@ -0,0 +1,40 @@
+//===-- PISAKernelByValArgsLowering.h -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass lowers byval arguments. The byval semantics cannot work with the
+// PISA calling convention, so the pass replaces byval with byref as follows:
+//
+// 1. The pass creates a new function with the same signature as the original
+// function, but with byref arguments instead of byval arguments. The new
+// byref arguments are pointers with constant address space.
+// 2. The pass creates alloca instructions for each byval argument of the
+// original function and replaces the byval argument uses with the alloca.
+// 3. The pass creates a memcpy intrinsic to copy the new byref argument to the
+// alloca.
+// 4. The pass replaces old function uses with the new function and removes the
+// old function.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_PISA_PISAKERNELBYVALARGSLOWERING_H
+#define LLVM_LIB_TARGET_PISA_PISAKERNELBYVALARGSLOWERING_H
+
+#include <llvm/IR/PassManager.h>
+
+namespace llvm {
+namespace PISA {
+class KernelByValArgsLoweringPass
+ : public OptionalPassInfoMixin<KernelByValArgsLoweringPass> {
+public:
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+};
+
+} // namespace PISA
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_PISA_PISAKERNELBYVALARGSLOWERING_H
diff --git a/llvm/lib/Target/PISA/PISALayout.cpp b/llvm/lib/Target/PISA/PISALayout.cpp
new file mode 100644
index 0000000000000..ab43298ae8d5c
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISALayout.cpp
@@ -0,0 +1,581 @@
+//===-- PISALayout.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define GET_PISAAtomicSubOpcode_DECL
+#include "PISAGenSearchableTables.inc"
+
+#include "PISA.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+#define DEBUG_TYPE "pisa-layout"
+#define DEBUG_NAME "PISA layout"
+
+using namespace llvm;
+
+namespace {
+/// This is a temporary solution to the issues with divergent barriers.
+/// Ultimately it's going to be replaced with MachineBlockPlacement, which
+/// currently is causing issues in the PISA backend and cannot be
+/// enabled yet.
+///
+/// @brief sort basic blocks into topological order
+/// Arbitrary reverse postorder is not sufficient.
+/// Whenever it is possible, we want to layout blocks in such way
+/// that the we can recognize the control-flow structures.
+class PISALayout : public FunctionPass {
+public:
+ static char ID;
+ PISALayout() : FunctionPass(ID) {}
+
+ virtual void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ /// @brief Provides name of pass
+ virtual StringRef getPassName() const override { return DEBUG_NAME; }
+
+ virtual bool runOnFunction(Function &F) override;
+
+private:
+ BasicBlock *getLastReturnBlock(Function &F);
+ void layoutBlocks(Function &F, LoopInfo &LI);
+ void layoutBlocks(Function &F);
+ BasicBlock *selectSucc(BasicBlock *CurrBlk, bool SelectNoInstBlk,
+ const LoopInfo &LI,
+ const SmallSet<BasicBlock *, 8> &VisitSet);
+ bool isAtomicWrite(Instruction *Inst, bool OnlyLocalMem);
+ bool isAtomicRead(Instruction *Inst, bool OnlyLocalMem);
+ Value *getMemoryOperand(Instruction *Inst, bool OnlyLocalMem);
+ bool isReturnBlock(BasicBlock *BB);
+ bool tryMovingWrite(Instruction *Write, Loop *Loop, LoopInfo &LI);
+ void moveAtomicWrites2Loop(Function &F, LoopInfo &LI, bool OnlyLocalMem);
+
+ PostDominatorTree *PDT = nullptr;
+ DominatorTree *DT = nullptr;
+};
+} // namespace
+
+char PISALayout::ID = 0;
+INITIALIZE_PASS_BEGIN(PISALayout, DEBUG_TYPE, DEBUG_NAME, false, false)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
+INITIALIZE_PASS_END(PISALayout, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+constexpr unsigned BreakBlockSizeLimit = 3;
+
+static void pushSucc(BasicBlock *BB, std::function<bool(BasicBlock *)> Cond,
+ SmallVector<BasicBlock *> &VisitVec,
+ SmallSet<BasicBlock *, 8> &VisitSet) {
+ for (succ_iterator IT = succ_begin(BB), End = succ_end(BB); IT != End; ++IT) {
+ BasicBlock *Succ = *IT;
+ if (!VisitSet.count(Succ) && Cond(Succ)) {
+ VisitVec.push_back(Succ);
+ VisitSet.insert(Succ);
+ break;
+ }
+ }
+}
+
+inline static auto sizeWithoutDebug(const BasicBlock *BB) { return BB->size(); }
+
+void PISALayout::getAnalysisUsage(AnalysisUsage &AU) const {
+ // Doesn't change the IR at all, it just move the blocks so no changes in the
+ // IR
+ AU.setPreservesAll();
+ AU.addRequired<LoopInfoWrapperPass>();
+ AU.addRequired<PostDominatorTreeWrapperPass>();
+ AU.addRequired<DominatorTreeWrapperPass>();
+}
+
+bool PISALayout::runOnFunction(Function &Func) {
+ PDT = &getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
+ DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+ LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ if (LI.empty())
+ layoutBlocks(Func);
+ else
+ layoutBlocks(Func, LI);
+
+ return true;
+}
+
+// check if the instruction is atomic write (xchg or cmpxchng)
+bool PISALayout::isAtomicWrite(Instruction *Inst, bool OnlyLocalMem) {
+ auto MemCond = [Inst, OnlyLocalMem]() {
+ Value *Ptr = Inst->getOperand(0);
+ bool IsLocalMem = Ptr->getType()->getPointerAddressSpace() ==
+ static_cast<unsigned>(PISAAS::AddressSpace::SHARED);
+ return (!OnlyLocalMem || IsLocalMem);
+ };
+
+ if (isa<AtomicCmpXchgInst>(Inst) && MemCond())
+ return true;
+
+ if (auto *AtomicRMW = dyn_cast<AtomicRMWInst>(Inst))
+ return AtomicRMW->getOperation() == AtomicRMWInst::Xchg && MemCond();
+
+ if (auto *CI = dyn_cast<CallInst>(Inst)) {
+ Function *F = CI->getCalledFunction();
+ if (!F)
+ return false;
+
+ Intrinsic::ID IntrID = F->getIntrinsicID();
+ if (IntrID == Intrinsic::pisa_cas_fatom && MemCond())
+ return true;
+ }
+
+ return false;
+}
+
+// check if the instruction is atomic read (ATOMIC_OR with src == 0)
+bool PISALayout::isAtomicRead(Instruction *Inst, bool OnlyLocalMem) {
+ auto LocalMemCond = [Inst, OnlyLocalMem]() {
+ return !OnlyLocalMem ||
+ Inst->getOperand(0)->getType()->getPointerAddressSpace() ==
+ static_cast<unsigned>(PISAAS::AddressSpace::SHARED);
+ };
+
+ auto *AtomicRMW = dyn_cast<AtomicRMWInst>(Inst);
+ if (AtomicRMW && AtomicRMW->getOperation() == AtomicRMWInst::Or &&
+ LocalMemCond()) {
+ ConstantInt *Src = dyn_cast<ConstantInt>(AtomicRMW->getValOperand());
+ return Src && Src->getZExtValue() == 0;
+ }
+
+ return false;
+}
+
+// get memory operand for atomic read or write
+Value *PISALayout::getMemoryOperand(Instruction *Inst, bool OnlyLocalMem) {
+ if (!isAtomicRead(Inst, OnlyLocalMem) && !isAtomicWrite(Inst, OnlyLocalMem))
+ return nullptr;
+
+ Value *DstAddr = Inst->getOperand(0);
+ if (auto *PTI = dyn_cast<PtrToIntInst>(DstAddr))
+ return PTI->getPointerOperand();
+
+ return DstAddr;
+}
+
+bool PISALayout::isReturnBlock(BasicBlock *BB) {
+ return isa<ReturnInst>(BB->getTerminator());
+}
+
+// Try moving atomic write (or its loop) into the given destination loop
+// If there are no direct predecessor in the needed loop,
+// Try to move it together with a chain of predecessors. New BB is added in
+// chain if it is either single predecessor or it is a previous node in current
+// layout.
+//
+bool PISALayout::tryMovingWrite(Instruction *Write, Loop *LP, LoopInfo &LI) {
+ SmallVector<BasicBlock *> BlocksToMove;
+
+ if (Loop *WritingLoop = LI.getLoopFor(Write->getParent())) {
+ auto Blocks = WritingLoop->getBlocks();
+ for (auto &BB : Blocks) {
+ if (isReturnBlock(BB))
+ return false;
+ BlocksToMove.push_back(BB);
+ }
+ } else {
+ if (!isReturnBlock(Write->getParent()))
+ BlocksToMove.push_back(Write->getParent());
+ else
+ return false;
+ }
+
+ // Loop exits when:
+ // - all BasicBlocks with the Write or its loop has been moved
+ // - processed BasicBlock contains a return or has more than one predecessor
+ while (true) {
+ BasicBlock *Blk = BlocksToMove.back();
+
+ // If one (and only one) of the predecessors is in the needed loop, move
+ // blocks after it
+ BasicBlock *InsertPoint = nullptr;
+ int PredsInLoop = 0;
+ for (pred_iterator PredIter = pred_begin(Blk), PredEnd = pred_end(Blk);
+ PredIter != PredEnd; ++PredIter) {
+ BasicBlock *Pred = *PredIter;
+ if (LP->contains(Pred)) {
+ PredsInLoop++;
+ InsertPoint = Pred;
+ }
+ }
+ if (PredsInLoop == 1) {
+ for (auto *BB : BlocksToMove)
+ BB->moveAfter(InsertPoint);
+ return true;
+ }
+ if (PredsInLoop > 1)
+ return false;
+
+ // Add prev node if it is the predecessor of the block
+ bool PredPushed = false;
+ for (pred_iterator PredIter = pred_begin(Blk), PredEnd = pred_end(Blk);
+ PredIter != PredEnd; ++PredIter) {
+ BasicBlock *Pred = *PredIter;
+
+ if ((Pred == Blk->getPrevNode()) && !isReturnBlock(Pred)) {
+ BlocksToMove.push_back(Pred);
+ PredPushed = true;
+ break;
+ }
+ }
+
+ if (PredPushed)
+ continue;
+
+ // Add predecessor if it is single
+ BasicBlock *Pred = Blk->getSinglePredecessor();
+ if (Pred && !isReturnBlock(Pred)) {
+ BlocksToMove.push_back(Pred);
+ PredPushed = true;
+ } else
+ // Don't move the blocks and return
+ return false;
+ }
+}
+
+// Place basic blocks with atomic write (or the whole loop with the
+// atomic write) into the other loop if there is an atomic read
+// from the same memory, which dominates the write.
+//
+// It benefits cases like:
+//
+// Loop:
+// Load A
+// if (!pred(Load A))
+// {
+// break;
+// }
+// if (success(do_work())
+// {
+// Store A;
+// break;
+// }
+// Br Loop
+//
+// If the Store is placed after the back edge of the loop
+// there will be goto instruction disabling channels based on some
+// "success(do_work())" condition placed before the back edge in SIMD control
+// flow, and the store will be delayed until the whole loop is finished. It
+// makes "if (!pred(Load A))" checking useless and doesn't allow to perform
+// early break based on the condition.
+//
+void PISALayout::moveAtomicWrites2Loop(Function &F, LoopInfo &LI,
+ bool OnlyLocalMem) {
+ SmallVector<Instruction *> Writes;
+ SmallVector<Instruction *> Reads;
+ for (auto &I : instructions(F)) {
+ if (isAtomicWrite(&I, OnlyLocalMem))
+ Writes.push_back(&I);
+ else if (isAtomicRead(&I, OnlyLocalMem))
+ Reads.push_back(&I);
+ }
+
+ // write: LoopWhereToMove mapping
+ MapVector<Instruction *, Loop *> WritesToMove;
+
+ for (auto *Read : Reads)
+ for (auto *Write : Writes)
+ if (getMemoryOperand(Read, OnlyLocalMem) ==
+ getMemoryOperand(Write, OnlyLocalMem)) {
+ Loop *ReadLoop = LI.getLoopFor(Read->getParent());
+ Loop *WriteLoop = LI.getLoopFor(Write->getParent());
+ if (ReadLoop && (ReadLoop != WriteLoop) &&
+ ((DT->dominates(Read, Write))))
+ WritesToMove[Write] = LI.getLoopFor(Read->getParent());
+ }
+
+ for (const auto &Pair : WritesToMove) {
+ Instruction *Write = Pair.first;
+ Loop *Loop = Pair.second;
+
+ tryMovingWrite(Write, Loop, LI);
+ }
+}
+
+static bool hasThreadGroupBarrierInBlock(BasicBlock *BB) {
+ Module *M = BB->getParent()->getParent();
+ for (Function &F : *M) {
+ if (!F.isDeclaration() || !F.isIntrinsic())
+ continue;
+
+ Intrinsic::ID IntrID = F.getIntrinsicID();
+ if (IntrID == Intrinsic::pisa_workgroup_barrier)
+ for (auto *U : F.users()) {
+ auto *Inst = dyn_cast<Instruction>(U);
+ if (Inst && Inst->getParent() == BB)
+ return true;
+ }
+ }
+ return false;
+}
+
+BasicBlock *PISALayout::getLastReturnBlock(Function &F) {
+ // If Func has any return BB, return the last return BB (may have multiple);
+ // otherwise, return the last BB that has no succ;
+ // or nullptr if every BB has Succ (infinite looping)
+ BasicBlock *NoRetAndNoSucc = nullptr; // for func that never returns
+ for (auto RI = std::make_reverse_iterator(F.end()),
+ RE = std::make_reverse_iterator(F.begin());
+ RI != RE; ++RI) {
+ BasicBlock *BB = &*RI;
+ if (succ_begin(BB) == succ_end(BB)) {
+ if (isa_and_nonnull<ReturnInst>(BB->getTerminator()))
+ return BB;
+ if (!NoRetAndNoSucc)
+ NoRetAndNoSucc = BB;
+ }
+ }
+ // Function does not have a return block
+ return NoRetAndNoSucc;
+}
+
+//
+// selectSucc: select a succ with condition SelectNoInstBlk and return it.
+//
+// This is used to select one if there are two Successors with condition
+// SelectNoInstBlk, rather than take the first one in the succ list.
+//
+// Condition SelectNoInstBlk: If SelectNoInstBlk is true, select an empty
+// block, if it is false, select non-empty block.
+//
+BasicBlock *PISALayout::selectSucc(BasicBlock *CurrBlk, bool SelectNoInstBlk,
+ const LoopInfo &LI,
+ const SmallSet<BasicBlock *, 8> &VisitSet) {
+ SmallVector<BasicBlock *, 4> Succs;
+ for (succ_iterator SI = succ_begin(CurrBlk), SE = succ_end(CurrBlk); SI != SE;
+ ++SI) {
+ BasicBlock *Succ = *SI;
+ auto Size = sizeWithoutDebug(Succ);
+ if (VisitSet.count(Succ) == 0 &&
+ ((SelectNoInstBlk && Size <= 1) || (!SelectNoInstBlk && Size > 1)))
+ Succs.push_back(Succ);
+ }
+
+ // Right now, only handle the case of two empty blocks.
+ // If it has no two empty blocks, just take the first
+ // one and return it.
+ if (Succs.size() != 2 || !SelectNoInstBlk)
+ return Succs.empty() ? nullptr : Succs[0];
+
+ // For two empty blocks, the case we want to handle
+ // is the following:
+ //
+ // (B0 = CurrBlk)
+ // B0 : if (c) goto THEN (else goto ELSE)
+ // ELSE : goto B2
+ // B1 : ....
+ // B2 : ....
+ // ......
+ // Bn :
+ // (ELSE, B1, B2, ..., Bn) has END as single exit
+ // THEN: goto END:
+ // END :
+ // PHI...
+ //
+ // where ELSE and THEN are empty BBs, and END has phi in it.
+ // In this case, THEN and ELSE might have phi moves as the result
+ // DeSSA when emitting visa. For example, suppose d0 = s0 will
+ // be emitted in THEN. If s0 is dead after THEN, it would be good
+ // to lay out THEN right after B0 as the live-range of s0 will not
+ // be overlapped with ones in ELSE. (If s0 is live out of THEN,
+ // moving THEN right after B0 or right before END does not matter
+ // as far as liveness is concerned.). To lay out THEN first, this
+ // function will select ELSE to return (as the algo does layout
+ // backward).
+ //
+ // For simplicity, assume those BBs are not inside loops. It could
+ // be applied to Loop later when appropriate testing is done.
+ BasicBlock *Suc0 = Succs[0], *Suc1 = Succs[1];
+ BasicBlock *SS0 = Suc0->getSingleSuccessor();
+
+ if (SS0 && (SS0 != Suc1) && isa<PHINode>(&*SS0->begin()) &&
+ !LI.getLoopFor(Suc0) && PDT->dominates(SS0, Suc1))
+ return Suc1;
+
+ return Suc0;
+}
+
+void PISALayout::layoutBlocks(Function &F, LoopInfo &LI) {
+ SmallVector<BasicBlock *> VisitVec;
+ SmallSet<BasicBlock *, 8> VisitSet;
+ // Insertion Position per loop header
+ MapVector<BasicBlock *, BasicBlock *> InsPos;
+
+ BasicBlock *Entry = &(F.getEntryBlock());
+ VisitVec.push_back(Entry);
+ VisitSet.insert(Entry);
+ InsPos[Entry] = Entry;
+
+ // Push a return block to make sure the last BB is the return block.
+ if (BasicBlock *LastReturnBlock = getLastReturnBlock(F)) {
+ if (LastReturnBlock != Entry) {
+ VisitVec.push_back(LastReturnBlock);
+ VisitSet.insert(LastReturnBlock);
+ }
+ }
+
+ while (!VisitVec.empty()) {
+ BasicBlock *BB = VisitVec.back();
+ Loop *CurLoop = LI.getLoopFor(BB);
+ if (CurLoop) {
+ auto *HD = CurLoop->getHeader();
+ if (BB == HD && InsPos.find(HD) == InsPos.end())
+ InsPos[BB] = BB;
+ }
+
+ // push: time for DFS visit
+ auto PushHasInstCond = [](BasicBlock *Succ) -> bool {
+ return sizeWithoutDebug(Succ) > 1;
+ };
+ pushSucc(BB, PushHasInstCond, VisitVec, VisitSet);
+ if (BB != VisitVec.back())
+ continue;
+ // push: time for DFS visit
+ if (BasicBlock *ABlk = selectSucc(BB, true, LI, VisitSet)) {
+ VisitVec.push_back(ABlk);
+ VisitSet.insert(ABlk);
+ continue;
+ }
+
+ // pop: time to move the block to the right location
+ if (BB == VisitVec.back()) {
+ VisitVec.pop_back();
+ if (CurLoop) {
+ auto *HD = CurLoop->getHeader();
+ if (BB != HD) {
+ // move the block to the beginning of the loop
+ auto *Insp = InsPos[HD];
+ assert(Insp);
+ if (BB != Insp) {
+ BB->moveBefore(Insp);
+ InsPos[HD] = BB;
+ }
+ } else {
+ // move the entire loop to the beginning of
+ // the parent loop
+ auto *LoopStart = InsPos[HD];
+ assert(LoopStart);
+ auto *PaLoop = CurLoop->getParentLoop();
+ auto *PaHd = PaLoop ? PaLoop->getHeader() : Entry;
+ auto *Insp = InsPos[PaHd];
+ if (LoopStart == HD)
+ // single-block loop
+ HD->moveBefore(Insp);
+ else {
+ // loop-header is not moved yet, so should be at the end
+ // use splice
+ F.splice(Insp->getIterator(), &F, LoopStart->getIterator(),
+ HD->getIterator());
+ HD->moveBefore(LoopStart);
+ }
+ InsPos[PaHd] = HD;
+ }
+ } else {
+ auto *Insp = InsPos[Entry];
+ if (BB != Insp) {
+ BB->moveBefore(Insp);
+ InsPos[Entry] = BB;
+ }
+ }
+ }
+ }
+
+ moveAtomicWrites2Loop(F, LI, false);
+
+ // if function has a single exit, then the last block must be an exit
+ // fix the loop-exit pattern, put break-blocks into the loop
+ for (BasicBlock &BB : F) {
+ Loop *CurLoop = LI.getLoopFor(&BB);
+ bool AllPredLoopExit = true;
+ unsigned NumPreds = 0;
+ SmallPtrSet<BasicBlock *, 4> PredSet;
+ for (pred_iterator PredIter = pred_begin(&BB), PredEnd = pred_end(&BB);
+ PredIter != PredEnd; ++PredIter) {
+ BasicBlock *Pred = *PredIter;
+ NumPreds++;
+ Loop *PredLoop = LI.getLoopFor(Pred);
+ if (CurLoop == PredLoop) {
+ BasicBlock *PredPred = Pred->getSinglePredecessor();
+ if (PredPred) {
+ Loop *PredPredLoop = LI.getLoopFor(PredPred);
+ if (PredPredLoop != CurLoop &&
+ (!CurLoop || CurLoop->contains(PredPredLoop))) {
+ // Debug instructions should not be counted into considered size
+ if (sizeWithoutDebug(Pred) <= BreakBlockSizeLimit &&
+ !hasThreadGroupBarrierInBlock(Pred))
+ PredSet.insert(Pred);
+ else
+ AllPredLoopExit = false;
+ break;
+ }
+ }
+ } else if (!CurLoop || CurLoop->contains(PredLoop))
+ continue;
+ else {
+ AllPredLoopExit = false;
+ break;
+ }
+ }
+ if (AllPredLoopExit && NumPreds > 1) {
+ for (BasicBlock *Pred : PredSet) {
+ BasicBlock *PredPred = Pred->getSinglePredecessor();
+ Pred->moveAfter(PredPred);
+ }
+ }
+ }
+}
+
+void PISALayout::layoutBlocks(Function &F) {
+ SmallVector<BasicBlock *> VisitVec;
+ SmallSet<BasicBlock *, 8> VisitSet;
+ // Reorder basic block to allow more fall-through
+ BasicBlock *Entry = &(F.getEntryBlock());
+ VisitVec.push_back(Entry);
+
+ // Push a return block to make sure the last BB is the return block.
+ if (BasicBlock *LastReturnBlock = getLastReturnBlock(F))
+ if (LastReturnBlock != Entry) {
+ VisitVec.push_back(LastReturnBlock);
+ VisitSet.insert(LastReturnBlock);
+ }
+
+ while (!VisitVec.empty()) {
+ BasicBlock *BB = VisitVec.back();
+ // push in the empty successor
+ auto PushNoInstCond = [](BasicBlock *Succ) -> bool {
+ return sizeWithoutDebug(Succ) <= 1;
+ };
+ pushSucc(BB, PushNoInstCond, VisitVec, VisitSet);
+ if (BB != VisitVec.back())
+ continue;
+ // push in all the same-loop successors
+ auto PushAnyCond = [](BasicBlock *Succ) -> bool { return true; };
+ pushSucc(BB, PushAnyCond, VisitVec, VisitSet);
+ // pop
+ if (BB == VisitVec.back()) {
+ VisitVec.pop_back();
+ if (BB != Entry) {
+ BB->moveBefore(Entry);
+ Entry = BB;
+ }
+ }
+ }
+}
+
+FunctionPass *llvm::createPISALayoutPass() { return new PISALayout(); }
diff --git a/llvm/lib/Target/PISA/PISALegalizeCalls.cpp b/llvm/lib/Target/PISA/PISALegalizeCalls.cpp
new file mode 100644
index 0000000000000..d2972e6ab525b
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISALegalizeCalls.cpp
@@ -0,0 +1,505 @@
+//===-- PISALegalizeCalls.cpp - modify function signatures ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Modify function argument/return types to the ones support by PISA:
+// - natively supported types are used as is
+// - i8,i16,i32, and i64 scalar types
+// - vectors if 2,3, and 4 elements, with element types being i8,i16, and i64
+// - vectors if 2,3,4,5,6,7,8,16 and 32 elements, with element type being i32
+// - vectors of a single element are scalarized
+// - vectors of pointers are treated as equivalent native integer type
+// - vectors of p3/p4 pointers are treated as vectors of i32
+// - vectors of p0/p1/p2 pointers are treated as vectors of i64
+// - i1 and i4 types are extended/truncated to i16
+// - vectors of i1 are extended to ^2 size, cast to native integer type
+// - all other types are passed in via memory argument
+//===------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "llvm/IR/AttributeMask.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/Support/PISAAddrSpace.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
+
+#define DEBUG_TYPE "pisa-legalize-calls"
+#define DEBUG_NAME "PISA legalize calls"
+
+using namespace llvm;
+
+namespace {
+
+class PISALegalizeCalls : public ModulePass,
+ public InstVisitor<PISALegalizeCalls> {
+public:
+ static char ID;
+ PISALegalizeCalls() : ModulePass(ID) {}
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ void visitCallInst(CallInst &CI);
+ void visitIntrinsicInst(IntrinsicInst &I);
+ void visitReturnInst(ReturnInst &RI);
+
+private:
+ SmallVector<Function *, 8> Funcs;
+ SmallVector<CallInst *, 8> Calls;
+ SmallVector<ReturnInst *, 8> Returns;
+
+ bool runOnModule(Module &M) override;
+ bool needsModification(Type *Ty, const DataLayout &DL);
+ Type *getModifiedType(Type *, const DataLayout &, LLVMContext &);
+
+ void collectFuncs(Function &F);
+ void modifyFunctionSignature(Function &F);
+
+ void modifyReturnInst(ReturnInst *RI);
+ void modifyCallInst(CallInst *CI);
+};
+} // namespace
+
+char PISALegalizeCalls::ID = 0;
+INITIALIZE_PASS(PISALegalizeCalls, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+// determine if type will be modified in new function
+bool PISALegalizeCalls::needsModification(Type *Ty, const DataLayout &DL) {
+ bool Modify = false;
+ if (Ty->isVoidTy() || Ty->isPointerTy())
+ return false;
+
+ if (auto *VecTy = dyn_cast<FixedVectorType>(Ty)) {
+ auto NumElts = VecTy->getNumElements();
+ auto EltSize = VecTy->getScalarSizeInBits();
+ if (EltSize == 0) {
+ // vectors of pointers are treated as underlying integer
+ assert(VecTy->getElementType()->isPointerTy());
+ auto AS = VecTy->getElementType()->getPointerAddressSpace();
+ EltSize = DL.getPointerSizeInBits(AS);
+ }
+ switch (EltSize) {
+ case 0: { // vector of pointers
+ assert(VecTy->getElementType()->isPointerTy());
+ Modify = (NumElts > 4);
+ } break;
+ case 1:
+ case 4:
+ Modify = true;
+ break;
+ case 8:
+ case 16:
+ case 64:
+ Modify = (NumElts == 1) || (NumElts > 4);
+ break;
+ case 32:
+ Modify = (NumElts == 1) || ((NumElts > 8) && (NumElts != 16) &&
+ (NumElts != 32) && (NumElts != 64));
+ break;
+ }
+ } else if (auto *IntTy = dyn_cast<IntegerType>(Ty)) {
+ switch (IntTy->getScalarSizeInBits()) {
+ default:
+ llvm_unreachable("unsupported integer type");
+ break;
+ case 1:
+ case 4:
+ case 128:
+ Modify = true;
+ break;
+ case 8:
+ case 16:
+ case 32:
+ case 64:
+ break;
+ }
+ } else if (Ty->isFloatingPointTy()) {
+ switch (Ty->getScalarSizeInBits()) {
+ default:
+ llvm_unreachable("unsupported fp type");
+ break;
+ case 16:
+ case 32:
+ case 64:
+ break;
+ }
+ } else if (isa<StructType>(Ty)) {
+ Modify = true;
+ } else {
+ llvm_unreachable("unsupported type");
+ }
+ return Modify;
+}
+
+Type *PISALegalizeCalls::getModifiedType(Type *Ty, const DataLayout &DL,
+ LLVMContext &Ctx) {
+ Type *NewTy = Ty;
+ if (needsModification(Ty, DL)) {
+ if (isa<IntegerType>(Ty)) {
+ if (Ty->getScalarSizeInBits() == 1 || Ty->getScalarSizeInBits() == 4) {
+ NewTy = IntegerType::get(Ctx, 16); // i{1, 4} => i16
+ } else if (Ty->getScalarSizeInBits() == 128) {
+ auto *EltTy = IntegerType::get(Ctx, 64);
+ NewTy = FixedVectorType::get(EltTy, 2); // i128 => v2i64
+ } else {
+ llvm_unreachable("unsupported type to be modified");
+ }
+ } else if (auto *VecTy = dyn_cast<FixedVectorType>(Ty)) {
+ if (VecTy->getNumElements() == 1) { // <1 x i?> => i?
+ NewTy = VecTy->getElementType();
+ } else if (VecTy->getScalarSizeInBits() == 1) { // <? x i1> => i16
+ auto NewSize =
+ std::max((unsigned)PowerOf2Ceil(VecTy->getNumElements()), 16u);
+ NewTy = IntegerType::get(Ctx, NewSize);
+ } else { // e.g. <8 x i8>
+ int TotalSize = VecTy->getNumElements() * VecTy->getScalarSizeInBits();
+ int NumEle = TotalSize / 32;
+ if (NumEle > 0 && TotalSize % 32 == 0) {
+ Type *NewElemTy = IntegerType::get(Ctx, 32);
+ NewTy = FixedVectorType::get(NewElemTy, NumEle);
+ }
+ if (needsModification(NewTy, DL))
+ NewTy =
+ PointerType::get(Ctx, (unsigned)PISAAS::AddressSpace::PRIVATE);
+ }
+ } else {
+ NewTy = PointerType::get(Ctx, (unsigned)PISAAS::AddressSpace::PRIVATE);
+ }
+ }
+ return NewTy;
+}
+
+// record all call instructions that will require modification
+void PISALegalizeCalls::visitCallInst(CallInst &CI) {
+ auto *Caller = CI.getFunction();
+ auto *Callee = CI.getCalledFunction();
+
+ if (!Callee)
+ Callee = dyn_cast<Function>(CI.getCalledOperand());
+ if (!Callee && dyn_cast<InlineAsm>(CI.getCalledOperand()))
+ return;
+
+ assert((Callee || isa<PointerType>(CI.getCalledOperand()->getType())) &&
+ "unable to extract callee info");
+
+ // Indirect call
+ if (!Callee && !isa<PointerType>(CI.getCalledOperand()->getType()))
+ return;
+
+ // call must with pisa cc
+ if (CI.getCallingConv() == CallingConv::PISA_KERNEL)
+ return;
+
+ bool ArgNeedModification = llvm::any_of(CI.args(), [&](Value *Arg) {
+ auto *SType = Arg->getType();
+ return needsModification(SType, Caller->getParent()->getDataLayout());
+ });
+ if (ArgNeedModification ||
+ needsModification(CI.getType(), Caller->getParent()->getDataLayout()))
+ Calls.push_back(&CI);
+}
+
+void PISALegalizeCalls::visitIntrinsicInst(IntrinsicInst &I) {
+ assert(!IntrinsicInst::mayLowerToFunctionCall(I.getIntrinsicID()) &&
+ "Pre-Isel Intrinsics should be lowered to function calls by now");
+ return;
+}
+
+// record all return instructions that will require modification
+void PISALegalizeCalls::visitReturnInst(ReturnInst &RI) {
+ auto *F = RI.getFunction();
+
+ if (F->getCallingConv() == CallingConv::PISA_KERNEL)
+ return;
+
+ if (auto *RV = RI.getReturnValue()) {
+ if (needsModification(RV->getType(), F->getParent()->getDataLayout())) {
+ Returns.push_back(&RI);
+ }
+ }
+}
+
+void PISALegalizeCalls::collectFuncs(Function &F) {
+ if (F.isIntrinsic() || F.getCallingConv() == CallingConv::PISA_KERNEL)
+ return;
+
+ bool ToAdd = false;
+ for (auto &Arg : F.args()) {
+ if (needsModification(Arg.getType(), F.getParent()->getDataLayout()))
+ ToAdd = true;
+ }
+ auto *Ty = F.getFunctionType()->getReturnType();
+ if (needsModification(Ty, F.getParent()->getDataLayout()))
+ ToAdd = true;
+
+ if (ToAdd)
+ Funcs.push_back(&F);
+}
+
+void PISALegalizeCalls::modifyFunctionSignature(Function &F) {
+ auto &DL = F.getParent()->getDataLayout();
+ auto &Ctx = F.getContext();
+
+ AttributeList AL = F.getAttributes();
+ SmallVector<Type *> NewArgTys;
+ for (unsigned I = 0, E = F.arg_size(); I < E; ++I) {
+ auto *NewTy = getModifiedType(F.getArg(I)->getType(), DL, Ctx);
+ NewArgTys.push_back(NewTy);
+ AL = AL.removeParamAttributes(
+ Ctx, I,
+ AttributeFuncs::typeIncompatible(NewTy, F.getArg(I)->getAttributes()));
+ }
+ auto *RetTy = F.getFunctionType()->getReturnType();
+ auto *NewRetTy = getModifiedType(RetTy, DL, Ctx);
+ if (RetTy != NewRetTy) {
+ // return via hidden memory arg
+ if (NewRetTy->isPointerTy()) {
+ NewArgTys.push_back(NewRetTy);
+ NewRetTy = Type::getVoidTy(Ctx);
+ // void functions cannot return any argument value
+ for (const auto &Arg : F.args())
+ AL = AL.removeParamAttribute(Ctx, Arg.getArgNo(), Attribute::Returned);
+ }
+ AL = AL.removeRetAttributes(
+ Ctx, AttributeFuncs::typeIncompatible(NewRetTy,
+ F.getAttributes().getRetAttrs()));
+ }
+
+ // update function definition
+ FunctionType *FTy = FunctionType::get(NewRetTy, NewArgTys, false);
+ Function *NewF = Function::Create(FTy, F.getLinkage(), F.getAddressSpace());
+ ValueToValueMapTy VMap;
+
+ // map args 1:1, but types will be different.
+ // modify actual references to args below.
+ for (unsigned I = 0; I < F.arg_size(); I++) {
+ auto *SArg = F.getArg(I);
+ VMap[SArg] = SArg;
+ }
+
+ SmallVector<ReturnInst *, 8> Returns;
+ CloneFunctionInto(NewF, &F, VMap,
+ llvm::CloneFunctionChangeType::LocalChangesOnly, Returns,
+ "", 0);
+ F.getParent()->getFunctionList().insert(F.getIterator(), NewF);
+
+ // transform new arg types into ones expected within function
+ IRBuilder<> IRB(Ctx);
+ if (!NewF->isDeclaration()) {
+ auto FirstBB = NewF->begin();
+ IRB.SetInsertPoint(FirstBB->begin());
+ }
+ for (unsigned I = 0; I < F.arg_size(); I++) {
+ auto *SArg = F.getArg(I);
+ auto *DArg = NewF->getArg(I);
+ auto *SType = SArg->getType(); // type to change from
+ auto *DType = DArg->getType(); // type to change to
+ if (!NewF->isDeclaration()) {
+ if (SType != DType) {
+ if (DType->isPointerTy()) { // value passed via memory
+ auto *Load = IRB.CreateLoad(SType, DArg);
+ SArg->replaceAllUsesWith(Load);
+ } else {
+ auto SSize = DL.getTypeSizeInBits(SType);
+ auto DSize = DL.getTypeSizeInBits(DType);
+ if (SSize == DSize) { // <1 x i?> => i?
+ auto *DCast = IRB.CreateBitCast(DArg, SType);
+ SArg->replaceAllUsesWith(DCast);
+ } else if (SType->isIntegerTy(1) ||
+ SType->isIntegerTy(4)) { // i8 => i{1, 4}
+ auto *Trunc = IRB.CreateTrunc(DArg, SType);
+ SArg->replaceAllUsesWith(Trunc);
+ } else if (SType->isVectorTy()) { // i16 => <? x i1>
+ assert(SType->getScalarSizeInBits() == 1);
+ auto *ScalarType = IntegerType::get(Ctx, SSize);
+ auto *Trunc = IRB.CreateTrunc(DArg, ScalarType);
+ auto *Cast = IRB.CreateBitCast(Trunc, SType);
+ SArg->replaceAllUsesWith(Cast);
+ } else {
+ llvm_unreachable("unsupported argument type");
+ }
+ }
+ } else {
+ SArg->replaceAllUsesWith(DArg);
+ }
+ }
+ }
+ // return modification handled in visitReturnInst
+ NewF->takeName(&F);
+ NewF->setAttributes(AL);
+ NewF->setCallingConv(F.getCallingConv());
+ F.replaceAllUsesWith(NewF);
+ F.eraseFromParent();
+}
+
+void PISALegalizeCalls::modifyReturnInst(ReturnInst *RI) {
+ auto *F = RI->getFunction();
+ auto &Ctx = F->getContext();
+ assert(F->getCallingConv() != CallingConv::PISA_KERNEL &&
+ "return instruction in kernel");
+
+ auto &DL = F->getParent()->getDataLayout();
+ auto *RV = RI->getReturnValue();
+ auto *SType = RV->getType(); // type to change from
+ auto *DType = F->getFunctionType()->getReturnType(); // type to change to
+ if (!SType->isVoidTy() && (SType != DType)) {
+ IRBuilder<> IRB(dyn_cast<Instruction>(RI));
+ if (DType->isVoidTy()) {
+ // return via hidden memory arg
+ auto *Ptr = F->getArg(F->arg_size() - 1);
+ IRB.CreateStore(RV, Ptr);
+ IRB.CreateRet(nullptr);
+ } else {
+ auto SSize = DL.getTypeSizeInBits(SType);
+ auto DSize = DL.getTypeSizeInBits(DType);
+ if (SSize == DSize) { // <1 x i?> => i?
+ auto *DCast = IRB.CreateBitCast(RV, DType);
+ IRB.CreateRet(DCast);
+ } else if (SType->isIntegerTy(1) ||
+ SType->isIntegerTy(4)) { // i{1, 4} => i8
+ auto SExt = F->hasRetAttribute(Attribute::SExt);
+ auto *Extend =
+ SExt ? IRB.CreateSExt(RV, DType) : IRB.CreateZExt(RV, DType);
+ IRB.CreateRet(Extend);
+ } else if (SType->isVectorTy()) { // <? x i1> => i16
+ assert(SType->getScalarSizeInBits() == 1);
+ auto *ScalarType = IntegerType::get(Ctx, SSize);
+ auto *Cast = IRB.CreateBitCast(RV, ScalarType);
+ auto *Extend = IRB.CreateZExt(Cast, DType);
+ IRB.CreateRet(Extend);
+ } else {
+ llvm_unreachable("unsupported return type");
+ }
+ }
+ RI->eraseFromParent();
+ }
+}
+
+void PISALegalizeCalls::modifyCallInst(CallInst *CI) {
+ auto *F = CI->getFunction();
+ auto &Ctx = F->getContext();
+ auto &DL = F->getParent()->getDataLayout();
+
+ auto *Callee = CI->getCalledOperand();
+ SmallVector<Value *, 8> NewArgs;
+ SmallVector<Type *, 8> NewArgTys;
+
+ IRBuilder<> IRB(dyn_cast<Instruction>(CI));
+ for (unsigned I = 0; I < CI->arg_size(); I++) {
+ auto *Arg = CI->getArgOperand(I);
+ auto *SType = Arg->getType(); // type to change from
+ auto *DType = getModifiedType(SType, DL, Ctx); // type to change to
+ Value *NewV = nullptr;
+ if (SType != DType) {
+ if (DType->isPointerTy()) { // pass via memory
+ NewV = IRB.CreateAlloca(SType);
+ IRB.CreateStore(Arg, NewV);
+ } else {
+ auto SSize = DL.getTypeSizeInBits(SType);
+ auto DSize = DL.getTypeSizeInBits(DType);
+ if (SSize == DSize) { // <1 x i?> => i?
+ NewV = IRB.CreateBitCast(Arg, DType);
+ } else if (SType->isIntegerTy(1) ||
+ SType->isIntegerTy(4)) { // i{1, 4} => i8
+ auto SExt = CI->getParamAttr(I, Attribute::SExt).getKindAsEnum() ==
+ Attribute::SExt;
+ NewV = SExt ? IRB.CreateSExt(Arg, DType) : IRB.CreateZExt(Arg, DType);
+ } else if (SType->isVectorTy()) { // <? x i1> => i16
+ assert(SType->getScalarSizeInBits() == 1);
+ auto *ScalarType = IntegerType::get(Ctx, SSize);
+ auto *Cast = IRB.CreateBitCast(Arg, ScalarType);
+ NewV = IRB.CreateZExt(Cast, DType);
+ } else {
+ llvm_unreachable("unsupported call argument type");
+ }
+ }
+ }
+ if (NewV) {
+ NewArgs.push_back(NewV);
+ NewArgTys.push_back(NewV->getType());
+ } else {
+ NewArgs.push_back(Arg);
+ NewArgTys.push_back(Arg->getType());
+ }
+ }
+
+ auto *RetTy = CI->getType();
+ auto *NewRetTy = RetTy;
+ Value *RetAlloca = nullptr;
+ NewRetTy = getModifiedType(RetTy, DL, Ctx);
+ if ((RetTy != NewRetTy) && NewRetTy->isPointerTy()) {
+ // return via hidden memory arg
+ RetAlloca = IRB.CreateAlloca(RetTy);
+ NewArgs.push_back(RetAlloca);
+ NewArgTys.push_back(NewRetTy);
+ NewRetTy = Type::getVoidTy(Ctx);
+ }
+ auto *FTy = FunctionType::get(NewRetTy, NewArgTys, false);
+ auto *NewCI = IRB.CreateCall(FTy, Callee, NewArgs);
+ NewCI->setCallingConv(CI->getCallingConv());
+
+ // handle return value
+ if (RetTy != NewRetTy) {
+ auto SSize = DL.getTypeSizeInBits(RetTy);
+ auto DSize = NewRetTy->isVoidTy() ? 0u : DL.getTypeSizeInBits(NewRetTy);
+ if (SSize == DSize) { // i? => <1 x i?>
+ auto *Bitcast = IRB.CreateBitCast(NewCI, RetTy);
+ CI->replaceAllUsesWith(Bitcast);
+ } else if (RetTy->isIntegerTy(1) ||
+ RetTy->isIntegerTy(4)) { // i8 => i{1, 4}
+ auto *Trunc = IRB.CreateTrunc(NewCI, RetTy);
+ CI->replaceAllUsesWith(Trunc);
+ } else if (DSize == 0) {
+ // return via hidden memory arg
+ auto *Load = IRB.CreateLoad(RetTy, RetAlloca);
+ CI->replaceAllUsesWith(Load);
+ } else if (RetTy->isVectorTy()) { // i16 => <? x i1>
+ assert(RetTy->getScalarSizeInBits() == 1);
+ auto *ScalarType = IntegerType::get(Ctx, SSize);
+ auto *Trunc = IRB.CreateTrunc(NewCI, ScalarType);
+ auto *Cast = IRB.CreateBitCast(Trunc, RetTy);
+ CI->replaceAllUsesWith(Cast);
+ } else {
+ llvm_unreachable("unsupported return type");
+ }
+ } else {
+ CI->replaceAllUsesWith(NewCI);
+ }
+ CI->eraseFromParent();
+}
+
+bool PISALegalizeCalls::runOnModule(Module &M) {
+ // record functions to be modified
+ for (auto &F : M) {
+ collectFuncs(F);
+ }
+ // modify function signatures
+ for (auto &F : Funcs) {
+ modifyFunctionSignature(*F);
+ }
+
+ // record call/return instructions
+ for (auto &F : M) {
+ visit(F);
+ }
+ // modify call/return instructions
+ for (auto &I : Returns) {
+ modifyReturnInst(I);
+ }
+ for (auto &I : Calls) {
+ modifyCallInst(I);
+ }
+ return !(Calls.empty() && Returns.empty() && Funcs.empty());
+}
+
+ModulePass *llvm::createPISALegalizeCallsPass() {
+ return new PISALegalizeCalls();
+}
diff --git a/llvm/lib/Target/PISA/PISALegalizePredicates.cpp b/llvm/lib/Target/PISA/PISALegalizePredicates.cpp
new file mode 100644
index 0000000000000..86f9e37238329
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISALegalizePredicates.cpp
@@ -0,0 +1,994 @@
+//=== PISALegalizePredicates.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Unified pre-legalization strategy for bitwise operations on predicate
+// (i1) values. PISA has no bitwise operations on predicate registers, and
+// `cmp` instructions yield i32 -1/0 values. The legalizer would
+// widen i1 chains piecewise (i1 -> i32 for bitwise, i8/i16 for other sub-i16
+// types), leaving redundant ext/trunc pairs and missing fusion opportunities
+// (e.g. bfn).
+//
+// This pass walks forward from every G_ICMP/G_FCMP result and rewrites any
+// chain of bitwise-on-i1, sext/zext/anyext-of-i1, vector-extract from a
+// vector-of-i1, and trunc-back-to-i1 to operate uniformly on i32. Consumers
+// that still need an i1 (G_BRCOND, G_SELECT condition, etc.) are fed via a
+// single G_ICMP ne 0 restoration at the boundary.
+//
+// Pipeline placement: scheduled in addPreLegalizeMachineIR(); the pass only
+// rewrites scalar/vector s1 chains and leaves other pre-legalization results
+// undisturbed.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "PISA.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "pisa-legalize-predicates"
+#define DEBUG_NAME "PISA Legalize Predicates"
+
+using namespace llvm;
+
+STATISTIC(NumPromoted, "Number of registers promoted to s32");
+STATISTIC(NumRestorations, "Number of sink restorations inserted");
+STATISTIC(NumErased, "Number of dead original instructions erased");
+
+namespace {
+
+// ===================== Pass boilerplate =====================
+
+class PISALegalizePredicates : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISALegalizePredicates() : MachineFunctionPass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+ bool runOnMachineFunction(MachineFunction &MF) override;
+};
+
+// ===================== Implementation =====================
+
+class PISALegalizePredicatesImpl {
+ MachineFunction &MF;
+ MachineRegisterInfo &MRI;
+ MachineIRBuilder B;
+
+ // Map from each promotable original Register to its s32 replacement.
+ // Scalar s1 -> s32; <Nxs1> -> <Nxs32>; sN (N>1) coming from sext-of-s1 ->
+ // s32.
+ DenseMap<Register, Register> Promoted;
+
+ // Cache: classification status for registers we've already inspected.
+ enum Status { Unknown, Yes, No };
+ DenseMap<Register, Status> Cache;
+
+ // Magnitude domain of a register's *original* value. The internal promoted
+ // form is always the SEXT representation ({0, -1}); the domain records what
+ // the value was before promotion so the exact bits can be restored at a
+ // non-i1 (integer) sink:
+ // Sext -- true == all-ones ({0, -1}); e.g. OpenCL vector relational.
+ // Zext -- true == 1 ({0, 1}); e.g. OpenCL scalar relational.
+ // Any -- no magnitude (a 1-bit value or a literal 0); matches either side
+ // of a bitwise op.
+ // A bitwise op mixing Sext and Zext operands has an ambiguous result
+ // magnitude and is excluded from promotion (tryGetDomain returns false).
+ enum class Domain : unsigned char { Any, Sext, Zext };
+ // Memo codes: 0 = visiting (cycle guard), 1 = Any, 2 = Sext, 3 = Zext,
+ // 4 = bail.
+ DenseMap<Register, unsigned char> DomMemo;
+
+public:
+ PISALegalizePredicatesImpl(MachineFunction &MF)
+ : MF(MF), MRI(MF.getRegInfo()) {
+ B.setMF(MF);
+ }
+
+ bool run();
+
+private:
+ // ---- Type helpers ----
+ // Returns true for s1 or <N x s1>; getScalarSizeInBits() returns 1 in
+ // either case.
+ static bool isBoolType(LLT Ty) { return Ty.getScalarSizeInBits() == 1; }
+ static LLT promotedType(LLT Ty) {
+ // s1 -> s32; <Nxs1> -> <Nxs32>; sN -> s32; <NxsM> -> <Nxs32>.
+ LLT S32 = LLT::integer(32);
+ if (Ty.isVector())
+ return LLT::fixed_vector(Ty.getNumElements(), S32);
+ return S32;
+ }
+
+ // Opcodes whose originals stay alive in MIR after rewrite (the new s32
+ // chain is built alongside, not in place).
+ static bool isKeptAliveOpcode(unsigned Opc) {
+ return Opc == TargetOpcode::G_ICMP || Opc == TargetOpcode::G_FCMP ||
+ Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::COPY;
+ }
+
+ // ---- Constant value helpers ----
+ // Boolean-domain test: returns true iff the constant is 0 (false),
+ // 1 (interpreted as true on s1), or all-ones (-1 on wider). Other constant
+ // values disqualify promotion because they aren't predicate values. Caller
+ // must guarantee MI is G_CONSTANT.
+ static bool isBoolConst(const MachineInstr &MI, int64_t &PromotedVal) {
+ assert(MI.getOpcode() == TargetOpcode::G_CONSTANT);
+ const APInt &V = MI.getOperand(1).getCImm()->getValue();
+ if (V.isZero()) {
+ PromotedVal = 0;
+ return true;
+ }
+ // For wider types -1 is "all-ones"; for i1 the bit "true" is value 1 and
+ // also already covered by isAllOnes (1-bit all-ones == 1). Either way map
+ // to -1 in the promoted s32 form.
+ if (V.isAllOnes()) {
+ PromotedVal = -1;
+ return true;
+ }
+ return false;
+ }
+
+ // ---- Classification (Phase 1) ----
+ // Returns true if R is part of the promotable chain rooted at some cmp.
+ bool isPromotable(Register R);
+
+ // ---- Use validation (Phase 2) ----
+ // Returns true if a use of R inside UseInstr can be handled (either the use
+ // is itself promotable, or we know how to insert a sink restoration).
+ bool isHandledUse(MachineInstr &UseInstr, Register R);
+
+ // Drop R from the promotion set transitively if any of its uses is
+ // unhandled. Re-iterates to fix-point.
+ void invalidateUnhandled();
+
+ // Compute the original magnitude domain of R (see Domain above). Returns
+ // false if the magnitude is ambiguous (a bitwise op mixing Sext and Zext
+ // operands), meaning R cannot be safely restored at an integer sink and must
+ // not be promoted. Memoized in DomMemo.
+ bool tryGetDomain(Register R, Domain &Out);
+
+ // ---- Rewrite (Phase 3) ----
+ Register getOrBuildPromoted(Register R);
+ Register restoreS1(Register PromotedReg, MachineInstr &InsertBeforeMI);
+ void eraseDeadOriginals();
+
+ // Inserts B at the right point for MI's *next* instruction.
+ void setInsertPointAfter(MachineInstr &MI) {
+ B.setInsertPt(*MI.getParent(), std::next(MI.getIterator()));
+ }
+ // Inserts B right before MI.
+ void setInsertPointBefore(MachineInstr &MI) {
+ B.setInsertPt(*MI.getParent(), MI.getIterator());
+ }
+};
+
+// -----------------------------------------------------------------------------
+// Phase 1: classification.
+// -----------------------------------------------------------------------------
+
+bool PISALegalizePredicatesImpl::isPromotable(Register R) {
+ auto It = Cache.find(R);
+ if (It != Cache.end())
+ return It->second == Yes;
+
+ // Mark as "currently visiting" by inserting No first. Cycles (e.g. through
+ // a G_PHI) thus terminate as not-promotable.
+ Cache[R] = No;
+
+ MachineInstr *Def = MRI.getVRegDef(R);
+ assert(Def && "expected a def for every virtual register in SSA MIR");
+
+ LLT Ty = MRI.getType(R);
+ bool Result = false;
+
+ switch (Def->getOpcode()) {
+ case TargetOpcode::G_ICMP:
+ case TargetOpcode::G_FCMP:
+ // Seed: a cmp whose result is i1 (scalar or vector); isBoolType covers
+ // both since getScalarSizeInBits == 1 for s1 and <N x s1>.
+ Result = isBoolType(Ty);
+ break;
+ case TargetOpcode::G_AND:
+ case TargetOpcode::G_OR:
+ case TargetOpcode::G_XOR:
+ // Promotable if both operands are promotable (or boolean-domain consts).
+ Result = isPromotable(Def->getOperand(1).getReg()) &&
+ isPromotable(Def->getOperand(2).getReg());
+ break;
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_ANYEXT:
+ case TargetOpcode::G_TRUNC:
+ case TargetOpcode::COPY:
+ Result = isPromotable(Def->getOperand(1).getReg());
+ break;
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT:
+ Result = isPromotable(Def->getOperand(1).getReg());
+ break;
+ case TargetOpcode::G_CONSTANT: {
+ int64_t Dummy;
+ Result = isBoolConst(*Def, Dummy);
+ break;
+ }
+ default:
+ break;
+ }
+
+ Cache[R] = Result ? Yes : No;
+ return Result;
+}
+
+// -----------------------------------------------------------------------------
+// Phase 2: validate uses; demote any register whose uses we can't handle.
+// -----------------------------------------------------------------------------
+
+bool PISALegalizePredicatesImpl::isHandledUse(MachineInstr &UseInstr,
+ Register R) {
+ // If UseInstr is itself promotable (its def is in the chain), the operand
+ // will be rewritten in lock-step with the def.
+ if (UseInstr.getNumDefs() >= 1 &&
+ Cache.lookup(UseInstr.getOperand(0).getReg()) == Yes)
+ return true;
+
+ // Otherwise, this is a sink. We need to be able to restore the original
+ // value at the boundary. For vector promotable defs feeding a
+ // non-promotable user, we don't try to vector-restore in v1. Bail.
+ if (MRI.getType(R).isVector())
+ return false;
+
+ // Scalar restoration: G_ICMP ne s32, 0 -> s1, or G_TRUNC s32 -> sN.
+ // Both are always materializable, so any scalar use is "handled" in
+ // principle, but we still skip a few opcodes that need special care:
+ switch (UseInstr.getOpcode()) {
+ case TargetOpcode::G_PHI:
+ // PHI of i1 across blocks requires inserting restoration in the
+ // predecessor blocks. v1 conservatively skips.
+ return false;
+ default:
+ return true;
+ }
+}
+
+void PISALegalizePredicatesImpl::invalidateUnhandled() {
+ bool Changed = true;
+ while (Changed) {
+ Changed = false;
+ for (auto &E : Cache) {
+ if (E.second != Yes)
+ continue;
+ Register R = E.first;
+
+ // (a) All uses of R must be handled (in-set or known sink).
+ bool OK = true;
+ for (MachineOperand &U : MRI.use_nodbg_operands(R)) {
+ MachineInstr *UseInstr = U.getParent();
+ if (!isHandledUse(*UseInstr, R)) {
+ OK = false;
+ break;
+ }
+ }
+ if (!OK) {
+ E.second = No;
+ Changed = true;
+ continue;
+ }
+
+ // (b) All required operands of R's def must still be promotable.
+ // If an operand was just demoted above, cascade the demotion so that
+ // Phase 3 doesn't recursively try to promote a non-promotable reg.
+ MachineInstr *Def = MRI.getVRegDef(R);
+ assert(Def);
+ auto IsOperandPromotable = [&](Register Op) {
+ return Cache.lookup(Op) == Yes;
+ };
+ switch (Def->getOpcode()) {
+ case TargetOpcode::G_AND:
+ case TargetOpcode::G_OR:
+ case TargetOpcode::G_XOR: {
+ if (!IsOperandPromotable(Def->getOperand(1).getReg()) ||
+ !IsOperandPromotable(Def->getOperand(2).getReg())) {
+ E.second = No;
+ Changed = true;
+ }
+ break;
+ }
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_ANYEXT:
+ case TargetOpcode::G_TRUNC:
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT:
+ case TargetOpcode::COPY: {
+ if (!IsOperandPromotable(Def->getOperand(1).getReg())) {
+ E.second = No;
+ Changed = true;
+ }
+ break;
+ }
+ // Seeds (G_ICMP, G_FCMP) and G_CONSTANT don't depend on the
+ // promotability of any operand.
+ default:
+ break;
+ }
+ }
+ }
+}
+
+bool PISALegalizePredicatesImpl::tryGetDomain(Register R, Domain &Out) {
+ auto It = DomMemo.find(R);
+ if (It != DomMemo.end()) {
+ switch (It->second) {
+ case 1:
+ Out = Domain::Any;
+ return true;
+ case 2:
+ Out = Domain::Sext;
+ return true;
+ case 3:
+ Out = Domain::Zext;
+ return true;
+ default: // 0 (cycle) or 4 (bail)
+ return false;
+ }
+ }
+ DomMemo[R] = 0; // visiting -- a cycle reaching back here resolves to bail.
+ auto Set = [&](Domain D) {
+ DomMemo[R] = D == Domain::Any ? 1 : D == Domain::Sext ? 2 : 3;
+ Out = D;
+ return true;
+ };
+ auto Bail = [&]() {
+ DomMemo[R] = 4;
+ return false;
+ };
+
+ // A 1-bit value (scalar s1 or <N x s1>) carries no magnitude; the extend
+ // that consumes it decides the domain.
+ if (MRI.getType(R).getScalarSizeInBits() == 1)
+ return Set(Domain::Any);
+
+ MachineInstr *Def = MRI.getVRegDef(R);
+ switch (Def->getOpcode()) {
+ case TargetOpcode::G_ZEXT:
+ return Set(Domain::Zext);
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ANYEXT:
+ // ANYEXT leaves the high bits undefined, so the sext form ({0, -1}) is an
+ // acceptable refinement.
+ return Set(Domain::Sext);
+ case TargetOpcode::G_CONSTANT: {
+ const APInt &V = Def->getOperand(1).getCImm()->getValue();
+ // 0 matches any domain; all-ones is a true value of -1 (Sext). Other
+ // constants are not classified promotable, so never reach here.
+ return Set(V.isZero() ? Domain::Any : Domain::Sext);
+ }
+ case TargetOpcode::G_TRUNC:
+ case TargetOpcode::COPY:
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
+ Domain D;
+ if (!tryGetDomain(Def->getOperand(1).getReg(), D))
+ return Bail();
+ return Set(D);
+ }
+ case TargetOpcode::G_AND:
+ case TargetOpcode::G_OR:
+ case TargetOpcode::G_XOR: {
+ Domain A, BD;
+ if (!tryGetDomain(Def->getOperand(1).getReg(), A) ||
+ !tryGetDomain(Def->getOperand(2).getReg(), BD))
+ return Bail();
+ if (A == Domain::Any)
+ return Set(BD);
+ if (BD == Domain::Any)
+ return Set(A);
+ if (A == BD)
+ return Set(A);
+ return Bail(); // mixed sext/zext -- ambiguous magnitude.
+ }
+ default:
+ // tryGetDomain is only ever called on registers in the promotable closure,
+ // whose defs are one of the opcodes handled above (i1-typed values, incl.
+ // G_ICMP/G_FCMP results, are handled before the switch).
+ llvm_unreachable("magnitude domain queried for a non-promotable opcode");
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Phase 3: rewrite.
+// -----------------------------------------------------------------------------
+
+Register PISALegalizePredicatesImpl::getOrBuildPromoted(Register R) {
+ auto It = Promoted.find(R);
+ if (It != Promoted.end())
+ return It->second;
+
+ assert(isPromotable(R) && "asked to promote a non-promotable reg");
+
+ MachineInstr *Def = MRI.getVRegDef(R);
+ assert(Def && "expected a def");
+ LLT OrigTy = MRI.getType(R);
+ LLT NewTy = promotedType(OrigTy);
+
+ // Allocate the destination s32 register up front so cyclic chains (none
+ // expected, but safe) terminate cleanly via Promoted lookup.
+ Register Dst = MRI.createGenericVirtualRegister(NewTy);
+ Promoted[R] = Dst;
+ LLVM_DEBUG(dbgs() << " Promoting %" << R.virtRegIndex() << " ("
+ << MRI.getType(R) << " -> " << NewTy << "): " << *Def);
+ ++NumPromoted;
+
+ switch (Def->getOpcode()) {
+ case TargetOpcode::G_ICMP: {
+ // Peephole for the common reduction pattern:
+ // %p = ... (promotable, in s32 represents 0 or -1)
+ // %r = G_ICMP eq/ne %p, <0 or all-ones>
+ // The cmp's promoted s32 form is just Promoted(%p) (or its XOR -1)
+ // depending on the predicate and constant -- skip the
+ // trunc -> icmp -> sext detour the legalizer would otherwise produce.
+ // The eq/ne peephole below materializes scalar values (a scalar -1 / xor
+ // and a scalar i1 compare), so it only applies to a scalar cmp result. A
+ // vector cmp (<N x i1>) falls through to the vector-safe G_SEXT default.
+ CmpInst::Predicate Pred =
+ (CmpInst::Predicate)Def->getOperand(1).getPredicate();
+ if (!MRI.getType(R).isVector() &&
+ (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE)) {
+ Register LhsReg = Def->getOperand(2).getReg();
+ Register RhsReg = Def->getOperand(3).getReg();
+
+ auto ClassifyOperands = [&](Register A, Register B, Register &PromOp,
+ bool &IsAllOnes) -> bool {
+ if (!isPromotable(A))
+ return false;
+ MachineInstr *BDef = MRI.getVRegDef(B);
+ if (BDef->getOpcode() != TargetOpcode::G_CONSTANT)
+ return false;
+ const APInt &V = BDef->getOperand(1).getCImm()->getValue();
+ if (V.isZero()) {
+ PromOp = A;
+ IsAllOnes = false;
+ return true;
+ }
+ if (V.isAllOnes()) {
+ PromOp = A;
+ IsAllOnes = true;
+ return true;
+ }
+ return false;
+ };
+
+ Register PromOp;
+ bool IsAllOnes = false;
+ if (ClassifyOperands(LhsReg, RhsReg, PromOp, IsAllOnes) ||
+ ClassifyOperands(RhsReg, LhsReg, PromOp, IsAllOnes)) {
+ Register PromS32 = getOrBuildPromoted(PromOp);
+ // eq(p, all_ones) == p ; eq(p, 0) == ~p
+ // ne(p, 0) == p ; ne(p, all_ones) == ~p
+ bool Negate = (Pred == CmpInst::ICMP_EQ) ^ IsAllOnes;
+ if (!Negate) {
+ Promoted[R] = PromS32;
+ return PromS32;
+ }
+ setInsertPointAfter(*Def);
+ auto AllOnesC = B.buildConstant(LLT::integer(32), -1);
+ B.buildXor(Dst, PromS32, AllOnesC.getReg(0));
+ break;
+ }
+
+ // Peephole: both operands are promotable (e.g. icmp eq of two AND
+ // chains). Compare the promoted s32 forms directly. Both are in the
+ // {0, -1} domain, so eq/ne is preserved.
+ if (isPromotable(LhsReg) && isPromotable(RhsReg)) {
+ Register PromLHS = getOrBuildPromoted(LhsReg);
+ Register PromRHS = getOrBuildPromoted(RhsReg);
+ setInsertPointAfter(*Def);
+ Register CmpS32 = MRI.createGenericVirtualRegister(LLT::integer(1));
+ B.buildICmp(Pred, CmpS32, PromLHS, PromRHS);
+ B.buildSExt(Dst, CmpS32);
+ break;
+ }
+ }
+ // Default: no peephole matched; promote via sext.
+ setInsertPointAfter(*Def);
+ B.buildSExt(Dst, R);
+ break;
+ }
+ case TargetOpcode::G_FCMP: {
+ // Materialize one G_SEXT of the cmp's i1 def to s32 (scalar or vector),
+ // right after the cmp. We keep the original cmp alive -- any non-promoted
+ // sink (e.g. G_BRCOND) keeps using its i1 def directly.
+ setInsertPointAfter(*Def);
+ B.buildSExt(Dst, R);
+ break;
+ }
+ case TargetOpcode::G_CONSTANT: {
+ int64_t Val = 0;
+ bool OK = isBoolConst(*Def, Val);
+ (void)OK;
+ assert(OK);
+ setInsertPointAfter(*Def);
+ auto NewC = B.buildConstant(NewTy, Val);
+ Promoted[R] = NewC.getReg(0);
+ return NewC.getReg(0);
+ }
+ case TargetOpcode::G_AND:
+ case TargetOpcode::G_OR:
+ case TargetOpcode::G_XOR: {
+ Register LHS = getOrBuildPromoted(Def->getOperand(1).getReg());
+ Register RHS = getOrBuildPromoted(Def->getOperand(2).getReg());
+ setInsertPointBefore(*Def);
+ B.buildInstr(Def->getOpcode(), {Dst}, {LHS, RHS});
+ break;
+ }
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_ANYEXT:
+ case TargetOpcode::G_TRUNC: {
+ // The promoted source is already an s32 (or <Nxs32>) representation of
+ // a predicate-domain value (0 / all-ones). The corresponding promoted
+ // form for *this* instruction is the same s32 representation, so just
+ // alias to the source's promoted reg.
+ Register Src = getOrBuildPromoted(Def->getOperand(1).getReg());
+ Promoted[R] = Src;
+ return Src;
+ }
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
+ // Replace the vector source with its <Nxs32> promoted form and extract
+ // the same index at s32.
+ Register VecS32 = getOrBuildPromoted(Def->getOperand(1).getReg());
+ Register Idx = Def->getOperand(2).getReg();
+ setInsertPointBefore(*Def);
+ B.buildExtractVectorElement(Dst, VecS32, Idx);
+ break;
+ }
+ case TargetOpcode::COPY: {
+ Register Src = getOrBuildPromoted(Def->getOperand(1).getReg());
+ Promoted[R] = Src;
+ return Src;
+ }
+ }
+ // The assert at the top of this function -- isPromotable(R) -- guarantees
+ // the opcode is one of the cases handled above.
+
+ return Dst;
+}
+
+Register PISALegalizePredicatesImpl::restoreS1(Register PromotedReg,
+ MachineInstr &InsertBeforeMI) {
+ // Build the restoration just before the sink so it trivially dominates.
+ // Each sink gets its own restoration -- sharing across sinks would need
+ // dominance analysis and is not worth the complexity in v1.
+ setInsertPointBefore(InsertBeforeMI);
+ auto Zero = B.buildConstant(LLT::integer(32), 0);
+ Register Dst = MRI.createGenericVirtualRegister(LLT::integer(1));
+ B.buildICmp(CmpInst::ICMP_NE, Dst, PromotedReg, Zero.getReg(0));
+ return Dst;
+}
+
+bool PISALegalizePredicatesImpl::run() {
+ // -- Phase 1: classify every reachable register.
+ // Seed with every cmp def, then transitively classify any user that reads
+ // those defs, etc. We rely on the memoized isPromotable() to do the work
+ // recursively when we encounter a use whose own operands haven't been
+ // classified yet.
+ SmallVector<MachineInstr *, 32> Cmps;
+ for (auto &MBB : MF) {
+ for (auto &MI : MBB) {
+ unsigned Opc = MI.getOpcode();
+ if (Opc != TargetOpcode::G_ICMP && Opc != TargetOpcode::G_FCMP)
+ continue;
+ // isPromotable below filters out cmps with non-i1 result; no need to
+ // pre-filter here.
+ Cmps.push_back(&MI);
+ }
+ }
+
+ if (Cmps.empty())
+ return false;
+
+ // BFS forward from each cmp to discover the promotable closure. The
+ // memoized isPromotable() returns by classifying defs; here we also need
+ // to populate the Cache for downstream uses that don't get directly
+ // queried. We do this by walking users.
+ SmallVector<Register, 64> Worklist;
+ for (MachineInstr *MI : Cmps) {
+ Register R = MI->getOperand(0).getReg();
+ if (isPromotable(R))
+ Worklist.push_back(R);
+ }
+ while (!Worklist.empty()) {
+ Register R = Worklist.pop_back_val();
+ for (MachineOperand &U : MRI.use_nodbg_operands(R)) {
+ MachineInstr *UI = U.getParent();
+ // Sinks (G_BRCOND/G_STORE/G_RETURN) have no def -- Phase 2 deals with
+ // them. operand(0).isReg() is guaranteed by MIR semantics whenever
+ // getNumDefs >= 1.
+ if (UI->getNumDefs() < 1)
+ continue;
+ Register UR = UI->getOperand(0).getReg();
+ auto It = Cache.find(UR);
+ if (It != Cache.end())
+ continue; // already classified
+ if (isPromotable(UR))
+ Worklist.push_back(UR);
+ }
+ }
+
+ // -- Phase 2: invalidate any promotable reg whose uses we can't handle.
+ invalidateUnhandled();
+
+ // -- Phase 2b: drop chains whose original integer magnitude is ambiguous
+ // (a bitwise op mixing sext- and zext-rooted operands). They cannot be
+ // restored exactly at an integer sink. Re-run use validation after each
+ // demotion round so the cascade reaches consumers of the dropped regs.
+ {
+ bool Changed = true;
+ while (Changed) {
+ Changed = false;
+ DomMemo.clear();
+ for (auto &E : Cache) {
+ if (E.second != Yes)
+ continue;
+ Domain D;
+ if (!tryGetDomain(E.first, D)) {
+ E.second = No;
+ Changed = true;
+ }
+ }
+ if (Changed)
+ invalidateUnhandled();
+ }
+ }
+
+ // Collect the final promote set and decide whether the rewrite is
+ // profitable. A single G_XOR-with-true (NOT of a cmp) feeding a branch is
+ // already handled by the existing opt_brcond_by_inverting_cond combiner --
+ // promoting it would only block that rule. We require *either*
+ // - at least one G_EXTRACT_VECTOR_ELT (a vectorized-cmp reduction), or
+ // - two or more bitwise ops in the promote set (a real reduction chain
+ // that the legalizer would otherwise widen piecewise).
+ //
+ // We additionally require the chain to genuinely *escape* the boolean
+ // domain at one of its *sinks* (see the HasEscapingSink scan below for the
+ // precise test). A pure boolean reduction whose every sink is an i1 consumer
+ // (G_BRCOND / G_SELECT condition) is deliberately left to the legalizer,
+ // which widens-and-fuses such chains optimally. Promoting them here instead
+ // restores the result with a `sext` + `icmp ne 0` round trip that is a no-op
+ // on a boolean (sext then "!= 0" is the identity). Some downstream consumers
+ // fold that pair away, but others do not -- where it survives, the round trip
+ // defeats `bfn`-with-flag fusion and emits extra scalar ops across many
+ // shaders. Keying the gate on the sinks (rather than on whether some internal
+ // node happens to be integer-typed) keeps the unified strategy where it helps
+ // -- vector reductions, genuine integer escapes, and the round-trip-free
+ // eq/ne operand-redirect fast path -- without regressing pure boolean chains.
+ SmallVector<MachineInstr *, 32> ToRewriteDefs;
+ bool HasVectorExtract = false;
+ unsigned NumBitwise = 0;
+ for (auto &E : Cache) {
+ if (E.second != Yes)
+ continue;
+ MachineInstr *Def = MRI.getVRegDef(E.first);
+ assert(Def);
+ ToRewriteDefs.push_back(Def);
+ switch (Def->getOpcode()) {
+ case TargetOpcode::G_XOR: {
+ // A NOT (xor with a boolean constant) is not a real reduction: promoting
+ // it just materializes an explicit `not` that the comparison-inverting
+ // combiner would otherwise fold into the cmp predicate (e.g. slt -> sge).
+ // Only count xors of two non-constant predicate values toward
+ // profitability. (Once promotion is decided by genuine reduction ops, a
+ // NOT inside the chain is still rewritten and is absorbed by `bfn`.)
+ Register X = Def->getOperand(1).getReg();
+ Register Y = Def->getOperand(2).getReg();
+ if (MRI.getVRegDef(X)->getOpcode() != TargetOpcode::G_CONSTANT &&
+ MRI.getVRegDef(Y)->getOpcode() != TargetOpcode::G_CONSTANT)
+ ++NumBitwise;
+ break;
+ }
+ case TargetOpcode::G_AND:
+ case TargetOpcode::G_OR:
+ ++NumBitwise;
+ break;
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT:
+ HasVectorExtract = true;
+ break;
+ default:
+ break;
+ }
+ }
+
+ // Decide whether the chain genuinely escapes the boolean domain by examining
+ // its *sinks* (uses outside the promote set, plus the kept-alive cmps that
+ // stay live). A sink "escapes" iff promoting it does not introduce the
+ // sext + `icmp ne 0` round trip:
+ // - integer sink: a non-bool consumer reading a promoted value at width
+ // > 1 (restored via trunc/sext/and, i.e. consumed as a number), or
+ // - clean eq/ne sink: a G_ICMP eq/ne both of whose operands are promoted,
+ // which is redirected to the s32 forms with no restore at all.
+ // Any other out-of-set consumer of a bool-typed promoted value is a
+ // *predicate sink* (G_BRCOND, G_SELECT condition, an `icmp ==/!= 0` flag
+ // test, an i1 store, ...) that must be fed an s1 reconstructed via the
+ // sext + `icmp ne/eq 0` round trip.
+ //
+ // Vector extracts are tracked separately above. A chain whose every sink is
+ // a predicate sink has only round-trip restorations and is left unpromoted
+ // (the i1-sink case). A chain that has *both* an escaping sink and a
+ // predicate sink is also left unpromoted on the scalar path: promoting it
+ // would still emit the round-trip restore at the predicate sink, and that
+ // restore defeats the backend's `bfn`-with-flag fusion (a single
+ // `bfn3.(...)::eq ...,0x1` becomes `bfn2 ... + cmp ::eq 0`, i.e. +1 op per
+ // flag) -- the dominant residual regression after the i1-sink gate. The
+ // legalizer widens the integer escape at its boundary and the matcher keeps
+ // the fused flag, so deferring the whole mixed chain matches the
+ // pre-promotion baseline. Only chains whose every sink escapes (pure integer
+ // / clean eq-ne reductions, with no predicate consumer) are promoted on the
+ // scalar path.
+ bool HasEscapingSink = false;
+ bool HasPredicateSink = false;
+ for (auto &E : Cache) {
+ if (E.second != Yes)
+ continue;
+ Register R = E.first;
+ LLT RTy = MRI.getType(R);
+ if (RTy.isVector())
+ continue;
+ for (MachineOperand &U : MRI.use_nodbg_operands(R)) {
+ MachineInstr *UI = U.getParent();
+ // In-set users that will be erased are rewritten in lock-step; not sinks.
+ bool InSetEraseable = UI->getNumDefs() >= 1 &&
+ Cache.lookup(UI->getOperand(0).getReg()) == Yes &&
+ !isKeptAliveOpcode(UI->getOpcode());
+ if (InSetEraseable)
+ continue;
+ // Clean eq/ne sink: redirected to the s32 promoted forms, no round trip.
+ if (UI->getOpcode() == TargetOpcode::G_ICMP) {
+ auto Pred = (CmpInst::Predicate)UI->getOperand(1).getPredicate();
+ auto Qualifies = [&](Register X) { return Cache.lookup(X) == Yes; };
+ if ((Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
+ Qualifies(UI->getOperand(2).getReg()) &&
+ Qualifies(UI->getOperand(3).getReg())) {
+ HasEscapingSink = true;
+ continue;
+ }
+ }
+ // Integer sink: the value is consumed at width > 1, restored as a number
+ // rather than via the i1 sext + (!=0) round trip. isBoolType keys off
+ // getScalarSizeInBits, which (unlike getSizeInBits) does not assert on a
+ // typeless register-class operand such as an inline-asm sink.
+ if (!isBoolType(RTy)) {
+ HasEscapingSink = true;
+ continue;
+ }
+ // Bool-typed value consumed out of set by anything else: a predicate
+ // sink that needs the fusion-defeating round-trip restore.
+ HasPredicateSink = true;
+ }
+ }
+
+ if (!HasVectorExtract &&
+ (NumBitwise < 2 || !HasEscapingSink || HasPredicateSink))
+ return false;
+
+ LLVM_DEBUG(dbgs() << "PISALegalizePredicates: processing " << MF.getName()
+ << " (" << ToRewriteDefs.size() << " defs to rewrite)\n");
+
+ // -- Phase 3: materialize s32 forms for every promoted def, then rewrite
+ // each use.
+ for (MachineInstr *Def : ToRewriteDefs)
+ (void)getOrBuildPromoted(Def->getOperand(0).getReg());
+
+ // The set of s32 promoted-form registers. Used to recognise an eq/ne operand
+ // that an earlier restore iteration already redirected to its s32 form (such
+ // a register is a *value* in Promoted, not a key, so Promoted.count() alone
+ // would miss it).
+ DenseSet<Register> PromotedVals;
+ for (auto &P : Promoted)
+ PromotedVals.insert(P.second);
+
+ // Walk each promoted register's uses; rewrite out-of-set sinks via a
+ // restored s1 (or a trunc to their iN width). In-set users will be
+ // erased shortly, so their use of the original need not be touched.
+ //
+ // Only "eraseable" originals require this -- cmps/constants/copies remain
+ // alive in MIR and feed their sinks unchanged.
+ for (auto &E : Cache) {
+ if (E.second != Yes)
+ continue;
+ Register OrigReg = E.first;
+ Register NewReg = Promoted.lookup(OrigReg);
+ // The G_ICMP peephole can lazily classify additional constant operands
+ // as promotable when probing operand orientation. Such entries land in
+ // Cache as Yes but never go through getOrBuildPromoted (the peephole
+ // only needed the bool-ness flag). Skip them here.
+ if (!NewReg)
+ continue;
+
+ MachineInstr *OrigDef = MRI.getVRegDef(OrigReg);
+ assert(OrigDef);
+
+ // Skip kinds whose originals stay alive -- they keep feeding their sinks
+ // directly, so we don't need to insert restoration.
+ if (isKeptAliveOpcode(OrigDef->getOpcode()))
+ continue;
+
+ LLT OrigTy = MRI.getType(OrigReg);
+
+ // Vector sinks not supported in v1 (we bailed in isHandledUse).
+ if (OrigTy.isVector())
+ continue;
+
+ // Snapshot uses up front -- the iterator is invalidated by rewrites.
+ SmallVector<MachineOperand *, 8> Uses;
+ for (MachineOperand &U : MRI.use_nodbg_operands(OrigReg))
+ Uses.push_back(&U);
+
+ for (MachineOperand *U : Uses) {
+ MachineInstr *UI = U->getParent();
+ // An in-set user whose own opcode is *also* eraseable will be removed
+ // shortly; its new promoted form already reads the right value, so
+ // we can leave its original operand alone.
+ // An in-set user whose opcode is kept-alive (cmp/const/copy) keeps
+ // its original instruction in MIR and still references this orig.
+ // We must redirect that operand to a restored value so the orig
+ // becomes truly dead and the legalizer doesn't widen it.
+ bool InSetEraseable = UI->getNumDefs() >= 1 &&
+ Cache.lookup(UI->getOperand(0).getReg()) == Yes &&
+ !isKeptAliveOpcode(UI->getOpcode());
+ if (InSetEraseable)
+ continue;
+
+ // Special case: a kept-alive G_ICMP eq/ne both of whose compared operands
+ // are promoted. eq/ne is invariant under the {0,-1} vs {0,1}
+ // representation (both promoted forms are the canonical {0,-1}), so let
+ // the comparison read the s32 promoted forms directly. This avoids
+ // restoring each operand back to i1 -- which would cost a ucmp.ne plus a
+ // select per operand -- and keeps the cmp's i1 result feeding its sink.
+ if (UI->getOpcode() == TargetOpcode::G_ICMP) {
+ auto Pred = (CmpInst::Predicate)UI->getOperand(1).getPredicate();
+ // An operand qualifies if it is a promoted chain (a key of Promoted) or
+ // already its s32 promoted form (redirected in an earlier iteration).
+ auto Qualifies = [&](Register X) {
+ return Promoted.count(X) || PromotedVals.count(X);
+ };
+ if ((Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
+ Qualifies(UI->getOperand(2).getReg()) &&
+ Qualifies(UI->getOperand(3).getReg())) {
+ U->setReg(NewReg);
+ ++NumRestorations;
+ continue;
+ }
+ }
+
+ Register Restored;
+ if (OrigTy.getSizeInBits() == 1) {
+ Restored = restoreS1(NewReg, *UI);
+ } else {
+ // Restore the *original* integer value at this sink. The promoted form
+ // is always SEXT-domain ({0, -1}); the original value may be
+ // ZEXT-domain ({0, 1}) if its chain was rooted in zero-extends.
+ // Reconstruct exactly from the tracked magnitude domain:
+ // Zext: boolean -> G_ZEXT to width ({0, 1}),
+ // Sext: trunc/sext of the {0, -1} form (sign bits already correct).
+ Domain Dom = Domain::Sext;
+ bool OK = tryGetDomain(OrigReg, Dom);
+ assert(OK && "promoted reg must have an unambiguous magnitude domain");
+ (void)OK;
+ if (Dom == Domain::Zext) {
+ // ZEXT domain ({0, 1}): mask the low bit of the {0, -1} form and
+ // adapt to width. This is cheaper than materializing a predicate and
+ // selecting 1/0 -- on an 8-bit sink the predicate+select path expands
+ // to ucmp + sel + trunc, whereas `and , 1` keeps the value in a GP
+ // register and matches the legalizer's fused boolean-mask lowering.
+ setInsertPointBefore(*UI);
+ auto One = B.buildConstant(LLT::integer(32), 1);
+ Register Masked = MRI.createGenericVirtualRegister(LLT::integer(32));
+ B.buildAnd(Masked, NewReg, One.getReg(0));
+ unsigned W = OrigTy.getSizeInBits();
+ if (W == 32) {
+ Restored = Masked;
+ } else {
+ Restored = MRI.createGenericVirtualRegister(OrigTy);
+ if (W < 32)
+ B.buildTrunc(Restored, Masked);
+ else
+ B.buildZExt(Restored, Masked);
+ }
+ } else if (OrigTy.getSizeInBits() == 32) {
+ Restored = NewReg;
+ } else {
+ setInsertPointBefore(*UI);
+ Restored = MRI.createGenericVirtualRegister(OrigTy);
+ if (OrigTy.getSizeInBits() < 32)
+ B.buildTrunc(Restored, NewReg);
+ else
+ B.buildSExt(Restored, NewReg);
+ }
+ }
+ U->setReg(Restored);
+ LLVM_DEBUG(dbgs() << " Restored use in: " << *UI);
+ ++NumRestorations;
+ }
+ }
+
+ // Erase dead originals (skip the cmps -- they're still alive for s1 sinks
+ // and may also be DCEd by later passes if they end up unused).
+ eraseDeadOriginals();
+
+ return true;
+}
+
+void PISALegalizePredicatesImpl::eraseDeadOriginals() {
+ // Collect erasable candidates into a worklist. Iterate until no more
+ // instructions become dead (removing a def may free its operands).
+ SmallVector<Register, 16> Worklist;
+ for (auto &E : Cache) {
+ if (E.second != Yes)
+ continue;
+ Register R = E.first;
+ MachineInstr *Def = MRI.getVRegDef(R);
+ assert(Def && "every classified vreg has a def in SSA MIR");
+ unsigned Opc = Def->getOpcode();
+ if (Opc == TargetOpcode::G_ICMP || Opc == TargetOpcode::G_FCMP ||
+ Opc == TargetOpcode::G_CONSTANT)
+ continue;
+ Worklist.push_back(R);
+ }
+
+ bool Changed = true;
+ while (Changed) {
+ Changed = false;
+ for (unsigned I = 0; I < Worklist.size(); ++I) {
+ Register R = Worklist[I];
+ MachineInstr *Def = MRI.getVRegDef(R);
+ if (!Def)
+ continue;
+ if (!MRI.use_nodbg_empty(R))
+ continue;
+ LLVM_DEBUG(dbgs() << " Erasing dead original: " << *Def);
+ ++NumErased;
+ Def->eraseFromParent();
+ Changed = true;
+ }
+ }
+}
+
+} // end anonymous namespace
+
+// -----------------------------------------------------------------------------
+// Pass plumbing.
+// -----------------------------------------------------------------------------
+
+void PISALegalizePredicates::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool PISALegalizePredicates::runOnMachineFunction(MachineFunction &MF) {
+ if (skipFunction(MF.getFunction()))
+ return false;
+ // No FailedISel guard: this pass runs in addPreLegalizeMachineIR(), before
+ // instruction selection, so that property is never set at this stage.
+ if (MF.getTarget().getOptLevel() == CodeGenOptLevel::None)
+ return false;
+ PISALegalizePredicatesImpl Impl(MF);
+ return Impl.run();
+}
+
+char PISALegalizePredicates::ID = 0;
+INITIALIZE_PASS(PISALegalizePredicates, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+MachineFunctionPass *llvm::createPISALegalizePredicatesPass() {
+ return new PISALegalizePredicates();
+}
diff --git a/llvm/lib/Target/PISA/PISALegalizeSubregAccess.cpp b/llvm/lib/Target/PISA/PISALegalizeSubregAccess.cpp
new file mode 100644
index 0000000000000..7202a2c40268a
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISALegalizeSubregAccess.cpp
@@ -0,0 +1,435 @@
+//=== PISALegalizeSubregAccess.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Enforce restrictions on instructions that may not be able to handle
+// a given subreg access (as generated by e.g., register coalescing).
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISAMCInstLower.h"
+#include "PISASubtarget.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define DEBUG_TYPE "pisa-legalize-subreg-access"
+#define DEBUG_NAME "PISA legalize subreg access"
+
+using namespace llvm;
+
+namespace {
+
+class PISALegalizeSubregAccess : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISALegalizeSubregAccess();
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+} // end anonymous namespace
+
+char PISALegalizeSubregAccess::ID = 0;
+INITIALIZE_PASS(PISALegalizeSubregAccess, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+void PISALegalizeSubregAccess::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+PISALegalizeSubregAccess::PISALegalizeSubregAccess() : MachineFunctionPass(ID) {
+ initializePISALegalizeSubregAccessPass(*PassRegistry::getPassRegistry());
+}
+
+bool PISALegalizeSubregAccess::runOnMachineFunction(MachineFunction &MF) {
+ auto &ST = MF.getSubtarget<PISASubtarget>();
+ auto *TII = ST.getInstrInfo();
+ auto *TRI = ST.getRegisterInfo();
+ auto &MRI = MF.getRegInfo();
+ bool Changed = false;
+ SmallVector<MachineInstr *, 8> FnArgs;
+ SmallVector<MachineInstr *> CopyV3Insts;
+ SmallVector<MachineInstr *> LargeCopyInsts;
+ SmallVector<MachineInstr *> NonCopyXYInsts;
+ for (auto &MBB : MF) {
+ for (auto &MI : MBB) {
+ if (TII->isFunctionParamInstr(MI))
+ FnArgs.push_back(&MI);
+ else if (MI.isCopy()) {
+ auto &Dst = MI.getOperand(0);
+ auto &Src = MI.getOperand(1);
+ if (Dst.getReg().isVirtual() && Src.getReg().isVirtual()) {
+ auto *DstRC = TRI->getSubRegisterClass(MRI.getRegClass(Dst.getReg()),
+ Dst.getSubReg());
+ auto *SrcRC = TRI->getSubRegisterClass(MRI.getRegClass(Src.getReg()),
+ Src.getSubReg());
+ // If either subreg isn't directly named on its RC, this COPY isn't
+ // a candidate for V3 or large-copy legalization; skip classification.
+ if (DstRC && SrcRC) {
+ auto DstNumElts = TRI->getNumEltsFromRegClass(DstRC);
+ auto DstEltSize = TRI->getBitSizeFromRegClass(DstRC);
+ auto SrcNumElts = TRI->getNumEltsFromRegClass(SrcRC);
+ if (((DstNumElts == 3) && (Dst.getSubReg() == 0)) &&
+ ((SrcNumElts == 3) && (Src.getSubReg() == 0))) {
+ CopyV3Insts.push_back(&MI);
+ } else if ((DstNumElts * DstEltSize) > 128) {
+ LargeCopyInsts.push_back(&MI);
+ }
+ }
+ }
+ } else {
+ bool NeedsLegalization = false;
+ for (unsigned I = 0; I < MI.getNumOperands(); I++) {
+ auto Opnd = MI.getOperand(I);
+ if (!Opnd.isReg() || !Opnd.getSubReg())
+ continue;
+ auto Swizzle = TRI->getSwizzle(Opnd.getSubReg());
+ if (TRI->isSelectorSwizzle(Swizzle))
+ continue;
+
+ // Tied operands collapse to a single MC operand during lowering and
+ // lose their swizzle. A composite-swizzle tied operand therefore
+ // cannot be emitted directly, so keep legalizing it into an explicit
+ // copy.
+ if (Opnd.isTied()) {
+ NeedsLegalization = true;
+ break;
+ }
+
+ // Composite swizzle (.xy / .zw) on a use or def operand.
+ // It is legal to leave in place when the subregister's shape
+ // matches the instruction's expected operand register class:
+ // the operand is then a naturally nameable, contiguous
+ // sub-slice (e.g. %v3h.xy) that register coalescing produced
+ // and that the AsmPrinter emits directly. This holds for both
+ // reads and partial-register writes; the printer applies the
+ // operand's swizzle uniformly, so a partial-register write is
+ // representable. A shape mismatch (or a subreg class that is
+ // not directly nameable, i.e. getSubRegisterClass == nullptr)
+ // still needs legalization.
+ bool IsLegal = false;
+ if (Opnd.getReg().isVirtual()) {
+ auto *SuperRC = MRI.getRegClass(Opnd.getReg());
+ auto *SubRC = TRI->getSubRegisterClass(SuperRC, Opnd.getSubReg());
+ auto *ExpectedRC = TII->getRegClass(TII->get(MI.getOpcode()), I);
+ if (SubRC && ExpectedRC) {
+ IsLegal = (TRI->getNumEltsFromRegClass(SubRC) ==
+ TRI->getNumEltsFromRegClass(ExpectedRC)) &&
+ (TRI->getBitSizeFromRegClass(SubRC) ==
+ TRI->getBitSizeFromRegClass(ExpectedRC));
+ }
+ }
+ if (!IsLegal) {
+ NeedsLegalization = true;
+ break;
+ }
+ }
+ if (NeedsLegalization)
+ NonCopyXYInsts.push_back(&MI);
+ }
+ }
+ }
+
+ for (auto *MI : reverse(FnArgs)) {
+ auto &Dst = MI->getOperand(0);
+ unsigned Subreg = Dst.getSubReg();
+ if (Subreg == 0 && !TRI->isSpecialReg(Dst.getReg()))
+ continue;
+
+ Changed = true;
+
+ auto *RC = TII->getRegClass(TII->get(MI->getOpcode()), 0);
+ Register NewDstReg = MRI.createVirtualRegister(RC);
+
+ MachineInstr *CopyMI =
+ BuildMI(MF, MI->getDebugLoc(), TII->get(TargetOpcode::COPY))
+ .add(Dst)
+ .addReg(NewDstReg);
+
+ auto *InsertPt = FnArgs[FnArgs.size() - 1];
+
+ Dst.setReg(NewDstReg);
+ Dst.setSubReg(0);
+ Dst.setIsUndef(false);
+ MI->getParent()->insertAfter(InsertPt, CopyMI);
+ }
+
+ for (auto *MI : CopyV3Insts) {
+ // .v3.16b A = mov .v3.16b B
+ // => .v3.16b A.x = mov .v3.16b B.x
+ // => .v3.16b A.y = mov .v3.16b B.y
+ // => .v3.16b A.z = mov .v3.16b B.z
+ auto &Dst = MI->getOperand(0);
+ auto &Src = MI->getOperand(1);
+ auto *DstRC = TRI->getSubRegisterClass(MRI.getRegClass(Dst.getReg()),
+ Dst.getSubReg());
+ auto NumElts = TRI->getNumEltsFromRegClass(DstRC);
+ auto EltSize = TRI->getBitSizeFromRegClass(DstRC);
+
+ DebugLoc DL = MI->getDebugLoc();
+ auto DstReg = Dst.getReg();
+ auto SrcReg = Src.getReg();
+ for (unsigned I = 0; I < NumElts; I++) {
+ auto NewMI =
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY));
+ auto SubRegIdx = TRI->getSubRegIdx(EltSize, I);
+ auto Undef = (I == 0) ? RegState::Undef : RegState::NoFlags;
+ NewMI.addDef(DstReg, Undef, SubRegIdx);
+ NewMI.addReg(SrcReg, {}, SubRegIdx);
+ }
+ Changed = true;
+ MI->eraseFromParent();
+ }
+
+ for (auto *MI : LargeCopyInsts) {
+ // (#1).v?.64b A = mov .v?.64b B
+ // => .v?.64b A.x = mov .v?.64b B.x
+ // => .v?.64b A.y = mov .v?.64b B.y
+ // => ... possibly .z, .w
+ // (#2) .v2.64b A = mov .v4.32b B
+ // => .v2.32b Lo.x = mov .v4.32b B.x
+ // => .v2.32b Lo.y = mov .v4.32b B.y
+ // => .v2.32b Hi.x = mov .v4.32b B.z
+ // => .v2.32b Hi.y = mov .v4.32b B.w
+ // => .64b Lo64 = mov .v2.32b Lo
+ // => .64b Hi64 = mov .v2.32b Hi
+ // => .v2.64b A.x = Lo64
+ // => .v2.64b A.y = Hi64
+ // (#3) .v4.32b A = mov .v2.64b B
+ // => .64b Lo64 = mov .v2.64b B.x
+ // => .64b Hi64 = mov .v2.64b B.y
+ // => .v2.32b Lo = mov .64b Lo64
+ // => .v2.32b Hi = mov .64b Hi64
+ // => .v4.32b A.x = mov .v2.32b Lo.x
+ // => .v4.32b A.y = mov .v2.32b Lo.y
+ // => .v4.32b A.z = mov .v2.32b Lo.z
+ // => .v4.32b A.w = mov .v2.32b Lo.w
+ auto &Dst = MI->getOperand(0);
+ auto &Src = MI->getOperand(1);
+ auto *DstRC = TRI->getSubRegisterClass(MRI.getRegClass(Dst.getReg()),
+ Dst.getSubReg());
+ auto *SrcRC = TRI->getSubRegisterClass(MRI.getRegClass(Src.getReg()),
+ Src.getSubReg());
+ auto DstNumElts = TRI->getNumEltsFromRegClass(DstRC);
+ auto DstEltSize = TRI->getBitSizeFromRegClass(DstRC);
+ auto SrcNumElts = TRI->getNumEltsFromRegClass(SrcRC);
+ auto SrcEltSize = TRI->getBitSizeFromRegClass(SrcRC);
+
+ DebugLoc DL = MI->getDebugLoc();
+ auto DstReg = Dst.getReg();
+ auto SrcReg = Src.getReg();
+ if (SrcEltSize == DstEltSize) { // (#1)
+ if (DstNumElts > 4) {
+ // large vector support
+ assert(DstEltSize == 32);
+ if (MRI.def_empty(SrcReg)) {
+ BuildMI(*MI->getParent(), MI, DL,
+ TII->get(TargetOpcode::IMPLICIT_DEF))
+ .addDef(DstReg);
+ } else {
+ unsigned Opcode = 0;
+ switch (DstNumElts) {
+ case 5:
+ Opcode = PISA::extract_0_v5i32_v5i32_r;
+ break;
+ case 6:
+ Opcode = PISA::extract_0_v6i32_v6i32_r;
+ break;
+ case 7:
+ Opcode = PISA::extract_0_v7i32_v7i32_r;
+ break;
+ case 8:
+ Opcode = PISA::extract_0_v8i32_v8i32_r;
+ break;
+ case 16:
+ Opcode = PISA::extract_0_v16i32_v16i32_r;
+ break;
+ case 32:
+ Opcode = PISA::extract_0_v32i32_v32i32_r;
+ break;
+ case 64:
+ Opcode = PISA::extract_0_v64i32_v64i32_r;
+ break;
+ default:
+ assert(0 && "implement");
+ break;
+ }
+ BuildMI(*MI->getParent(), MI, DL, TII->get(Opcode))
+ .addDef(DstReg)
+ .addReg(SrcReg);
+ }
+ } else {
+ // copy using swizzle
+ for (unsigned I = 0; I < DstNumElts; I++) {
+ auto NewMI =
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY));
+ auto SubRegIdx = TRI->getSubRegIdx(DstEltSize, I);
+ auto Undef = (I == 0) ? RegState::Undef : RegState::NoFlags;
+ NewMI.addDef(DstReg, Undef, SubRegIdx);
+ NewMI.addReg(SrcReg, {}, SubRegIdx);
+ }
+ }
+ } else {
+ auto *SmallVecRC =
+ (DstEltSize > SrcEltSize)
+ ? TRI->getVectorRegClass(SrcNumElts / 2, SrcEltSize)
+ : TRI->getVectorRegClass(DstNumElts / 2, DstEltSize); // .v2.32b
+ auto *LargeEltRC =
+ (DstEltSize > SrcEltSize)
+ ? TRI->getSubRegisterClass(MRI.getRegClass(DstReg),
+ TRI->getSubRegIdx(DstEltSize, 0))
+ : TRI->getSubRegisterClass(
+ MRI.getRegClass(SrcReg),
+ TRI->getSubRegIdx(SrcEltSize, 0)); // .64b
+ Register LoReg = MRI.createVirtualRegister(SmallVecRC);
+ Register HiReg = MRI.createVirtualRegister(SmallVecRC);
+ Register Lo64Reg = MRI.createVirtualRegister(LargeEltRC);
+ Register Hi64Reg = MRI.createVirtualRegister(LargeEltRC);
+ if (DstEltSize > SrcEltSize) { // (#2)
+ auto IdX = TRI->getSubRegIdx(SrcEltSize, 0);
+ auto IdY = TRI->getSubRegIdx(SrcEltSize, 1);
+ auto IdZ = TRI->getSubRegIdx(SrcEltSize, 2);
+ auto IdW = TRI->getSubRegIdx(SrcEltSize, 3);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(LoReg, RegState::Undef, IdX)
+ .addReg(SrcReg, {}, IdX);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(LoReg, {}, IdY)
+ .addReg(SrcReg, {}, IdY);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(HiReg, RegState::Undef, IdX)
+ .addReg(SrcReg, {}, IdZ);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(HiReg, {}, IdY)
+ .addReg(SrcReg, {}, IdW);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(Hi64Reg)
+ .addReg(HiReg);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(Lo64Reg)
+ .addReg(LoReg);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(DstReg, RegState::Undef, TRI->getSubRegIdx(DstEltSize, 0))
+ .addReg(Lo64Reg);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(DstReg, {}, TRI->getSubRegIdx(DstEltSize, 1))
+ .addReg(Hi64Reg);
+ } else { // (#3)
+ auto IdX = TRI->getSubRegIdx(DstEltSize, 0);
+ auto IdY = TRI->getSubRegIdx(DstEltSize, 1);
+ auto IdZ = TRI->getSubRegIdx(DstEltSize, 2);
+ auto IdW = TRI->getSubRegIdx(DstEltSize, 3);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(Lo64Reg)
+ .addReg(SrcReg, {}, TRI->getSubRegIdx(SrcEltSize, 0));
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(Hi64Reg)
+ .addReg(SrcReg, {}, TRI->getSubRegIdx(SrcEltSize, 1));
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(LoReg)
+ .addReg(Lo64Reg);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(HiReg)
+ .addReg(Hi64Reg);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(DstReg, RegState::Undef, IdX)
+ .addReg(LoReg, {}, IdX);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(DstReg, {}, IdY)
+ .addReg(LoReg, {}, IdY);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(DstReg, {}, IdZ)
+ .addReg(HiReg, {}, IdX);
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(DstReg, {}, IdW)
+ .addReg(HiReg, {}, IdY);
+ }
+ }
+ Changed = true;
+ MI->eraseFromParent();
+ }
+
+ for (auto *MI : NonCopyXYInsts) {
+ // A = add B.xy, C.zw
+ // => mov B', B.xy
+ // => mov C', C.zw
+ // => A = add B', C'
+ DebugLoc DL = MI->getDebugLoc();
+ SmallVector<MachineOperand, 4> NewOperands;
+ for (unsigned I = 0; I < MI->getNumOperands(); I++) {
+ auto Opnd = MI->getOperand(I);
+ if (Opnd.isReg() && Opnd.getSubReg() &&
+ (!TRI->isSelectorSwizzle(TRI->getSwizzle(Opnd.getSubReg())))) {
+ auto SubReg = Opnd.getSubReg();
+ auto Reg = Opnd.getReg();
+ // Prefer the register class the instruction declares for this operand
+ // (always allocatable). Fall back to the operand's sub-register class;
+ // if that is a non-allocatable structural sub-class (e.g. the .zw half
+ // Reg16bx2H), substitute the equivalent allocatable vector class so the
+ // temporary register can be created.
+ auto *RC = TII->getRegClass(TII->get(MI->getOpcode()), I);
+ if (!RC) {
+ RC = TRI->getSubRegisterClass(MRI.getRegClass(Reg), SubReg);
+ if (RC && !RC->isAllocatable())
+ RC = TRI->getVectorRegClass(TRI->getNumEltsFromRegClass(RC),
+ TRI->getBitSizeFromRegClass(RC));
+ }
+ auto NewReg = MRI.createVirtualRegister(RC);
+ if (!Opnd.isDef() || Opnd.isTied())
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(NewReg)
+ .addReg(Reg, {}, SubReg);
+ NewOperands.push_back(MachineOperand::CreateReg(NewReg, Opnd.isDef()));
+ } else {
+ NewOperands.push_back(Opnd);
+ }
+ }
+ auto NewMI = BuildMI(*MI->getParent(), MI, DL, TII->get(MI->getOpcode()));
+ for (unsigned I = 0; I < MI->getNumOperands(); I++) {
+ auto OldOpnd = MI->getOperand(I);
+ auto IsTied = OldOpnd.isReg() && OldOpnd.isTied();
+ auto NewOpnd = NewOperands[I];
+ if (IsTied && (I >= MI->getNumDefs())) {
+ auto TiedIdx = MI->findTiedOperandIdx(I);
+ // for originally tied register, specifying same
+ // register number will generate 'tied-def' entry
+ NewMI.addReg(NewOperands[TiedIdx].getReg(), {},
+ NewOperands[TiedIdx].getSubReg());
+ } else {
+ NewMI.add(NewOpnd);
+ }
+ }
+ for (unsigned I = 0; I < MI->getNumOperands(); I++) {
+ auto Opnd = MI->getOperand(I);
+ if (Opnd.isReg() && Opnd.getSubReg() &&
+ (!TRI->isSelectorSwizzle(TRI->getSwizzle(Opnd.getSubReg())))) {
+ auto SubReg = Opnd.getSubReg();
+ auto Reg = Opnd.getReg();
+ auto NewReg = NewOperands[I].getReg();
+ if (Opnd.isDef()) {
+ auto Undef = (I == 0) ? RegState::Undef : RegState::NoFlags;
+ BuildMI(*MI->getParent(), MI, DL, TII->get(TargetOpcode::COPY))
+ .addDef(Reg, Undef, SubReg)
+ .addReg(NewReg);
+ }
+ }
+ }
+ Changed = true;
+ MI->eraseFromParent();
+ }
+ return Changed;
+}
+
+namespace llvm {
+FunctionPass *createPISALegalizeSubregAccess() {
+ return new PISALegalizeSubregAccess();
+}
+} // end namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAMarkConvergentNoMerge.cpp b/llvm/lib/Target/PISA/PISAMarkConvergentNoMerge.cpp
new file mode 100644
index 0000000000000..16041c31e978d
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAMarkConvergentNoMerge.cpp
@@ -0,0 +1,64 @@
+//===-- PISAMarkConvergentNoMerge.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Mark convergent instructions with the NoMerge flag to prevent the branch
+// folder from tail-merging blocks that contain them. Convergent instructions
+// must execute with specific thread convergence guarantees, and tail merging
+// can violate those guarantees by changing the control flow paths that reach
+// the instruction.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+
+#define DEBUG_TYPE "pisa-mark-convergent-no-merge"
+#define DEBUG_NAME "PISA mark convergent instructions as NoMerge"
+
+using namespace llvm;
+
+namespace {
+
+class PISAMarkConvergentNoMerge : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISAMarkConvergentNoMerge() : MachineFunctionPass(ID) {
+ initializePISAMarkConvergentNoMergePass(*PassRegistry::getPassRegistry());
+ }
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ if (MI.isConvergent() && !MI.getFlag(MachineInstr::NoMerge)) {
+ MI.setFlag(MachineInstr::NoMerge);
+ Changed = true;
+ }
+ }
+ }
+ return Changed;
+ }
+};
+
+} // end anonymous namespace
+
+char PISAMarkConvergentNoMerge::ID = 0;
+INITIALIZE_PASS(PISAMarkConvergentNoMerge, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+FunctionPass *llvm::createPISAMarkConvergentNoMerge() {
+ return new PISAMarkConvergentNoMerge();
+}
diff --git a/llvm/lib/Target/PISA/PISAOptimizeRedundantCopies.cpp b/llvm/lib/Target/PISA/PISAOptimizeRedundantCopies.cpp
new file mode 100644
index 0000000000000..b7e536dd70fdd
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAOptimizeRedundantCopies.cpp
@@ -0,0 +1,171 @@
+//=== PISAOptimizeRedundantCopies.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Remove redundant COPY operations generated as result of previous passes.
+//
+// B = COPY A
+// C = COPY B
+// => B = COPY A
+// => C = COPY A
+//
+// Subsequent DCE will eliminate 'B = COPY A' if B is no longer used.
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISAMCInstLower.h"
+#include "PISASubtarget.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define DEBUG_TYPE "pisa-optimize-redundant-copies"
+#define DEBUG_NAME "PISA optimize redundant copies"
+
+using namespace llvm;
+
+namespace {
+
+class PISAOptimizeRedundantCopies : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISAOptimizeRedundantCopies();
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+} // end anonymous namespace
+
+char PISAOptimizeRedundantCopies::ID = 0;
+INITIALIZE_PASS(PISAOptimizeRedundantCopies, DEBUG_TYPE, DEBUG_NAME, false,
+ false)
+
+// Erase RegMap entries for any register or subregister that is defined (written
+// to) by MI. This ensures that if a register or any of its subregisters is
+// redefined between COPYs, we do not propagate stale mappings. Example:
+// A = COPY B
+// B = INST ... // B is redefined here
+// C = COPY A
+// => cannot replace C = COPY B
+//
+// Also handles cases where a subregister of B is modified:
+// A = COPY B
+// B.sub_0 = INST ... // subregister of B is redefined here
+// C = COPY A
+// => cannot replace C = COPY B
+static void eraseRegMapEntriesForDef(
+ const MachineInstr &MI,
+ SmallDenseMap<std::pair<Register, unsigned>, std::pair<Register, unsigned>>
+ &RegMap) {
+ for (const MachineOperand &Op : MI.operands()) {
+ if (Op.isReg() && Op.isDef()) {
+ Register WrittenReg = Op.getReg();
+ RegMap.remove_if([WrittenReg](const auto &Entry) {
+ return Entry.second.first == WrittenReg;
+ });
+ }
+ }
+}
+
+// Erase RegMap entries for registers or subregisters that overlap with the
+// destination of MI. This handles cases where an instruction overwrites part of
+// a register, invalidating previous mappings. Example:
+// A = COPY B
+// A.sub_0 = INST C // overwrites part of A
+// D = COPY A
+// => cannot replace D = COPY B
+static void eraseRegMapEntriesForOverlap(
+ const MachineInstr &MI,
+ SmallDenseMap<std::pair<Register, unsigned>, std::pair<Register, unsigned>>
+ &RegMap) {
+ const static SmallDenseMap<unsigned, unsigned> OverLap = {
+ {PISA::sub8_0, PISA::sub8_xy}, {PISA::sub8_1, PISA::sub8_xy},
+ {PISA::sub8_2, PISA::sub8_zw}, {PISA::sub8_3, PISA::sub8_zw},
+ {PISA::sub16_0, PISA::sub16_xy}, {PISA::sub16_1, PISA::sub16_xy},
+ {PISA::sub16_2, PISA::sub16_zw}, {PISA::sub16_3, PISA::sub16_zw},
+ {PISA::sub32_0, PISA::sub32_xy}, {PISA::sub32_1, PISA::sub32_xy},
+ {PISA::sub32_2, PISA::sub32_zw}, {PISA::sub32_3, PISA::sub32_zw},
+ {PISA::sub64_0, PISA::sub64_xy}, {PISA::sub64_1, PISA::sub64_xy},
+ {PISA::sub64_2, PISA::sub64_zw}, {PISA::sub64_3, PISA::sub64_zw},
+ };
+
+ if (MI.getNumOperands() == 0 || !MI.getOperand(0).isReg())
+ return;
+
+ auto Dst = MI.getOperand(0);
+ auto DstReg = Dst.getReg();
+ auto DstSubReg = Dst.getSubReg();
+
+ auto EraseOverlapReg = [&RegMap](Register DstReg, unsigned DstSubReg) {
+ auto Key = std::make_pair(DstReg, DstSubReg);
+ RegMap.erase(Key);
+ };
+
+ if (DstSubReg) {
+ EraseOverlapReg(DstReg, DstSubReg);
+ if (auto It = OverLap.find(DstSubReg); It != OverLap.end())
+ EraseOverlapReg(DstReg, It->second); // overlap subreg
+ }
+ EraseOverlapReg(DstReg, 0); // overlap full reg
+}
+
+static void processInterveningInsts(
+ const MachineInstr &MI,
+ SmallDenseMap<std::pair<Register, unsigned>, std::pair<Register, unsigned>>
+ &RegMap) {
+ eraseRegMapEntriesForDef(MI, RegMap);
+ eraseRegMapEntriesForOverlap(MI, RegMap);
+}
+
+void PISAOptimizeRedundantCopies::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+PISAOptimizeRedundantCopies::PISAOptimizeRedundantCopies()
+ : MachineFunctionPass(ID) {
+ initializePISAOptimizeRedundantCopiesPass(*PassRegistry::getPassRegistry());
+}
+
+bool PISAOptimizeRedundantCopies::runOnMachineFunction(MachineFunction &MF) {
+ bool Changed = false;
+ for (auto &MBB : MF) {
+ SmallDenseMap<std::pair<Register, unsigned>, std::pair<Register, unsigned>>
+ RegMap;
+ for (auto &MI : make_early_inc_range(MBB)) {
+ processInterveningInsts(MI, RegMap);
+ if (!MI.isCopy())
+ continue;
+
+ auto Dst = MI.getOperand(0);
+ auto Opnd = MI.getOperand(1);
+ auto Key = std::make_pair(Opnd.getReg(), Opnd.getSubReg());
+ if (auto It = RegMap.find(Key); It != RegMap.end()) {
+ auto [SrcReg, SrcSubReg] = It->second;
+ MachineIRBuilder B(MI);
+ B.buildInstr(TargetOpcode::COPY)
+ .addDef(Dst.getReg(), getRegState(Dst), Dst.getSubReg())
+ .addUse(SrcReg, {}, SrcSubReg);
+ MI.eraseFromParent();
+ Changed = true;
+ continue;
+ }
+
+ auto DKey = std::make_pair(Dst.getReg(), Dst.getSubReg());
+ RegMap[DKey] = std::make_pair(Opnd.getReg(), Opnd.getSubReg());
+ }
+ }
+ return Changed;
+}
+
+namespace llvm {
+FunctionPass *createPISAOptimizeRedundantCopies() {
+ return new PISAOptimizeRedundantCopies();
+}
+} // end namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAOptimizeSubregAccess.cpp b/llvm/lib/Target/PISA/PISAOptimizeSubregAccess.cpp
new file mode 100644
index 0000000000000..7b86c34c2c782
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAOptimizeSubregAccess.cpp
@@ -0,0 +1,195 @@
+//=== PISAOptimizeSubregAccess.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Given two consecutive COPY operations utilizing same destination and source
+// registers (and differing in subregister index), attempt to combine them into
+// a single COPY operation, e.g.
+//
+// %v.sub16_2:regv4_16b = COPY %4.sub16_0:regv2_16b
+// %v.sub16_3:regv4_16b = COPY %4.sub16_1:regv2_16b
+// => undef %v.sub16_zw:regv4_16b = COPY %4:regv2_16b
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISAMCInstLower.h"
+#include "PISASubtarget.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define DEBUG_TYPE "pisa-optimize-subreg-access"
+#define DEBUG_NAME "PISA optimize subreg accesses"
+
+using namespace llvm;
+
+namespace {
+
+class PISAOptimizeSubregAccess : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISAOptimizeSubregAccess();
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+ int getCombineSubreg(unsigned, unsigned, unsigned, unsigned);
+};
+} // end anonymous namespace
+
+char PISAOptimizeSubregAccess::ID = 0;
+INITIALIZE_PASS(PISAOptimizeSubregAccess, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+void PISAOptimizeSubregAccess::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+PISAOptimizeSubregAccess::PISAOptimizeSubregAccess() : MachineFunctionPass(ID) {
+ initializePISAOptimizeSubregAccessPass(*PassRegistry::getPassRegistry());
+}
+
+int PISAOptimizeSubregAccess::getCombineSubreg(unsigned RegSize,
+ unsigned SubRegSize,
+ unsigned Idx0, unsigned Idx1) {
+ int NewIdx = -1; // invalid
+ switch (SubRegSize) {
+ case 8: {
+ if ((Idx0 == PISA::sub8_0) && (Idx1 == PISA::sub8_1))
+ NewIdx = PISA::sub8_xy;
+ if ((Idx0 == PISA::sub8_2) && (Idx1 == PISA::sub8_3))
+ NewIdx = PISA::sub8_zw;
+ } break;
+ case 16: {
+ if ((Idx0 == PISA::sub16_0) && (Idx1 == PISA::sub16_1))
+ NewIdx = PISA::sub16_xy;
+ if ((Idx0 == PISA::sub16_2) && (Idx1 == PISA::sub16_3))
+ NewIdx = PISA::sub16_zw;
+ } break;
+ case 32: {
+ if ((Idx0 == PISA::sub32_0) && (Idx1 == PISA::sub32_1))
+ NewIdx = PISA::sub32_xy;
+ if ((Idx0 == PISA::sub32_2) && (Idx1 == PISA::sub32_3))
+ NewIdx = PISA::sub32_zw;
+ } break;
+ case 64: {
+ if ((Idx0 == PISA::sub64_0) && (Idx1 == PISA::sub64_1))
+ NewIdx = PISA::sub64_xy;
+ if ((Idx0 == PISA::sub64_2) && (Idx1 == PISA::sub64_3))
+ NewIdx = PISA::sub64_zw;
+ } break;
+ default:
+ break;
+ }
+ if ((NewIdx > 0) && (SubRegSize * 2 == RegSize))
+ NewIdx = 0; // use full reg
+ return NewIdx;
+}
+
+bool PISAOptimizeSubregAccess::runOnMachineFunction(MachineFunction &MF) {
+ auto &ST = MF.getSubtarget<PISASubtarget>();
+ auto *TII = ST.getInstrInfo();
+ auto *TRI = ST.getRegisterInfo();
+ auto &MRI = MF.getRegInfo();
+
+ bool Changed = false;
+ SmallVector<MachineInstr *> DeleteMIs;
+ MachineInstr *LastMI = nullptr;
+ for (auto &MBB : MF) {
+ LastMI = nullptr;
+ for (auto &MI : MBB) {
+ if (!LastMI || !MI.isCopy() || !LastMI->isCopy()) {
+ LastMI = &MI;
+ continue;
+ }
+ auto &Dst = MI.getOperand(0);
+ auto DstReg = Dst.getReg();
+ auto DstSubreg = Dst.getSubReg();
+ if (!(DstReg.isVirtual() && DstSubreg)) {
+ LastMI = &MI;
+ continue;
+ }
+ auto LDst = LastMI->getOperand(0);
+ auto LDstReg = LDst.getReg();
+ auto LDstSubreg = LDst.getSubReg();
+ if (!(LDstReg.isVirtual() && LDstSubreg && (LDstReg == DstReg))) {
+ LastMI = &MI;
+ continue;
+ }
+ auto *DstRC =
+ TRI->getSubRegisterClass(MRI.getRegClass(DstReg), DstSubreg);
+ auto *LDstRC =
+ TRI->getSubRegisterClass(MRI.getRegClass(LDstReg), LDstSubreg);
+ if (!DstRC || !LDstRC) {
+ LastMI = &MI;
+ continue;
+ }
+ auto DstRegSize = TRI->getRegSizeInBits(*MRI.getRegClass(DstReg));
+ auto DstSubRegSize = TRI->getSubRegIdxSize(DstSubreg);
+ auto LDstSubRegSize = TRI->getSubRegIdxSize(LDstSubreg);
+ if ((DstSubRegSize + LDstSubRegSize) > 128) { // exceed max 'mov' size
+ LastMI = &MI;
+ continue;
+ }
+ auto NewDstIdx =
+ getCombineSubreg(DstRegSize, DstSubRegSize, LDstSubreg, DstSubreg);
+ if (NewDstIdx >= 0) {
+ auto &Src = MI.getOperand(1);
+ auto SrcReg = Src.getReg();
+ auto SrcSubreg = Src.getSubReg();
+ if (!(SrcReg.isVirtual() && SrcSubreg)) {
+ LastMI = &MI;
+ continue;
+ }
+ auto LSrc = LastMI->getOperand(1);
+ auto LSrcReg = LSrc.getReg();
+ auto LSrcSubreg = LSrc.getSubReg();
+ if (!(LSrcReg.isVirtual() && LSrcSubreg && (LSrcReg == SrcReg))) {
+ LastMI = &MI;
+ continue;
+ }
+ auto *SrcRC =
+ TRI->getSubRegisterClass(MRI.getRegClass(SrcReg), SrcSubreg);
+ if (!SrcRC) {
+ LastMI = &MI;
+ continue;
+ }
+ auto SrcRegSize = TRI->getRegSizeInBits(*MRI.getRegClass(SrcReg));
+ auto SrcSubRegSize = TRI->getSubRegIdxSize(SrcSubreg);
+ auto NewSrcIdx =
+ getCombineSubreg(SrcRegSize, SrcSubRegSize, LSrcSubreg, SrcSubreg);
+ if (NewSrcIdx >= 0) {
+ DebugLoc DL = MI.getDebugLoc();
+ auto NewMI =
+ BuildMI(*MI.getParent(), MI, DL, TII->get(TargetOpcode::COPY));
+ auto DstUndef = NewDstIdx == 0 ? RegState::NoFlags : RegState::Undef;
+ auto SrcUndef = (Src.isUndef() && LSrc.isUndef()) ? RegState::Undef
+ : RegState::NoFlags;
+ NewMI.addDef(DstReg, DstUndef, NewDstIdx);
+ NewMI.addReg(SrcReg, SrcUndef, NewSrcIdx);
+ DeleteMIs.push_back(LastMI);
+ DeleteMIs.push_back(&MI);
+ Changed = true;
+ LastMI = nullptr;
+ continue;
+ }
+ }
+ LastMI = &MI;
+ }
+ }
+ for (auto *MI : DeleteMIs)
+ MI->eraseFromParent();
+ return Changed;
+}
+
+namespace llvm {
+FunctionPass *createPISAOptimizeSubregAccess() {
+ return new PISAOptimizeSubregAccess();
+}
+} // end namespace llvm
diff --git a/llvm/lib/Target/PISA/PISAPropagateNullPointers.cpp b/llvm/lib/Target/PISA/PISAPropagateNullPointers.cpp
new file mode 100644
index 0000000000000..9b9a200f88045
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAPropagateNullPointers.cpp
@@ -0,0 +1,171 @@
+//===-- PISAPropagateNullPointers.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass ensures null pointers remain null when cast between address
+// spaces, handling both direct addrspacecast instructions and constant
+// expressions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISASubtarget.h"
+#include "PISATargetMachine.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+using namespace llvm;
+using namespace llvm::PISA;
+using namespace llvm::PISAAS;
+
+#define DEBUG_TYPE "pisa-propagate-null-pointers"
+#define DEBUG_NAME "PISA propagate null pointers"
+
+namespace {
+
+class PISAPropagateNullPointers : public ModulePass {
+public:
+ static char ID;
+
+ PISAPropagateNullPointers() : ModulePass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+ bool runOnModule(Module &M) override;
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ ModulePass::getAnalysisUsage(AU);
+ }
+};
+
+} // namespace
+
+char PISAPropagateNullPointers::ID = 0;
+INITIALIZE_PASS(PISAPropagateNullPointers, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+// Creates a null pointer for the specified address space without using
+// ConstantPointerNull::get, as LLVM assumes null pointers have a zero-bit
+// representation regardless of address space which can lead to incorrect code.
+static Constant *createNullPtr(PointerType *PtrTy, const DataLayout &DL) {
+ unsigned AS = PtrTy->getAddressSpace();
+ unsigned BitSize = DL.getPointerSizeInBits(AS);
+ return ConstantExpr::getIntToPtr(
+ ConstantInt::get(Type::getIntNTy(PtrTy->getContext(), BitSize),
+ PISATargetMachine::getNullPointerValue(AS), true),
+ PtrTy);
+}
+
+static bool isPtrKnownNonNull(const Value *Src, const DataLayout &DL) {
+ if (isa<GlobalValue, AllocaInst>(Src))
+ return true;
+
+ if (const auto *Arg = dyn_cast<Argument>(Src)) {
+ if (Arg->hasNonNullAttr())
+ return true;
+ if (Arg->getParent()->getCallingConv() == CallingConv::PISA_KERNEL) {
+ unsigned AS = Arg->getType()->getPointerAddressSpace();
+ if (AS == static_cast<unsigned>(AddressSpace::PRIVATE) ||
+ AS == static_cast<unsigned>(AddressSpace::SHARED))
+ return true;
+ }
+ }
+
+ return isKnownNonZero(Src, DL);
+}
+
+// Only casts between generic and shared/private address spaces are processed.
+// Casts between global and generic address spaces are skipped since they do not
+// change the pointer's bit representation.
+static bool isCandidate(const AddrSpaceCastInst &ASC) {
+ if (ASC.getType()->isVectorTy())
+ return false;
+
+ const unsigned SrcAS = ASC.getSrcAddressSpace();
+ const unsigned DstAS = ASC.getDestAddressSpace();
+
+ constexpr unsigned PrivateAS = static_cast<unsigned>(AddressSpace::PRIVATE);
+ constexpr unsigned SharedAS = static_cast<unsigned>(AddressSpace::SHARED);
+ constexpr unsigned GenericAS = static_cast<unsigned>(AddressSpace::GENERIC);
+
+ if (SrcAS == GenericAS && (DstAS == PrivateAS || DstAS == SharedAS))
+ return true;
+ if ((SrcAS == PrivateAS || SrcAS == SharedAS) && DstAS == GenericAS)
+ return true;
+ return false;
+}
+
+static bool processASC(AddrSpaceCastInst &ASC, const DataLayout &DL) {
+ if (!isCandidate(ASC))
+ return false;
+
+ auto *Src = ASC.getPointerOperand();
+ SmallVector<const Value *, 4> WorkList;
+ getUnderlyingObjects(Src, WorkList);
+ if (all_of(WorkList,
+ [&DL](const Value *V) { return isPtrKnownNonNull(V, DL); }))
+ return false;
+
+ auto *SrcNull = createNullPtr(cast<PointerType>(Src->getType()), DL);
+ auto *DstNull = createNullPtr(cast<PointerType>(ASC.getType()), DL);
+
+ IRBuilder<> IRB(&ASC);
+ auto *ASCCopy = IRB.CreateAddrSpaceCast(Src, ASC.getType(), ASC.getName());
+ auto *IsNonNull = IRB.CreateICmpNE(Src, SrcNull);
+ auto *Select = IRB.CreateSelect(IsNonNull, ASCCopy, DstNull);
+ ASC.replaceAllUsesWith(Select);
+ ASC.eraseFromParent();
+ return true;
+}
+
+// Replaces Clang-generated constant expression casts from generic null pointers
+// to shared/private address spaces with inttoptr expressions. Only casts from
+// generic to shared/private address spaces are processed.
+static bool updateConstExprCasts(LLVMContext &Ctx, const DataLayout &DL) {
+ auto *NullGeneric = ConstantPointerNull::get(
+ PointerType::get(Ctx, static_cast<unsigned>(AddressSpace::GENERIC)));
+ auto *NullPrivate = ConstantPointerNull::get(
+ PointerType::get(Ctx, static_cast<unsigned>(AddressSpace::PRIVATE)));
+ auto *NullShared = ConstantPointerNull::get(
+ PointerType::get(Ctx, static_cast<unsigned>(AddressSpace::SHARED)));
+
+ auto *GenericToPrivateCast =
+ ConstantExpr::getAddrSpaceCast(NullGeneric, NullPrivate->getType());
+ auto *GenericToSharedCast =
+ ConstantExpr::getAddrSpaceCast(NullGeneric, NullShared->getType());
+
+ bool Changed =
+ !GenericToPrivateCast->use_empty() || !GenericToSharedCast->use_empty();
+ GenericToPrivateCast->replaceAllUsesWith(
+ createNullPtr(cast<PointerType>(NullPrivate->getType()), DL));
+ GenericToSharedCast->replaceAllUsesWith(
+ createNullPtr(cast<PointerType>(NullShared->getType()), DL));
+ return Changed;
+}
+
+bool PISAPropagateNullPointers::runOnModule(Module &M) {
+ bool Changed = false;
+ for (auto &F : M)
+ for (auto &I : make_early_inc_range(instructions(F)))
+ if (auto *ASC = dyn_cast<AddrSpaceCastInst>(&I))
+ Changed |= processASC(*ASC, M.getDataLayout());
+
+ Changed |= updateConstExprCasts(M.getContext(), M.getDataLayout());
+ return Changed;
+}
+
+ModulePass *llvm::createPISAPropagateNullPointersPass() {
+ return new PISAPropagateNullPointers();
+}
diff --git a/llvm/lib/Target/PISA/PISAReplaceIntrinsics.cpp b/llvm/lib/Target/PISA/PISAReplaceIntrinsics.cpp
new file mode 100644
index 0000000000000..13f16fb594f61
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAReplaceIntrinsics.cpp
@@ -0,0 +1,99 @@
+//===-- PISAReplaceIntrinsics.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass runs just after IRTranslator. It will replace some of intrinsics
+// with an equivalent GMIR opcode, so that subsequent optimizations can be made.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISAInstrInfo.h"
+#include "PISATargetMachine.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define DEBUG_TYPE "pisa-replace-intrinsics"
+#define DEBUG_NAME "PISA replace intrinsics"
+
+using namespace llvm;
+
+namespace {
+
+class PISAReplaceIntrinsics : public MachineFunctionPass {
+
+public:
+ static char ID;
+ PISAReplaceIntrinsics() : MachineFunctionPass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+private:
+ const PISAInstrInfo *TII = nullptr;
+};
+
+} // namespace
+
+char PISAReplaceIntrinsics::ID = 0;
+INITIALIZE_PASS(PISAReplaceIntrinsics, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+bool PISAReplaceIntrinsics::runOnMachineFunction(MachineFunction &MF) {
+ TII = MF.getSubtarget<PISASubtarget>().getInstrInfo();
+ bool Changed = false;
+
+ SmallVector<MachineInstr *, 8> Delete;
+ for (auto &MBB : MF) {
+ for (auto &MI : MBB) {
+ if (MI.getOpcode() == TargetOpcode::G_INTRINSIC) {
+ auto ID = cast<GIntrinsic>(MI).getIntrinsicID();
+ switch (ID) {
+ case Intrinsic::pisa_sbfe:
+ case Intrinsic::pisa_ubfe: {
+ // Res = pisa_[su]bfe (Base,Width,Offset)
+ // Res = G_[SU]BFX (Base,LSB,Width)
+ auto Opcode = (ID == Intrinsic::pisa_sbfe) ? TargetOpcode::G_SBFX
+ : TargetOpcode::G_UBFX;
+ DebugLoc DL = MI.getDebugLoc();
+ auto &Dst = MI.getOperand(0);
+ BuildMI(MBB, &MI, DL, TII->get(Opcode))
+ .addDef(Dst.getReg()) // Operand(0) is actual intrinsic
+ .add(MI.getOperand(2))
+ .add(MI.getOperand(4))
+ .add(MI.getOperand(3));
+ Delete.push_back(&MI);
+ Changed = true;
+ } break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+ for (auto *MI : Delete) {
+ MI->eraseFromParent();
+ }
+
+ return Changed;
+}
+
+FunctionPass *llvm::createPISAReplaceIntrinsicsPass() {
+ return new PISAReplaceIntrinsics();
+}
diff --git a/llvm/lib/Target/PISA/PISAScopeSelector.cpp b/llvm/lib/Target/PISA/PISAScopeSelector.cpp
new file mode 100644
index 0000000000000..45899737a1b7b
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAScopeSelector.cpp
@@ -0,0 +1,112 @@
+//===-- PISAScopeSelector.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// syncscope("<target-scope>") support for atomics
+// - atomic operations take a 'scope' argument (part of MachineMemOperand)
+// - PISA instructions define $scope input used in instruction printing
+// - this pass extract 'scope' from MI and encodes associated value into $scope
+#include "MCTargetDesc/PISAInstPrinter.h"
+#include "PISA.h"
+
+#define GET_INSTRINFO_OPERAND_ENUM
+#include "PISAGenInstrInfo.inc"
+#define GET_AtomicScopeControl_DECL
+#include "PISAGenSearchableTables.inc"
+
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/IR/DiagnosticInfo.h"
+
+#define DEBUG_TYPE "pisa-scope-selector"
+#define DEBUG_NAME "PISA Scope Selector"
+
+using namespace llvm;
+using namespace llvm::PISA;
+
+namespace {
+class PISAScopeSelector : public llvm::MachineFunctionPass {
+public:
+ static char ID;
+
+ PISAScopeSelector() : MachineFunctionPass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+ bool runOnMachineFunction(MachineFunction &MF) override;
+};
+} // namespace
+
+char PISAScopeSelector::ID = 0;
+INITIALIZE_PASS(PISAScopeSelector, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+MachineFunctionPass *llvm::createPISAScopeSelectorPass() {
+ return new PISAScopeSelector();
+}
+
+void PISAScopeSelector::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+namespace {
+const StringMap<unsigned> ScopeName2Encoding = {
+ {"workgroup", AtomicScopeControl_WORKGROUP},
+ {"gpu", AtomicScopeControl_GPU},
+ {"system", AtomicScopeControl_SYSTEM},
+ // using workgroup scope
+ {"subgroup", AtomicScopeControl_WORKGROUP},
+ {"workitem", AtomicScopeControl_WORKGROUP},
+};
+} // namespace
+
+bool PISAScopeSelector::runOnMachineFunction(MachineFunction &MF) {
+ // SyncScopeID are dynamically assigned during parsing, so
+ // we need to map them back to AtomicScopeControl definitions
+ auto &Ctx = MF.getFunction().getContext();
+ DenseMap<SyncScope::ID, unsigned> ScopeID2Encoding;
+ for (const auto &[Name, Encoding] : ScopeName2Encoding) {
+ auto ID = Ctx.getOrInsertSyncScopeID(Name);
+ ScopeID2Encoding.emplace_or_assign(ID, Encoding);
+ }
+ ScopeID2Encoding.emplace_or_assign(SyncScope::System,
+ AtomicScopeControl_SYSTEM);
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (MachineInstr &MI : MBB) {
+ if (MI.memoperands_empty())
+ continue;
+
+ auto *MMO = *MI.memoperands_begin();
+ if (!(MMO->isLoad() || MMO->isStore()))
+ continue;
+
+ auto Ordering = MMO->getSuccessOrdering();
+ // Catch inconsistent atomic ordering
+ if (!isValidAtomicOrdering(static_cast<unsigned>(Ordering))) {
+ MI.emitGenericError("invalid atomic ordering in MachineMemOperand");
+ continue;
+ }
+
+ if (!isStrongerThanMonotonic(Ordering))
+ continue;
+
+ auto OpName = PISA::OpName::scope;
+ auto OpIdx = PISA::getNamedOperandIdx(MI.getOpcode(), OpName);
+ if (OpIdx == -1)
+ continue;
+
+ if (MI.getOperand(OpIdx).getImm() != AtomicScopeControl_NONE)
+ continue; // skip if already set (pisa2pisa)
+
+ auto ScopeID = MMO->getSyncScopeID();
+ auto Entry = ScopeID2Encoding.find(ScopeID);
+ if (Entry == ScopeID2Encoding.end())
+ llvm_unreachable("unsupported syncscope");
+ MI.getOperand(OpIdx).setImm(Entry->second);
+ }
+ }
+ return false;
+}
diff --git a/llvm/lib/Target/PISA/PISATargetMachine.cpp b/llvm/lib/Target/PISA/PISATargetMachine.cpp
index 9f1d150394055..0721f49d171de 100644
--- a/llvm/lib/Target/PISA/PISATargetMachine.cpp
+++ b/llvm/lib/Target/PISA/PISATargetMachine.cpp
@@ -13,6 +13,7 @@
#include "TargetInfo/PISATargetInfo.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/DeadMachineInstructionElim.h"
#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
#include "llvm/CodeGen/GlobalISel/Legalizer.h"
@@ -25,11 +26,18 @@
#include "llvm/IR/IntrinsicsPISA.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/PISAAddrSpace.h"
#include "llvm/Target/TargetOptions.h"
+#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
using namespace llvm;
+static cl::opt<bool>
+ EnableLoadStoreVectorizer("pisa-load-store-vectorizer",
+ cl::desc("Enable load store vectorizer"),
+ cl::init(true), cl::Hidden);
+
// NOLINTNEXTLINE(readability-identifier-naming)
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePISATarget() {
// Register the target.
@@ -37,8 +45,25 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePISATarget() {
PassRegistry &PR = *PassRegistry::getPassRegistry();
initializeGlobalISel(PR);
+ initializePISALegalizeCallsPass(PR);
+ initializePISAEmitIntrinsicsPass(PR);
+ initializePISAExpandIntrinsicsPass(PR);
+ initializePISALegalizeSubregAccessPass(PR);
initializePISAPreLegalizerCombinerPass(PR);
initializePISAPostLegalizerCombinerPass(PR);
+ initializePISALegalizePredicatesPass(PR);
+ initializePISAReplaceIntrinsicsPass(PR);
+ initializePISAPropagateNullPointersPass(PR);
+ initializePISACacheHintSelectorPass(PR);
+ initializePISAVerifierPass(PR);
+ initializePISAOptimizeRedundantCopiesPass(PR);
+ initializePISAOptimizeSubregAccessPass(PR);
+ initializePISAInsertLifetimeStartPass(PR);
+ initializePISAMarkConvergentNoMergePass(PR);
+ initializePISAScopeSelectorPass(PR);
+ initializePISAVerifyTypesPass(PR);
+ initializePISAKernelByValArgsLoweringLegacyPass(PR);
+ initializePISALayoutPass(PR);
}
MachineFunctionInfo *PISATargetMachine::createMachineFunctionInfo(
@@ -135,82 +160,157 @@ namespace {
// PISA Code Generator Pass Configuration Options.
//
// PISA is a virtual-register-only target: it maintains virtual registers
-// throughout the pipeline and does not run register allocation. This
-// configuration wires up the GlobalISel selection stages and disables the
-// standard machine passes that assume physical registers exist.
+// throughout the pipeline and does not run register allocation.
class PISAPassConfig : public TargetPassConfig {
public:
PISAPassConfig(PISATargetMachine &TM, PassManagerBase &PM)
: TargetPassConfig(TM, PM) {
disablePass(&GCLoweringID);
disablePass(&ShadowStackGCLoweringID);
+ disablePass(&XRayInstrumentationID);
}
PISATargetMachine &getPISATargetMachine() const {
return getTM<PISATargetMachine>();
}
- void addIRPasses() override {
- TargetPassConfig::addIRPasses();
-
- // Disable passes that assume physical registers exist.
- disablePass(&PrologEpilogCodeInserterID);
- disablePass(&MachineLateInstrsCleanupID);
- disablePass(&MachineCopyPropagationID);
- disablePass(&TailDuplicateLegacyID);
- disablePass(&StackMapLivenessID);
- disablePass(&LiveDebugValuesID);
- disablePass(&PostRAMachineSinkingID);
- disablePass(&PostRASchedulerID);
- disablePass(&FuncletLayoutID);
- disablePass(&PatchableFunctionID);
- disablePass(&ShrinkWrapID);
- disablePass(&RemoveLoadsIntoFakeUsesID);
- disablePass(&GCMachineCodeAnalysisID);
- }
+ void addIRPasses() override;
+ void addISelPrepare() override;
+ bool addIRTranslator() override;
+ void addPreLegalizeMachineIR() override;
+ bool addLegalizeMachineIR() override;
+ void addPreRegBankSelect() override;
+ bool addRegBankSelect() override;
+ bool addGlobalInstructionSelect() override;
+ FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
+ bool addRegAssignAndRewriteFast() override { return false; }
+ bool addRegAssignAndRewriteOptimized() override { return false; }
+ void addPreRegAlloc() override;
+ void addPostRegAlloc() override;
+ void addPreEmitPass() override;
+};
+} // namespace
- bool addIRTranslator() override {
- addPass(new IRTranslator(getOptLevel()));
- return false;
- }
+void PISAPassConfig::addIRPasses() {
+ addPass(createPISAVerifierPass());
- void addPreLegalizeMachineIR() override {
- if (getOptLevel() != CodeGenOptLevel::None)
- addPass(createPISAPreLegalizerCombiner());
- }
+ // Legalize atomics with LLVM's AtomicExpandPass, driven by the
+ // PISATargetLowering atomic hooks. Keep it first in addIRPasses().
+ addPass(createAtomicExpandLegacyPass());
- bool addLegalizeMachineIR() override {
- addPass(new Legalizer());
- return false;
- }
+ TargetPassConfig::addIRPasses();
+
+ addPass(createPISAPropagateNullPointersPass());
+ addPass(createPISAKernelByValArgsLoweringLegacyPass());
+ addPass(createPISAExpandIntrinsicsPass());
+ addPass(createPISALegalizeCallsPass());
+
+ if ((getOptLevel() != CodeGenOptLevel::None) && EnableLoadStoreVectorizer)
+ addPass(createLoadStoreVectorizerPass());
+
+ // A temporary solution to prevent divergent barrier calls.
+ addPass(createPISALayoutPass());
+
+ // Disable passes that assume physical registers exist.
+ disablePass(&PrologEpilogCodeInserterID);
+ disablePass(&MachineLateInstrsCleanupID);
+ disablePass(&MachineCopyPropagationID);
+ disablePass(&TailDuplicateLegacyID);
+ disablePass(&StackMapLivenessID);
+ disablePass(&LiveDebugValuesID);
+ disablePass(&PostRAMachineSinkingID);
+ disablePass(&PostRASchedulerID);
+ disablePass(&FuncletLayoutID);
+ disablePass(&PatchableFunctionID);
+ disablePass(&ShrinkWrapID);
+ disablePass(&RemoveLoadsIntoFakeUsesID);
+ disablePass(&GCMachineCodeAnalysisID);
+}
+
+void PISAPassConfig::addISelPrepare() {
+ addPass(createPISAEmitIntrinsicsPass());
+ TargetPassConfig::addISelPrepare();
+}
+
+bool PISAPassConfig::addIRTranslator() {
+ addPass(new IRTranslator(getOptLevel()));
+ addPass(createPISAVerifyTypesPass());
+ addPass(createPISAReplaceIntrinsicsPass());
+ return false;
+}
- void addPreRegBankSelect() override {
- if (getOptLevel() != CodeGenOptLevel::None) {
- addPass(&MachineCSELegacyID);
- addPass(createPISAPostLegalizerCombiner());
- }
+void PISAPassConfig::addPreLegalizeMachineIR() {
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ addPass(createPISALegalizePredicatesPass());
+ addPass(createPISAPreLegalizerCombiner());
+ addPass(createPISAVerifyTypesPass());
}
+}
+
+bool PISAPassConfig::addLegalizeMachineIR() {
+ addPass(new Legalizer());
+ addPass(createPISAVerifyTypesPass());
+ return false;
+}
- bool addRegBankSelect() override {
- addPass(new RegBankSelect());
- return false;
+void PISAPassConfig::addPreRegBankSelect() {
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ addPass(&MachineCSELegacyID);
+ addPass(createPISAPostLegalizerCombiner());
+ addPass(createPISAVerifyTypesPass());
}
+}
+
+bool PISAPassConfig::addRegBankSelect() {
+ addPass(new RegBankSelect());
+ return false;
+}
+
+bool PISAPassConfig::addGlobalInstructionSelect() {
+ // The MachineCSE pass doesn't detect common subexpressions on instructions
+ // using IMPLICIT_DEF instructions. These are sometimes inserted by
+ // InstructionSelect, so we run MachineCSE before that to ensure good CSE.
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(&MachineCSELegacyID);
+
+ addPass(createPISAVerifyTypesPass());
+ addPass(new InstructionSelect());
+ addPass(createPISAScopeSelectorPass());
+ addPass(createPISACacheHintSelectorPass());
+ // G_BUILD_VECTOR will produce IMPLICIT_DEFS that must be removed.
+ if (getOptLevel() == CodeGenOptLevel::None)
+ addPass(&ProcessImplicitDefsID);
+
+ return false;
+}
- bool addGlobalInstructionSelect() override {
- if (getOptLevel() != CodeGenOptLevel::None)
- addPass(&MachineCSELegacyID);
- addPass(new InstructionSelect());
- if (getOptLevel() == CodeGenOptLevel::None)
- addPass(&ProcessImplicitDefsID);
- return false;
+void PISAPassConfig::addPreRegAlloc() {
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(&LiveRangeShrinkID);
+ TargetPassConfig::addPreRegAlloc();
+}
+
+void PISAPassConfig::addPostRegAlloc() {
+ addPass(createPISALegalizeSubregAccess());
+ if (getOptLevel() != CodeGenOptLevel::None) {
+ addPass(createPISAOptimizeSubregAccess());
+ addPass(createPISAOptimizeRedundantCopies());
+ addPass(&DeadMachineInstructionElimID);
}
+ addPass(createPISAMarkConvergentNoMerge());
+ // The machine block placement pass is able to rearrange blocks in a way that
+ // breaks control flow for kernels with disabled IFP.
+ disablePass(&MachineBlockPlacementID);
+ TargetPassConfig::addPostRegAlloc();
+}
- // PISA does not allocate physical registers.
- FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
- bool addRegAssignAndRewriteFast() override { return false; }
- bool addRegAssignAndRewriteOptimized() override { return false; }
-};
-} // namespace
+void PISAPassConfig::addPreEmitPass() {
+ // The lifetime.start marker names a *post-coalescing* virtual register, so it
+ // must run after Register Coalescer and PISAOptimizeRedundantCopies.
+ if (getOptLevel() != CodeGenOptLevel::None)
+ addPass(createPISAInsertLifetimeStart());
+ TargetPassConfig::addPreEmitPass();
+}
TargetPassConfig *PISATargetMachine::createPassConfig(PassManagerBase &PM) {
return new PISAPassConfig(*this, PM);
diff --git a/llvm/lib/Target/PISA/PISAVerifier.cpp b/llvm/lib/Target/PISA/PISAVerifier.cpp
new file mode 100644
index 0000000000000..3c536b05fb537
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAVerifier.cpp
@@ -0,0 +1,301 @@
+//===-- PISAVerifier.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "PISATargetMachine.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PISAIntrinsicUtils.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/PISAAddrSpace.h"
+
+#define DEBUG_TYPE "pisa-verifier"
+#define DEBUG_NAME "PISA verifier"
+
+using namespace llvm;
+
+namespace {
+
+class PISAVerifier : public ModulePass, public InstVisitor<PISAVerifier> {
+public:
+ static char ID;
+ PISAVerifier() : ModulePass(ID) {}
+
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<TargetPassConfig>();
+ AU.setPreservesAll();
+ }
+
+ bool runOnModule(Module &M) override;
+
+ void visitIntrinsicInst(IntrinsicInst &I);
+ void visitStoreInst(StoreInst &I);
+ void visitAtomicRMWInst(AtomicRMWInst &I);
+
+private:
+ void verifyFunction(Function &F);
+ void verifyGlobalVariable(GlobalVariable &GV);
+ void verifyKernelArg(Argument &Arg);
+ void verifyRoundingMode(IntrinsicInst &I, bool HasSaturation = true);
+ void verifyEnumArg(IntrinsicInst &I, unsigned ArgIdx, unsigned MaxVal,
+ StringRef ArgName);
+ void verifyHostAccessMetadata(const GlobalVariable &GV,
+ ArrayRef<const MDNode *> MDs);
+
+ void illegal(Twine Message) {
+ assert(Ctx);
+ Ctx->diagnose(DiagnosticInfoGeneric({Twine("PISA Verifier: ") + Message}));
+ }
+
+ void warning(Twine Message) {
+ assert(Ctx);
+ Ctx->diagnose(DiagnosticInfoGeneric(
+ {Twine("PISA Verifier: ") + Message, DS_Warning}));
+ }
+
+ SmallSet<StringRef, 4> HostAccessNamesSeen;
+ LLVMContext *Ctx = nullptr;
+ const TargetMachine *TM = nullptr;
+ const Function *CurrFunc = nullptr;
+};
+
+} // namespace
+
+char PISAVerifier::ID = 0;
+INITIALIZE_PASS_BEGIN(PISAVerifier, DEBUG_TYPE, DEBUG_NAME, false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(PISAVerifier, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+void PISAVerifier::verifyKernelArg(Argument &Arg) {
+ // According to PISA spec:
+ // - Kernel arguments are modeled as being flattened out and stored in
+ // abstract location pointed to by corresponding kernel argument with actual
+ // storage location of kernel parameters being implementation defined.
+ // - Kernel parameters are read-only.
+ // If a kernel argument is passed by value, PISA backend
+ // substitutes the pointer argument with the actual value it points to.
+ // Based on all the above:
+ // - byref arguments have no sense in PISA and are not expected
+ // - byval arguments are expected to be in the private addrspace in order to
+ // allow creation of copies
+ if (!Arg.getType()->isPointerTy())
+ return;
+ if (Arg.hasByRefAttr())
+ illegal(Twine("Kernel pointer arguments with the byref attribute are not "
+ "allowed\nKernel: ") +
+ Twine(Arg.getParent()->getName()) +
+ "\nArg no: " + Twine(Arg.getArgNo()));
+ else if (Arg.hasByValAttr() &&
+ Arg.getType()->getPointerAddressSpace() !=
+ static_cast<unsigned>(PISAAS::AddressSpace::PRIVATE))
+ illegal(Twine("Kernel pointer arguments with byval attribute are "
+ "expected to be in the private addrspace\nKernel: ") +
+ Twine(Arg.getParent()->getName()) +
+ "\nArg no: " + Twine(Arg.getArgNo()));
+}
+
+void PISAVerifier::verifyRoundingMode(IntrinsicInst &I, bool HasSaturation) {
+ const llvm::Function *F = I.getCalledFunction();
+ auto RndOpndIdx =
+ HasSaturation ? I.getNumOperands() - 3 : I.getNumOperands() - 2;
+ auto RndValue = static_cast<llvm::RoundingMode>(
+ cast<ConstantInt>(I.getOperand(RndOpndIdx))->getZExtValue());
+ switch (RndValue) {
+ case llvm::RoundingMode::TowardZero:
+ case llvm::RoundingMode::NearestTiesToEven:
+ case llvm::RoundingMode::TowardPositive:
+ case llvm::RoundingMode::TowardNegative:
+ case llvm::RoundingMode::NearestTiesToAway:
+ case llvm::RoundingMode::Invalid:
+ break;
+ default:
+ illegal("Intrinsic " + F->getName() +
+ " specifies invalid rounding mode value " +
+ std::to_string(static_cast<int>(RndValue)));
+ }
+}
+
+void PISAVerifier::verifyEnumArg(IntrinsicInst &I, unsigned ArgIdx,
+ unsigned MaxVal, StringRef ArgName) {
+ const llvm::Function *F = I.getCalledFunction();
+ auto Val = cast<ConstantInt>(I.getArgOperand(ArgIdx))->getZExtValue();
+ if (Val > MaxVal)
+ illegal("Intrinsic " + F->getName() + " has invalid " + ArgName +
+ " value " + std::to_string(Val));
+}
+
+void PISAVerifier::visitIntrinsicInst(IntrinsicInst &I) {
+ auto IID = I.getIntrinsicID();
+ switch (IID) {
+ case Intrinsic::log:
+ case Intrinsic::log2:
+ case Intrinsic::log10:
+ case Intrinsic::exp:
+ case Intrinsic::sin:
+ case Intrinsic::cos:
+ case Intrinsic::pow:
+ case Intrinsic::powi:
+ if (I.getType()->isDoubleTy()) {
+ const llvm::Function *F = I.getCalledFunction();
+ assert(F && "Intrinsic must have a called function");
+ illegal("Intrinsic " + F->getName() +
+ " is not supported on the PISA target");
+ }
+ break;
+ case Intrinsic::pisa_bfn: {
+ uint8_t Lut = static_cast<uint8_t>(
+ cast<ConstantInt>(I.getArgOperand(0))->getZExtValue());
+ if (Lut == 0x00)
+ illegal("BFN operation 0x00 is invalid; valid range is 0x01 to 0xfe");
+ else if (Lut == 0xff)
+ illegal("BFN operation 0xff is invalid; valid range is 0x01 to 0xfe");
+ break;
+ }
+ case Intrinsic::pisa_fadd:
+ case Intrinsic::pisa_fsub:
+ case Intrinsic::pisa_fmul:
+ case Intrinsic::pisa_fma:
+ case Intrinsic::pisa_sitofp:
+ case Intrinsic::pisa_uitofp:
+ case Intrinsic::pisa_ftrunc:
+ verifyRoundingMode(I);
+ break;
+ case Intrinsic::pisa_fdiv_rnd:
+ case Intrinsic::pisa_pow_rnd:
+ case Intrinsic::pisa_fsqrt_rnd:
+ case Intrinsic::pisa_frnd_rnd:
+ case Intrinsic::pisa_frcp_rnd:
+ case Intrinsic::pisa_sin_rnd:
+ case Intrinsic::pisa_cos_rnd:
+ case Intrinsic::pisa_tanh_rnd:
+ case Intrinsic::pisa_exp_rnd:
+ case Intrinsic::pisa_exp2_rnd:
+ case Intrinsic::pisa_log_rnd:
+ case Intrinsic::pisa_log2_rnd:
+ case Intrinsic::pisa_log10_rnd:
+ case Intrinsic::pisa_fptosi_rnd:
+ case Intrinsic::pisa_fptoui_rnd:
+ verifyRoundingMode(I, /*HasSaturation=*/false);
+ break;
+ case Intrinsic::pisa_shfl:
+ verifyEnumArg(I, 0, pisa::SHFLMode::Last - 1, "shfl mode");
+ break;
+ case Intrinsic::pisa_ired:
+ verifyEnumArg(I, 0, pisa::IRedOp::Last - 1, "ired op");
+ break;
+ case Intrinsic::pisa_fred:
+ verifyEnumArg(I, 0, pisa::FRedOp::Last - 1, "fred op");
+ break;
+ default:
+ break;
+ }
+}
+
+void PISAVerifier::visitStoreInst(StoreInst &I) {
+ if (I.getPointerAddressSpace() ==
+ static_cast<unsigned>(PISAAS::AddressSpace::CONSTANT))
+ illegal("Store to constant memory is not allowed");
+}
+
+void PISAVerifier::visitAtomicRMWInst(AtomicRMWInst &I) {
+ if (I.getPointerAddressSpace() ==
+ static_cast<unsigned>(PISAAS::AddressSpace::CONSTANT))
+ illegal("AtomicRMW on constant memory is not allowed");
+}
+
+void PISAVerifier::verifyFunction(Function &F) {
+ CurrFunc = &F;
+ if (F.getCallingConv() == CallingConv::PISA_KERNEL)
+ for (auto &Arg : F.args())
+ verifyKernelArg(Arg);
+
+ visit(F);
+}
+
+// The frontend attaches !intel_host_access metadata to a global variable to
+// describe its host-side visibility. The metadata node has two operands: the
+// host access mode (a 32-bit integer in the range [0, 3]) and the host-visible
+// name (a string). The backend lowers it to the ".host_access" variable
+// directive documented in the PISA spec (intel.github.io/pisa/variables.html).
+void PISAVerifier::verifyHostAccessMetadata(const GlobalVariable &GV,
+ ArrayRef<const MDNode *> MDs) {
+ if (MDs.size() != 1) {
+ illegal("!intel_host_access metadata attached more than once to global '" +
+ GV.getName() + "'");
+ return;
+ }
+
+ const MDNode *MD = MDs[0];
+ if (MD->getNumOperands() != 2) {
+ illegal("!intel_host_access metadata must have exactly 2 operands");
+ return;
+ }
+
+ auto VerifyFirstOp = [&]() -> bool {
+ auto *HostAccessVal = mdconst::dyn_extract<ConstantInt>(MD->getOperand(0));
+ if (!HostAccessVal)
+ return false;
+
+ if (HostAccessVal->getBitWidth() != 32)
+ return false;
+
+ if (HostAccessVal->getZExtValue() > 3)
+ return false;
+
+ return true;
+ };
+
+ if (!VerifyFirstOp())
+ illegal("Host access mode (first operand) of !intel_host_access metadata "
+ "must be a 32-bit integer in the range [0, 3].");
+
+ auto *NameMD = dyn_cast<MDString>(MD->getOperand(1));
+ if (!NameMD) {
+ illegal(
+ "Host name (second operand) of !intel_host_access metadata must be a "
+ "string.");
+ return;
+ }
+
+ const bool Inserted = HostAccessNamesSeen.insert(NameMD->getString()).second;
+ if (!Inserted)
+ illegal("Host access name '" + NameMD->getString() +
+ "' is specified for more than one global variable");
+}
+
+void PISAVerifier::verifyGlobalVariable(GlobalVariable &GV) {
+ SmallVector<MDNode *, 1> MDs;
+ const unsigned HostAccessKindId = Ctx->getMDKindID("intel_host_access");
+ if (GV.getMetadata(HostAccessKindId, MDs); !MDs.empty())
+ verifyHostAccessMetadata(GV, MDs);
+}
+
+bool PISAVerifier::runOnModule(Module &M) {
+ Ctx = &M.getContext();
+ auto &TPC = getAnalysis<TargetPassConfig>();
+ TM = &TPC.getTM<TargetMachine>();
+ for (Function &F : M)
+ verifyFunction(F);
+ for (GlobalVariable &GV : M.globals())
+ verifyGlobalVariable(GV);
+ return false;
+}
+
+ModulePass *llvm::createPISAVerifierPass() { return new PISAVerifier(); }
diff --git a/llvm/lib/Target/PISA/PISAVerifyTypes.cpp b/llvm/lib/Target/PISA/PISAVerifyTypes.cpp
new file mode 100644
index 0000000000000..00039e2e0ea89
--- /dev/null
+++ b/llvm/lib/Target/PISA/PISAVerifyTypes.cpp
@@ -0,0 +1,87 @@
+//===-- PISAVerifyTypes.cpp - modify function signatures ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Mixing of scalar/integer/float types causes issues with extended LLT
+// (scalar == integer, scalar == float, but integer != float), especially
+// with combiner, e.g. folding multiple COPY into illegal instruction.
+//
+// We disallow usage of ANY_SCALAR types when extended LLT is on.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PISA.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "pisa-verify-types"
+#define DEBUG_NAME "PISA verify types"
+
+static cl::opt<bool> EnableVerifyTypes("pisa-verify-extended-types",
+ cl::desc("Enable PISA verify types"),
+#ifndef NDEBUG
+ cl::init(true),
+#else // NDEBUG
+ cl::init(false),
+#endif // NDEBUG
+ cl::Hidden);
+
+namespace {
+
+class PISAVerifyTypes : public MachineFunctionPass {
+public:
+ static char ID;
+
+ PISAVerifyTypes() : MachineFunctionPass(ID) {}
+ StringRef getPassName() const override { return DEBUG_NAME; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+};
+
+} // namespace
+
+char PISAVerifyTypes::ID = 0;
+INITIALIZE_PASS(PISAVerifyTypes, DEBUG_TYPE, DEBUG_NAME, false, false)
+
+bool PISAVerifyTypes::runOnMachineFunction(MachineFunction &MF) {
+ if (!EnableVerifyTypes)
+ return false;
+
+ // TODO: remove next 2 lines after extended LLT is enabled
+ if (!LLT::getUseExtended())
+ return false;
+
+ assert(LLT::getUseExtended() &&
+ "PISAVerifyTypes only works with extended LLT");
+ // verify that we do not have any scalar types
+ for (auto &MBB : MF) {
+ for (auto &MI : MBB) {
+ if (!MI.isPreISelOpcode())
+ continue;
+ for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
+ auto &MO = MI.getOperand(I);
+ if (!MO.isReg())
+ continue;
+ auto RegTy = MF.getRegInfo().getType(MO.getReg());
+ if (RegTy.getScalarType().getKind() == LLT::Kind::ANY_SCALAR)
+ MI.emitGenericError("use of scalar types in " +
+ MI.getMF()->getName() +
+ " not supported with extendedLLT");
+ }
+ }
+ }
+ return false;
+}
+
+MachineFunctionPass *llvm::createPISAVerifyTypesPass() {
+ return new PISAVerifyTypes();
+}
diff --git a/llvm/test/CodeGen/PISA/mark-convergent-no-merge.mir b/llvm/test/CodeGen/PISA/mark-convergent-no-merge.mir
new file mode 100644
index 0000000000000..b7836a729c330
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/mark-convergent-no-merge.mir
@@ -0,0 +1,56 @@
+# A convergent instruction without the NoMerge flag should have it set by the pass.
+# A function with no convergent instructions should pass through unchanged.
+# RUN: llc -march=pisa -run-pass=pisa-mark-convergent-no-merge \
+# RUN: -verify-machineinstrs -o - %s | FileCheck %s
+
+# Test 1: A convergent ired instruction without NoMerge should get NoMerge set.
+# CHECK-LABEL: name: convergent_gets_nomerge
+# CHECK: %0:reg32b = nomerge ired_umax_32b_rir
+
+# Test 2: A function with no convergent instructions should pass through unchanged.
+# CHECK-LABEL: name: no_convergent_no_change
+# CHECK-NOT: nomerge
+# CHECK: ret
+
+---
+name: convergent_gets_nomerge
+alignment: 1
+legalized: true
+regBankSelected: true
+selected: true
+tracksRegLiveness: true
+noPhis: true
+isSSA: false
+noVRegs: false
+hasFakeUses: false
+registers:
+ - { id: 0, class: reg32b }
+ - { id: 1, class: reg32b }
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.0:
+ %1:reg32b = functionParameter_i32 0
+ %0:reg32b = ired_umax_32b_rir %1, -1, undef %0
+ retValue_i32_r %0
+...
+---
+name: no_convergent_no_change
+alignment: 1
+legalized: true
+regBankSelected: true
+selected: true
+tracksRegLiveness: true
+noPhis: true
+isSSA: false
+noVRegs: false
+hasFakeUses: false
+registers:
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.0:
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/pisa-scope-selector.mir b/llvm/test/CodeGen/PISA/pisa-scope-selector.mir
new file mode 100644
index 0000000000000..e092e4eb0145b
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/pisa-scope-selector.mir
@@ -0,0 +1,105 @@
+# RUN: llc -march=pisa -run-pass=pisa-scope-selector -verify-machineinstrs \
+# RUN: -o - %s | FileCheck %s
+
+# Test 1: an atomic with scope already set to workgroup (value 2) must not be
+# overwritten by the pass.
+# CHECK-LABEL: name: scope_preset
+# CHECK: atomic_load_add_impl_i32_global_acq_rel_add_ri {{.*}}, 2, 0
+
+# Test 2: a non-atomic store has no $scope operand; the pass should skip it.
+# CHECK-LABEL: name: no_scope_operand
+# CHECK: st_global_i32_ri {{.*}}, 0
+
+# Test 3: monotonic ordering does not trigger a scope upgrade; scope stays at
+# 255 (NONE).
+# CHECK-LABEL: name: monotonic_no_change
+# CHECK: atomic_load_add_impl_i32_global_monotonic_add_ri {{.*}}, 255, 0
+
+--- |
+ define void @scope_preset(ptr addrspace(1) %ptr, i32 %a) {
+ %res = atomicrmw add ptr addrspace(1) %ptr, i32 %a acq_rel, align 4
+ ret void
+ }
+ define void @no_scope_operand(ptr addrspace(1) %ptr, i32 %a) {
+ store i32 %a, ptr addrspace(1) %ptr, align 4
+ ret void
+ }
+ define void @monotonic_no_change(ptr addrspace(1) %ptr, i32 %a) {
+ %res = atomicrmw add ptr addrspace(1) %ptr, i32 %a monotonic, align 4
+ ret void
+ }
+...
+---
+name: scope_preset
+alignment: 1
+legalized: true
+regBankSelected: true
+selected: true
+tracksRegLiveness: true
+noPhis: false
+isSSA: true
+noVRegs: false
+hasFakeUses: false
+registers:
+ - { id: 0, class: reg64b }
+ - { id: 1, class: reg32b }
+ - { id: 2, class: reg32b }
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.1 (%ir-block.0):
+ %0:reg64b = functionParameter_i64 0
+ %1:reg32b = functionParameter_i32 1
+ %2:reg32b = atomic_load_add_impl_i32_global_acq_rel_add_ri %0, 0, %1, 2, 0 :: (load store acq_rel (i32) on %ir.ptr, addrspace 1)
+ ret
+...
+---
+name: no_scope_operand
+alignment: 1
+legalized: true
+regBankSelected: true
+selected: true
+tracksRegLiveness: true
+noPhis: false
+isSSA: true
+noVRegs: false
+hasFakeUses: false
+registers:
+ - { id: 0, class: reg64b }
+ - { id: 1, class: reg32b }
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.1 (%ir-block.0):
+ %0:reg64b = functionParameter_i64 0
+ %1:reg32b = functionParameter_i32 1
+ st_global_i32_ri %0, 0, %1, 0 :: (store release (i32) into %ir.ptr, addrspace 1)
+ ret
+...
+---
+name: monotonic_no_change
+alignment: 1
+legalized: true
+regBankSelected: true
+selected: true
+tracksRegLiveness: true
+noPhis: false
+isSSA: true
+noVRegs: false
+hasFakeUses: false
+registers:
+ - { id: 0, class: reg64b }
+ - { id: 1, class: reg32b }
+ - { id: 2, class: reg32b }
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.1 (%ir-block.0):
+ %0:reg64b = functionParameter_i64 0
+ %1:reg32b = functionParameter_i32 1
+ %2:reg32b = atomic_load_add_impl_i32_global_monotonic_add_ri %0, 0, %1, 255, 0 :: (load store monotonic (i32) on %ir.ptr, addrspace 1)
+ ret
+...
diff --git a/llvm/test/CodeGen/PISA/propagate-null.ll b/llvm/test/CodeGen/PISA/propagate-null.ll
new file mode 100644
index 0000000000000..5a190dea3e5b0
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/propagate-null.ll
@@ -0,0 +1,173 @@
+; RUN: llc < %s -march=pisa -stop-after=pisa-propagate-null-pointers | FileCheck %s
+
+; CHECK: @global_null_ptr = dso_local local_unnamed_addr addrspace(1) global ptr addrspace(3) inttoptr (i32 -1 to ptr addrspace(3)), align 4
+; CHECK-DAG: @shared_null = {{.*}} addrspace(3) inttoptr (i32 -1 to ptr addrspace(3))
+; CHECK-DAG: @private_null = {{.*}} addrspace(4) inttoptr (i32 -1 to ptr addrspace(4))
+ at global_null_ptr = dso_local local_unnamed_addr addrspace(1) global ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)), align 4
+ at global_shared = addrspace(3) global i32 0, align 4
+
+; CHECK-LABEL: @test_generic_to_shared
+define ptr addrspace(3) @test_generic_to_shared(ptr %arg) {
+ ; CHECK: [[CAST:%.*]] = addrspacecast ptr %arg to ptr addrspace(3)
+ ; CHECK-NEXT: [[PRED:%.*]] = icmp ne ptr %arg, null
+ ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[PRED]], ptr addrspace(3) [[CAST]], ptr addrspace(3) inttoptr (i32 -1 to ptr addrspace(3))
+ ; CHECK-NEXT: ret ptr addrspace(3) [[SEL]]
+ %ptr = addrspacecast ptr %arg to ptr addrspace(3)
+ ret ptr addrspace(3) %ptr
+}
+
+; CHECK-LABEL: @test_shared_to_generic
+define ptr @test_shared_to_generic(ptr addrspace(3) %arg) {
+ ; CHECK: [[CAST:%.*]] = addrspacecast ptr addrspace(3) %arg to ptr
+ ; CHECK-NEXT: [[PRED:%.*]] = icmp ne ptr addrspace(3) %arg, inttoptr (i32 -1 to ptr addrspace(3))
+ ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[PRED]], ptr [[CAST]], ptr null
+ ; CHECK-NEXT: ret ptr [[SEL]]
+ %ptr = addrspacecast ptr addrspace(3) %arg to ptr
+ ret ptr %ptr
+}
+
+; CHECK-LABEL: @test_generic_to_private
+define ptr addrspace(4) @test_generic_to_private(ptr %arg) {
+ ; CHECK: [[CAST:%.*]] = addrspacecast ptr %arg to ptr
+ ; CHECK-NEXT: [[PRED:%.*]] = icmp ne ptr %arg, null
+ ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[PRED]], ptr addrspace(4) [[CAST]], ptr addrspace(4) inttoptr (i32 -1 to ptr addrspace(4))
+ ; CHECK-NEXT: ret ptr addrspace(4) [[SEL]]
+ %ptr = addrspacecast ptr %arg to ptr addrspace(4)
+ ret ptr addrspace(4) %ptr
+}
+
+; CHECK-LABEL: @test_private_to_generic
+define ptr @test_private_to_generic(ptr addrspace(4) %arg) {
+ ; CHECK: [[CAST:%.*]] = addrspacecast ptr addrspace(4) %arg to ptr
+ ; CHECK-NEXT: [[PRED:%.*]] = icmp ne ptr addrspace(4) %arg, inttoptr (i32 -1 to ptr addrspace(4))
+ ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[PRED]], ptr [[CAST]], ptr null
+ ; CHECK-NEXT: ret ptr [[SEL]]
+ %ptr = addrspacecast ptr addrspace(4) %arg to ptr
+ ret ptr %ptr
+}
+
+; CHECK-LABEL: @test_generic_to_global
+define ptr addrspace(1) @test_generic_to_global(ptr %arg) {
+ ; CHECK: %ptr = addrspacecast ptr %arg to ptr addrspace(1)
+ ; CHECK-NEXT: ret ptr addrspace(1) %ptr
+ %ptr = addrspacecast ptr %arg to ptr addrspace(1)
+ ret ptr addrspace(1) %ptr
+}
+
+; CHECK-LABEL: @test_global_to_generic
+define ptr @test_global_to_generic(ptr addrspace(1) %arg) {
+ ; CHECK: %ptr = addrspacecast ptr addrspace(1) %arg to ptr
+ ; CHECK-NEXT: ret ptr %ptr
+ %ptr = addrspacecast ptr addrspace(1) %arg to ptr
+ ret ptr %ptr
+}
+
+; CHECK-LABEL: @test_alloca_private
+define ptr @test_alloca_private() {
+ ; CHECK: %var = alloca i32, align 4, addrspace(4)
+ ; CHECK-NEXT: %ptr = addrspacecast ptr addrspace(4) %var to ptr
+ ; CHECK-NEXT: ret ptr %ptr
+ %var = alloca i32, align 4, addrspace(4)
+ %ptr = addrspacecast ptr addrspace(4) %var to ptr
+ ret ptr %ptr
+}
+
+; CHECK-LABEL: @test_global_in_shared
+define ptr @test_global_in_shared() {
+ ; CHECK: %ptr = addrspacecast ptr addrspace(3) @global_shared to ptr
+ ; CHECK-NEXT: ret ptr %ptr
+ %ptr = addrspacecast ptr addrspace(3) @global_shared to ptr
+ ret ptr %ptr
+}
+
+; CHECK-LABEL: @test_nonnull_arg
+define ptr @test_nonnull_arg(ptr addrspace(4) noundef nonnull %arg) {
+ ; CHECK: %ptr = addrspacecast ptr addrspace(4) %arg to ptr
+ ; CHECK-NEXT: ret ptr %ptr
+ %ptr = addrspacecast ptr addrspace(4) %arg to ptr
+ ret ptr %ptr
+}
+
+; CHECK-LABEL: @test_kernel_arg_shared
+define pisa_kernel void @test_kernel_arg_shared(ptr addrspace(3) %arg, ptr addrspace(1) %res) {
+ ; CHECK: %ptr = addrspacecast ptr addrspace(3) %arg to ptr
+ ; CHECK-NEXT: store ptr %ptr, ptr addrspace(1) %res, align 4
+ ; CHECK-NEXT: ret void
+ %ptr = addrspacecast ptr addrspace(3) %arg to ptr
+ store ptr %ptr, ptr addrspace(1) %res, align 4
+ ret void
+}
+
+%struct.byval = type { i32, i16, i64, i8 }
+
+; CHECK-LABEL: @test_kernel_arg_private
+define pisa_kernel void @test_kernel_arg_private(ptr addrspace(4) byval(%struct.byval) %arg, ptr addrspace(1) %res) {
+ ; CHECK: %ptr = addrspacecast ptr addrspace(4) %arg to ptr
+ ; CHECK-NEXT: store ptr %ptr, ptr addrspace(1) %res, align 4
+ ; CHECK-NEXT: ret void
+ %ptr = addrspacecast ptr addrspace(4) %arg to ptr
+ store ptr %ptr, ptr addrspace(1) %res, align 4
+ ret void
+}
+
+; CHECK-LABEL: @test_kernel_arg_different_memory
+define pisa_kernel void @test_kernel_arg_different_memory(ptr addrspace(1) %arg, ptr addrspace(1) %res) {
+ ; CHECK: [[CAST:%.*]] = addrspacecast ptr %cast to ptr addrspace(3)
+ ; CHECK-NEXT: [[PRED:%.*]] = icmp ne ptr %cast, null
+ ; CHECK-NEXT: [[RES:%.*]] = select i1 [[PRED]], ptr addrspace(3) [[CAST]], ptr addrspace(3) inttoptr (i32 -1 to ptr addrspace(3))
+ ; CHECK-NEXT: store ptr addrspace(3) [[RES]], ptr addrspace(1) %res, align 4
+ %cast = addrspacecast ptr addrspace(1) %arg to ptr
+ %cast_2 = addrspacecast ptr %cast to ptr addrspace(3)
+ store ptr addrspace(3) %cast_2, ptr addrspace(1) %res, align 4
+ ret void
+}
+
+; CHECK-LABEL: @test_constexpr_shared_null
+define i1 @test_constexpr_shared_null(i32 %val) {
+ ; CHECK: %cmp = icmp ne i32 %val, ptrtoint (ptr addrspace(3) inttoptr (i32 -1 to ptr addrspace(3)) to i32)
+ ; CHECK-NEXT: ret i1 %cmp
+ %cmp = icmp ne i32 %val, ptrtoint (ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)) to i32)
+ ret i1 %cmp
+}
+
+; CHECK-LABEL: @test_constexpr_private_null
+define i1 @test_constexpr_private_null(i32 %val) {
+ ; CHECK: %cmp = icmp ne i32 %val, ptrtoint (ptr addrspace(4) inttoptr (i32 -1 to ptr addrspace(4)) to i32)
+ ; CHECK-NEXT: ret i1 %cmp
+ %cmp = icmp ne i32 %val, ptrtoint (ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)) to i32)
+ ret i1 %cmp
+}
+
+; CHECK-LABEL: @test_complex_expression
+define ptr @test_complex_expression(i1 %cond) {
+ ; CHECK: [[CAST:%.*]] = addrspacecast ptr addrspace(3) %val to ptr
+ ; CHECK-NEXT: [[PRED:%.*]] = icmp ne ptr addrspace(3) %val, inttoptr (i32 -1 to ptr addrspace(3))
+ ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[PRED]], ptr [[CAST]], ptr null
+ %val = select i1 %cond, ptr addrspace(3) @global_shared, ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3))
+ %res = addrspacecast ptr addrspace(3) %val to ptr
+ ret ptr %res
+}
+
+; Both globals use constant expression casts from generic null to private/shared.
+; This ensures both GenericToPrivateCast and GenericToSharedCast are non-empty,
+; so the pass marks the module as changed and replaces const expr casts.
+
+ at shared_null = addrspace(1) global ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)), align 4
+
+ at private_null = addrspace(1) global ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)), align 4
+
+; A function that compares against both shared and private null constant exprs,
+; keeping both uses alive so the pass must replace both addrspacecasts.
+; CHECK-LABEL: @test_both_constexpr_used
+define i1 @test_both_constexpr_used(i32 %val, i32 %val2) {
+ ; CHECK: %cmp_shared = icmp ne i32 %val, ptrtoint (ptr addrspace(3) inttoptr (i32 -1 to ptr addrspace(3)) to i32)
+ %cmp_shared = icmp ne i32 %val, ptrtoint (ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)) to i32)
+
+ ; CHECK-NEXT: %cmp_private = icmp ne i32 %val2, ptrtoint (ptr addrspace(4) inttoptr (i32 -1 to ptr addrspace(4)) to i32)
+ %cmp_private = icmp ne i32 %val2, ptrtoint (ptr addrspace(4) addrspacecast (ptr null to ptr addrspace(4)) to i32)
+
+ ; CHECK-NEXT: %result = and i1 %cmp_shared, %cmp_private
+ %result = and i1 %cmp_shared, %cmp_private
+ ; CHECK-NEXT: ret i1 %result
+ ret i1 %result
+}
diff --git a/llvm/test/CodeGen/PISA/verify-types.mir b/llvm/test/CodeGen/PISA/verify-types.mir
new file mode 100644
index 0000000000000..b9e0911e9fb35
--- /dev/null
+++ b/llvm/test/CodeGen/PISA/verify-types.mir
@@ -0,0 +1,35 @@
+# The verify-types pass iterates pre-ISel instructions and checks for scalar
+# types when -pisa-verify-extended-types is enabled. A fully-selected function
+# with no pre-ISel instructions skips the inner loop and passes cleanly.
+
+# RUN: llc -march=pisa -run-pass=pisa-verify-types \
+# RUN: -pisa-verify-extended-types -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s
+
+# CHECK-LABEL: name: no_preisel
+# CHECK: ret
+
+--- |
+ define void @no_preisel() {
+ ret void
+ }
+...
+---
+name: no_preisel
+alignment: 1
+legalized: true
+regBankSelected: true
+selected: true
+tracksRegLiveness: true
+noPhis: true
+isSSA: false
+noVRegs: false
+hasFakeUses: false
+registers:
+frameInfo:
+ maxAlignment: 1
+machineFunctionInfo: {}
+body: |
+ bb.0:
+ ret
+...
>From 4300427bfe836170257c8a4f38b22dbfac278ffc Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Thu, 6 Aug 2026 22:53:38 -0700
Subject: [PATCH 2/3] Review fixes
---
llvm/lib/Target/PISA/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/PISA/CMakeLists.txt b/llvm/lib/Target/PISA/CMakeLists.txt
index a790f0abe2312..8c85277a615ca 100644
--- a/llvm/lib/Target/PISA/CMakeLists.txt
+++ b/llvm/lib/Target/PISA/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_target(PISACodeGen
PISAInstrInfo.cpp
PISAInstructionSelector.cpp
PISAKernelByValArgsLowering.cpp
+ PISALayout.cpp
PISALegalizeCalls.cpp
PISALegalizePredicates.cpp
PISALegalizeSubregAccess.cpp
@@ -50,7 +51,6 @@ add_llvm_target(PISACodeGen
PISAUtils.cpp
PISAVerifier.cpp
PISAVerifyTypes.cpp
- PISALayout.cpp
LINK_COMPONENTS
Analysis
>From f11fc5479cd41744ccae3eaba06d41aaf111232e Mon Sep 17 00:00:00 2001
From: Michal Paszkowski <michal.paszkowski at intel.com>
Date: Thu, 6 Aug 2026 22:59:05 -0700
Subject: [PATCH 3/3] Formatting fix
---
llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp b/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp
index 03b1d6141001a..0e201bef6ccc3 100644
--- a/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp
+++ b/llvm/lib/Target/PISA/PISAKernelByValArgsLowering.cpp
@@ -30,7 +30,8 @@ using namespace llvm::PISAAS;
namespace {
class PISAKernelByValArgsLowering {
public:
- explicit PISAKernelByValArgsLowering(Function &F) : F(F), Ctx(F.getContext()) {
+ explicit PISAKernelByValArgsLowering(Function &F)
+ : F(F), Ctx(F.getContext()) {
assert(F.getCallingConv() == CallingConv::PISA_KERNEL &&
"Expected PISA_KERNEL calling convention");
assert(F.getReturnType()->isVoidTy() && "Expected void return type");
@@ -90,8 +91,8 @@ bool PISAKernelByValArgsLowering::run() {
return true;
}
-AttributeList
-PISAKernelByValArgsLowering::getNewAttributes(const AttributeList &Attrs) const {
+AttributeList PISAKernelByValArgsLowering::getNewAttributes(
+ const AttributeList &Attrs) const {
AttributeList NewAttrs;
// Copy function attributes.
More information about the llvm-branch-commits
mailing list