[llvm-branch-commits] [llvm] [4/7][PISA] Add PISA GlobalISel lowering, legalization and combiners (PR #214373)
Vyacheslav Klochkov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Sep 18 19:15:17 PDT 2026
================
@@ -0,0 +1,1664 @@
+//===-- lib/CodeGen/GlobalISel/PISAPreLegalizerCombiner.cpp ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCTargetDesc/PISAMCTargetDesc.h"
+#include "PISA.h"
+#include "PISALegalizerInfo.h"
+#include "PISATargetMachine.h"
+#include "PISAUtils.h"
+#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
+#include "llvm/CodeGen/GlobalISel/Combiner.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
+#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IntrinsicsPISA.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define GET_GICOMBINER_DEPS
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_DEPS
+
+#define DEBUG_TYPE "pisa-prelegalizer-combiner"
+
+using namespace llvm;
+using namespace llvm::MIPatternMatch;
+
+namespace {
+
+#define GET_GICOMBINER_TYPES
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_TYPES
+
+class PISAPreLegalizerCombinerImpl : public Combiner {
+protected:
+ const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig;
+ const PISASubtarget &STI;
+ MachineDominatorTree *MDT;
+
+ // TODO: Make CombinerHelper methods const.
+ mutable CombinerHelper Helper;
+
+public:
+ PISAPreLegalizerCombinerImpl(
+ MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB,
+ GISelCSEInfo *CSEInfo,
+ const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig,
+ const PISASubtarget &STI, MachineDominatorTree *MDT,
+ const LegalizerInfo *LI);
+
+ static const char *getName() { return "PISAGenPreLegalizeGICombiner"; }
+
+ bool tryCombineAllImpl(MachineInstr &MI) const;
+ bool tryCombineAll(MachineInstr &I) const override;
+
+ void applyTruncatedLoad(MachineInstr &MI) const;
+
+ bool matchTruncatedStore(MachineInstr &MI) const;
+ void applyTruncatedStore(MachineInstr &MI) const;
+
+ bool matchExtendedLoad(MachineInstr &MI) const;
+ void applyExtendedLoad(MachineInstr &MI) const;
+
+ bool matchSimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const;
+ void applySimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const;
+
+ bool matchExpandNonPowerOf2LoadStore(MachineInstr &MI) const;
+ void applyExpandNonPowerOf2LoadStore(MachineInstr &MI) const;
+
+ bool matchTruncatedShift(MachineInstr &MI) const;
+ void applyTruncatedShift(MachineInstr &MI) const;
+
+ bool matchRedundantMovesPre(MachineInstr &MI) const;
+ void applyRedundantMovesPre(MachineInstr &MI) const;
+
+ bool
+ matchRcpSqrtToRsqrt(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+ bool
+ matchSubFloorToFrc(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchLaneIdLeftShiftChain(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchZExtAndToAndZExt(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchExtractInsertToBitcast(MachineInstr &MI, Register &) const;
+ void applyExtractInsertToBitcast(MachineInstr &MI, Register) const;
+
+ bool matchExtractBuildVectorToBitcast(MachineInstr &MI, Register &) const;
+ void applyExtractBuildVectorToBitcast(MachineInstr &MI, Register) const;
+
+ bool matchReducePredicates(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchCmpInt1(MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+ bool matchSelectTruncOneZero(
+ MachineInstr &MI,
+ std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+private:
+#define GET_GICOMBINER_CLASS_MEMBERS
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_CLASS_MEMBERS
+};
+
+#define GET_GICOMBINER_IMPL
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_IMPL
+
+PISAPreLegalizerCombinerImpl::PISAPreLegalizerCombinerImpl(
+ MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &KB,
+ GISelCSEInfo *CSEInfo,
+ const PISAPreLegalizerCombinerImplRuleConfig &RuleConfig,
+ const PISASubtarget &STI, MachineDominatorTree *MDT,
+ const LegalizerInfo *LI)
+ : Combiner(MF, CInfo, &KB, CSEInfo), RuleConfig(RuleConfig), STI(STI),
+ MDT(MDT), Helper(Observer, B, /*IsPreLegalize=*/true, &KB, MDT, LI),
+#define GET_GICOMBINER_CONSTRUCTOR_INITS
+#include "PISAGenPreLegalizeGICombiner.inc"
+#undef GET_GICOMBINER_CONSTRUCTOR_INITS
+{
+}
+
+bool PISAPreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
+ // G_FPTRUNC to bf16 needs special constant folding because
+ // getFltSemanticForLLT() doesn't support BFloat16 yet.
+ if (MI.getOpcode() == TargetOpcode::G_FPTRUNC &&
+ MRI.getType(MI.getOperand(0).getReg()).getScalarType().isBFloat16()) {
+ const ConstantFP *Cst = nullptr;
+ if (mi_match(MI.getOperand(1).getReg(), MRI, m_GFCst(Cst))) {
+ APFloat Result(Cst->getValue());
+ bool Unused;
+ Result.convert(APFloat::BFloat(), APFloat::rmNearestTiesToEven, &Unused);
+ const ConstantFP *NewCst = ConstantFP::get(B.getContext(), Result);
+ MachineIRBuilder Builder(MI);
+ Builder.buildFConstant(MI.getOperand(0), *NewCst);
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ if (tryCombineAllImpl(MI))
+ return true;
+
+ return false;
+}
+
+void PISAPreLegalizerCombinerImpl::applyTruncatedLoad(MachineInstr &MI) const {
+ auto *LoadMI = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+ auto *MMO = LoadMI->memoperands()[0];
+ auto Dst = MI.getOperand(0);
+ auto Addr = LoadMI->getOperand(1);
+ auto *NewMMO = MI.getMF()->getMachineMemOperand(MMO, MMO->getOffset(),
+ MRI.getType(Dst.getReg()));
+ B.buildLoad(Dst, Addr, *NewMMO);
+ MI.eraseFromParent();
+}
+
+bool PISAPreLegalizerCombinerImpl::matchExpandNonPowerOf2LoadStore(
+ MachineInstr &MI) const {
+ assert(MI.getOpcode() == TargetOpcode::G_LOAD ||
+ MI.getOpcode() == TargetOpcode::G_STORE);
+ GLoadStore &LS = cast<GLoadStore>(MI);
+
+ auto Size = LS.getMemSizeInBits().getValue();
+
+ if (isPowerOf2_32(Size))
+ return false;
+
+ if (LS.getMMO().getMemoryType().isVector())
+ return false;
+
+ // Check if this is a load and only has zext uses that are handled by the
+ // extended load pattern - if yes, we want to handle it via said pattern.
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ for (auto &Use : MRI.use_operands(MI.getOperand(0).getReg())) {
+ auto *Inst = Use.getParent();
+ if (Inst->getOpcode() != TargetOpcode::G_ZEXT ||
+ !matchExtendedLoad(*Inst))
+ return true;
+ }
+ return false;
+ }
+
+ return true;
+}
+void PISAPreLegalizerCombinerImpl::applyExpandNonPowerOf2LoadStore(
+ MachineInstr &MI) const {
+ assert(MI.getOpcode() == TargetOpcode::G_LOAD ||
+ MI.getOpcode() == TargetOpcode::G_STORE);
+
+ GLoadStore &LS = cast<GLoadStore>(MI);
+
+ auto PointerReg = LS.getPointerReg();
+ auto ValueReg = LS.getOperand(0).getReg();
+ auto &MMO = LS.getMMO();
+
+ /// The remaining size in Bits that still has to be loaded/stored
+ ssize_t Size = LS.getMemSizeInBits().getValue();
+ // Support sizes that are not a multiple of 8 by "promoting" them to the next
+ // multiple of 8. If the size is already a multiple of 8, it is not modified
+ Size = (Size + 7) & ~7;
+
+ unsigned Offset = 0;
+ const auto SizeTy = LLT::integer(Size);
+
+ // If the size was changed to the next power of 8 and we are storing a value,
+ // we need to modify the register's type as well.
+ // A similar check is needed for loads, but this is done at the end
+ if (MI.getOpcode() == TargetOpcode::G_STORE &&
+ SizeTy != MRI.getType(ValueReg)) {
+ auto NewValueReg = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildZExt(NewValueReg, ValueReg);
+ ValueReg = NewValueReg;
+ }
+
+ /// The register holding the loaded value at the end
+ Register LoadRes;
+ while (Size > 0) {
+ auto OpSize = bit_floor(static_cast<size_t>(Size));
+ const auto ShiftAmount = Offset * 8;
+
+ const LLT OpTy = LLT::integer(OpSize);
+
+ auto *NewMMO =
+ MI.getMF()->getMachineMemOperand(&MMO, MMO.getOffset() + Offset, OpTy);
+
+ /// Stores the (potentially modified) pointer register
+ auto AddrReg = PointerReg;
+ // We might need to change the store offset
+ if (Offset != 0) {
+ auto NewAddr = MRI.cloneVirtualRegister(AddrReg);
+
+ // Get the pointer size from the pointer register type
+ const LLT PtrTy = MRI.getType(AddrReg);
+ const LLT IntTy = LLT::integer(PtrTy.getSizeInBits());
+ auto Const = MRI.createGenericVirtualRegister(IntTy);
+ B.buildConstant(Const, Offset);
+ B.buildPtrAdd(NewAddr, AddrReg, Const);
+
+ AddrReg = NewAddr;
+ }
+
+ if (MI.getOpcode() == TargetOpcode::G_STORE) {
+ auto Res = ValueReg;
+ // If we're not storing from the start, we need to shift our value first
+ if (Offset != 0) {
+ auto ShrRes = MRI.createGenericVirtualRegister(SizeTy);
+ auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildConstant(ShiftConst, ShiftAmount);
+ B.buildLShr(ShrRes, ValueReg, ShiftConst);
+ Res = ShrRes;
+ }
+
+ // Next, truncate it to the correct size, if needed
+ if (SizeTy != OpTy) {
+ auto TruncRes = MRI.createGenericVirtualRegister(OpTy);
+ B.buildTrunc(TruncRes, Res);
+ Res = TruncRes;
+ }
+
+ B.buildStore(Res, AddrReg, *NewMMO);
+ } else {
+ // When loading, do the same thing but basically in reverse
+ // First, load the value
+ auto LoadedValReg = MRI.createGenericVirtualRegister(OpTy);
+ B.buildLoad(LoadedValReg, AddrReg, *NewMMO);
+ auto Res = LoadedValReg;
+
+ // Extend it to the correct size, if needed
+ if (SizeTy != OpTy) {
+ auto ZextRes = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildZExt(ZextRes, LoadedValReg);
+ Res = ZextRes;
+ }
+
+ // If we're not loading the first Bytes, then we need to shift it
+ if (Offset != 0) {
+ auto ShiftRes = MRI.createGenericVirtualRegister(SizeTy);
+
+ auto ShiftConst = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildConstant(ShiftConst, ShiftAmount);
+
+ B.buildShl(ShiftRes, Res, ShiftConst);
+ Res = ShiftRes;
+ }
+
+ if (!LoadRes.isValid()) {
+ LoadRes = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildConstant(LoadRes, 0);
+ }
+
+ auto NewLoadRes = MRI.createGenericVirtualRegister(SizeTy);
+ B.buildOr(NewLoadRes, LoadRes, Res);
+ LoadRes = NewLoadRes;
+ }
+
+ Offset += OpSize / 8;
+ Size -= OpSize;
+ }
+
+ if (MI.getOpcode() == TargetOpcode::G_LOAD) {
+ // If we modified the original size of the load (to support sizes that are
+ // not a multiple of 8), we need to truncate it to the correct size again,
+ // in order not to break any following instructions
+ if (SizeTy != MRI.getType(ValueReg)) {
+ auto TruncRes = MRI.createGenericVirtualRegister(MRI.getType(ValueReg));
+ B.buildTrunc(TruncRes, LoadRes);
+ LoadRes = TruncRes;
+ }
+ MRI.replaceRegWith(ValueReg, LoadRes);
+ }
+
+ MI.eraseFromParent();
+}
+
+/// If the value we store resulted from a G_LOAD that the rule above expanded,
+/// then we can use the individual load values directly instead of merging into
+/// one integer, and then splitting it again. Size modifications between the
+/// loads and store (i.e. SEXT/ZEXT/TRUNC) are also supported by
+/// truncating/extending the relevant load results (or, in the case of
+/// truncation, ignoring some loads entirely)
+///
+/// Below is a simple example where we simply load and store an i56:
+///
+/// bb.1.entry:
+/// %0:reg32b(p0) = functionParameter_32b 0
+/// -> %3:_(s56) = G_ZEXTLOAD %0:reg32b(p0) ::
+/// (load (s32) from %ir.dst)
+/// %4:_(s56) = G_CONSTANT i56 0
+/// %7:_(s32) = G_CONSTANT i32 4
+/// %6:reg32b(p0) = G_PTR_ADD %0:reg32b, %7:_(s32)
+/// -> %9:_(s56) = G_ZEXTLOAD %6:reg32b(p0) ::
+/// (load (s16) from %ir.dst + 4, align 4)
+/// %11:_(s56) = G_CONSTANT i56 32
+/// %10:_(s56) = G_SHL %9:_, %11:_(s56)
+/// %12:_(s56) = G_OR %3:_, %10:_
+/// %14:_(s32) = G_CONSTANT i32 6
+/// %13:reg32b(p0) = G_PTR_ADD %0:reg32b, %14:_(s32)
+/// -> %16:_(s56) = G_ZEXTLOAD %13:reg32b(p0) ::
+/// (load (s8) from %ir.dst + 6, align 2, basealign 4)
+/// %18:_(s56) = G_CONSTANT i56 48
+/// %17:_(s56) = G_SHL %16:_, %18:_(s56)
+/// %19:_(s56) = G_OR %12:_, %17:_
+/// G_STORE %19:_(s56), %0:reg32b(p0) ::
+/// (store (s56) into %ir.dst, align 4)
+/// ret
+///
+/// @param Loads stores all the load instructions that we find along the way.
+/// The second parameter in the pair is the offset from which the value was
+/// loaded
+/// @param SizeModificationOp stores the opcode to the instruction that modified
+/// the size of the integer after loading and before storing, if it exists.
+/// Possible opcode values are null (none), G_SEXT, G_SEXT_INREG, G_ZEXT, and
+/// G_TRUNC
+bool PISAPreLegalizerCombinerImpl::matchSimplifyNonPowerOf2LoadStoreChain(
+ MachineInstr &MI,
+ SmallVector<std::pair<MachineInstr *, unsigned>, 8> &Loads,
+ MachineInstr *&SizeModificationOp) const {
+ GLoadStore &StoreInst = cast<GLoadStore>(MI);
+
+ auto ValueReg = StoreInst.getOperand(0).getReg();
+
+ /// This has the integer size at the beginning, and each individual load
+ /// decreases the value by its load size. At the end, this must be zero.
+ /// NB: We use the size of the store here, promoted to the next multiple
+ /// of 8. This size might be modified later if we find a G_*EXT/G_TRUNC.
+ unsigned ValueSize = StoreInst.getMemSize().getValue() * 8;
+
+ /// This stores the size of the last load instruction in Bytes, s.t. we can
+ /// verify that the current load is larger than the previous one
----------------
v-klochkov wrote:
```suggestion
// This has the integer size at the beginning, and each individual load
// decreases the value by its load size. At the end, this must be zero.
// NB: We use the size of the store here, promoted to the next multiple
// of 8. This size might be modified later if we find a G_*EXT/G_TRUNC.
unsigned ValueSize = StoreInst.getMemSize().getValue() * 8;
// This stores the size of the last load instruction in Bytes, s.t. we can
// verify that the current load is larger than the previous one
```
https://github.com/llvm/llvm-project/pull/214373
More information about the llvm-branch-commits
mailing list