[llvm] 4311bcc - [NVPTX] Move NVPTXAsmPrinter and NVPTXDAGToDAGISel out of headers (NFC) (#213162)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 20:09:22 PDT 2026
Author: Alex MacLean
Date: 2026-07-30T20:09:16-07:00
New Revision: 4311bccd19e2db25e7c410acd9d77a86d2b779d1
URL: https://github.com/llvm/llvm-project/commit/4311bccd19e2db25e7c410acd9d77a86d2b779d1
DIFF: https://github.com/llvm/llvm-project/commit/4311bccd19e2db25e7c410acd9d77a86d2b779d1.diff
LOG: [NVPTX] Move NVPTXAsmPrinter and NVPTXDAGToDAGISel out of headers (NFC) (#213162)
Both headers were only included by their own `.cpp` file, so move the
class definitions into anonymous namespaces there and delete the
headers.
`getFromTypeWidthForLoad` was the only thing `NVPTXISelLowering.cpp`
needed from the ISel class, so it moves to `NVPTXUtilities` as a free
function.
Added:
Modified:
llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
llvm/lib/Target/NVPTX/NVPTXUtilities.h
Removed:
llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
################################################################################
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 0f011c18736b3..2b676bc239d86 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -11,10 +11,8 @@
//
//===----------------------------------------------------------------------===//
-#include "NVPTXAsmPrinter.h"
#include "MCTargetDesc/NVPTXBaseInfo.h"
#include "MCTargetDesc/NVPTXInstPrinter.h"
-#include "MCTargetDesc/NVPTXMCAsmInfo.h"
#include "MCTargetDesc/NVPTXTargetStreamer.h"
#include "NVPTX.h"
#include "NVPTXDwarfDebug.h"
@@ -44,6 +42,7 @@
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
@@ -70,18 +69,21 @@
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
+#include "llvm/IR/Value.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Pass.h"
#include "llvm/Support/Alignment.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
@@ -97,13 +99,279 @@
#include <cstdint>
#include <cstring>
#include <map>
+#include <memory>
#include <set>
#include <string>
+#include <type_traits>
+#include <vector>
using namespace llvm;
#define DEPOTNAME "__local_depot"
+// The ptx syntax and format is very
diff erent from that usually seem in a .s
+// file,
+// therefore we are not able to use the MCAsmStreamer interface here.
+//
+// We are handcrafting the output method here.
+//
+// A better approach is to clone the MCAsmStreamer to a MCPTXAsmStreamer
+// (subclass of MCStreamer).
+
+namespace {
+
+class NVPTXAsmPrinter : public AsmPrinter {
+
+ class AggBuffer {
+ // Used to buffer the emitted string for initializing global aggregates.
+ //
+ // Normally an aggregate (array, vector, or structure) is emitted as a u8[].
+ // However, if either element/field of the aggregate is a non-NULL address,
+ // and all such addresses are properly aligned, then the aggregate is
+ // emitted as u32[] or u64[]. In the case of unaligned addresses, the
+ // aggregate is emitted as u8[], and the mask() operator is used for all
+ // pointers.
+ //
+ // We first layout the aggregate in 'buffer' in bytes, except for those
+ // symbol addresses. For the i-th symbol address in the aggregate, its
+ // corresponding 4-byte or 8-byte elements in 'buffer' are filled with 0s.
+ // symbolPosInBuffer[i-1] records its position in 'buffer', and Symbols[i-1]
+ // records the Value*.
+ //
+ // Once we have this AggBuffer setup, we can choose how to print it out.
+ public:
+ // number of symbol addresses
+ unsigned numSymbols() const { return Symbols.size(); }
+
+ bool allSymbolsAligned(unsigned ptrSize) const {
+ return llvm::all_of(symbolPosInBuffer,
+ [=](unsigned pos) { return pos % ptrSize == 0; });
+ }
+
+ private:
+ const unsigned Size; // size of the buffer in bytes
+ std::vector<unsigned char> buffer; // the buffer
+ SmallVector<unsigned, 4> symbolPosInBuffer;
+ SmallVector<const Value *, 4> Symbols;
+ // SymbolsBeforeStripping[i] is the original form of Symbols[i] before
+ // stripping pointer casts, i.e.,
+ // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts().
+ //
+ // We need to keep these values because AggBuffer::print decides whether to
+ // emit a "generic()" cast for Symbols[i] depending on the address space of
+ // SymbolsBeforeStripping[i].
+ SmallVector<const Value *, 4> SymbolsBeforeStripping;
+ unsigned curpos;
+ const NVPTXAsmPrinter &AP;
+ const bool EmitGeneric;
+
+ public:
+ AggBuffer(unsigned Size, const NVPTXAsmPrinter &AP)
+ : Size(Size), buffer(Size), curpos(0), AP(AP),
+ EmitGeneric(AP.EmitGeneric) {}
+
+ unsigned getBufferSize() const { return Size; }
+
+ // Number of bytes written so far.
+ unsigned getCurpos() const { return curpos; }
+
+ // Copy Num bytes from Ptr.
+ // if Bytes > Num, zero fill up to Bytes.
+ void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) {
+ for (unsigned I : llvm::seq(Num))
+ addByte(Ptr[I]);
+ if (Bytes > Num)
+ addZeros(Bytes - Num);
+ }
+
+ void addByte(uint8_t Byte) {
+ assert(curpos < Size);
+ buffer[curpos] = Byte;
+ curpos++;
+ }
+
+ void addZeros(unsigned Num) {
+ for ([[maybe_unused]] unsigned _ : llvm::seq(Num)) {
+ addByte(0);
+ }
+ }
+
+ void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
+ symbolPosInBuffer.push_back(curpos);
+ Symbols.push_back(GVar);
+ SymbolsBeforeStripping.push_back(GVarBeforeStripping);
+ }
+
+ void printBytes(raw_ostream &os);
+ void printWords(raw_ostream &os);
+
+ private:
+ void printSymbol(unsigned nSym, raw_ostream &os);
+ };
+
+ friend class AggBuffer;
+
+public:
+ static char ID;
+
+ StringRef getPassName() const override { return "NVPTX Assembly Printer"; }
+
+private:
+ const Function *F;
+
+ NVPTXTargetStreamer *getTargetStreamer() const;
+
+ void emitStartOfAsmFile(Module &M) override;
+ void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
+ void emitFunctionEntryLabel() override;
+ void emitFunctionBodyStart() override;
+ void emitFunctionBodyEnd() override;
+ void emitImplicitDef(const MachineInstr *MI) const override;
+
+ void emitInstruction(const MachineInstr *) override;
+ void lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
+ MCOperand lowerOperand(const MachineOperand &MO);
+ MCOperand GetSymbolRef(const MCSymbol *Symbol);
+ MCRegister encodeVirtualRegister(Register Reg);
+
+ /// The number \p Reg was assigned within its register class, as declared by
+ /// this function's .reg directives.
+ unsigned getVirtualRegisterNumber(Register Reg) const;
+
+ void printMemOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O,
+ const char *Modifier = nullptr);
+ void printModuleLevelGV(const GlobalVariable *GVar, raw_ostream &O,
+ bool processDemoted, const NVPTXSubtarget &STI);
+ void emitGlobals(const Module &M);
+ void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
+ void emitHeader(Module &M, const NVPTXSubtarget &STI);
+ void emitKernelFunctionDirectives(const Function &F, raw_ostream &O) const;
+ void emitFunctionParamList(const Function *, raw_ostream &O);
+ void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF);
+ void encodeDebugInfoRegisterNumbers(const MachineFunction &MF);
+ void printReturnValStr(const Function *, raw_ostream &O);
+ void printReturnValStr(const MachineFunction &MF, raw_ostream &O);
+ void emitCallPrototype(const CallBase &CB, unsigned UniqueCallSite,
+ raw_ostream &O) const;
+ void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const;
+
+ /// Should a .noreturn directive be emitted for \p V, which is either a
+ /// function or a call site?
+ template <typename T> bool shouldEmitPTXNoReturn(const T &V) const {
+ static_assert(std::is_same_v<Function, T> || std::is_base_of_v<CallBase, T>,
+ "expected a function or a call site");
+
+ const auto &NTM = static_cast<const NVPTXTargetMachine &>(TM);
+ if (!NTM.getSubtargetImpl()->hasNoReturn())
+ return false;
+
+ if (!V.doesNotReturn() || !V.getFunctionType()->getReturnType()->isVoidTy())
+ return false;
+
+ if constexpr (std::is_same_v<Function, T>)
+ return !isKernelFunction(V);
+ else
+ return true;
+ }
+
+ bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+ const char *ExtraCode, raw_ostream &) override;
+ void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
+ bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
+ const char *ExtraCode, raw_ostream &) override;
+
+ const MCExpr *lowerConstantForGV(const Constant *CV,
+ bool ProcessingGeneric) const;
+ void printMCExpr(const MCExpr &Expr, raw_ostream &OS) const;
+ /// Emit a blob of inline asm to the output streamer.
+ void emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
+ const MCTargetOptions &MCOptions, const MDNode *LocMDNode,
+ InlineAsm::AsmDialect Dialect,
+ const MachineInstr *MI) override;
+
+protected:
+ bool doInitialization(Module &M) override;
+ bool doFinalization(Module &M) override;
+
+ /// Create NVPTX-specific DwarfDebug handler.
+ DwarfDebug *createDwarfDebug() override;
+
+private:
+ bool GlobalsEmitted;
+
+ // This is specific per MachineFunction.
+ const MachineRegisterInfo *MRI;
+
+ // The number assigned to each virtual register within its class, populated
+ // by setAndEmitFunctionVirtualRegisters and cleared between functions.
+ using VRegMap = DenseMap<Register, unsigned>;
+ using VRegRCMap = DenseMap<const TargetRegisterClass *, VRegMap>;
+ VRegRCMap VRegMapping;
+
+ // List of variables demoted to a function scope.
+ std::map<const Function *, std::vector<const GlobalVariable *>> localDecls;
+
+ void emitPTXGlobalVariable(const GlobalVariable *GVar, raw_ostream &O,
+ const NVPTXSubtarget &STI);
+ void emitPTXGlobalVariableDefinition(const GlobalVariable *GVar,
+ raw_ostream &O,
+ const NVPTXSubtarget &STI,
+ bool EmitInitializer);
+ void emitPTXAddressSpace(unsigned int AddressSpace, raw_ostream &O) const;
+ std::string getPTXFundamentalTypeStr(Type *Ty, bool = true) const;
+ void printScalarConstant(const Constant *CPV, raw_ostream &O);
+ void printFPConstant(const ConstantFP *Fp, raw_ostream &O) const;
+ void bufferLEByte(const Constant *CPV, int Bytes, AggBuffer *aggBuffer);
+ void bufferAggregateConstant(const Constant *CV, AggBuffer *aggBuffer);
+ void bufferAggregateConstVec(const ConstantVector *CV, AggBuffer *aggBuffer);
+
+ void emitLinkageDirective(const GlobalValue *V, raw_ostream &O);
+ void emitDeclarations(const Module &, raw_ostream &O);
+ void emitDeclaration(const Function *, raw_ostream &O);
+ void emitAliasDeclaration(const GlobalAlias *, raw_ostream &O);
+ void emitDeclarationWithName(const Function *, MCSymbol *, raw_ostream &O);
+ void emitDemotedVars(const Function *, raw_ostream &);
+
+ bool isLoopHeaderOfNoUnroll(const MachineBasicBlock &MBB) const;
+
+ // Used to control the need to emit .generic() in the initializer of
+ // module scope variables.
+ // Although ptx supports the hybrid mode like the following,
+ // .global .u32 a;
+ // .global .u32 b;
+ // .global .u32 addr[] = {a, generic(b)}
+ // we have
diff iculty representing the
diff erence in the NVVM IR.
+ //
+ // Since the address value should always be generic in CUDA C and always
+ // be specific in OpenCL, we use this simple control here.
+ //
+ const bool EmitGeneric;
+
+public:
+ NVPTXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
+ : AsmPrinter(TM, std::move(Streamer), ID),
+ EmitGeneric(static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() ==
+ NVPTX::CUDA) {}
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<MachineLoopInfoWrapperPass>();
+ AsmPrinter::getAnalysisUsage(AU);
+ }
+
+ std::string getVirtualRegisterName(Register Reg) const;
+
+ const MCSymbol *getFunctionFrameSymbol() const override;
+
+ // Make emitGlobalVariable() no-op for NVPTX.
+ // Global variables have been already emitted by the time the base AsmPrinter
+ // attempts to do so in doFinalization() (see NVPTXAsmPrinter::emitGlobals()).
+ void emitGlobalVariable(const GlobalVariable *GV) override {}
+};
+
+} // end anonymous namespace
+
static StringRef getTextureName(const Value &V) {
assert(V.hasName() && "Found texture variable with no name");
return V.getName();
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
deleted file mode 100644
index bea7fc9efc8ae..0000000000000
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ /dev/null
@@ -1,319 +0,0 @@
-//===-- NVPTXAsmPrinter.h - NVPTX LLVM assembly writer ----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains a printer that converts from our internal representation
-// of machine-dependent LLVM code to NVPTX assembly language.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NVPTX_NVPTXASMPRINTER_H
-#define LLVM_LIB_TARGET_NVPTX_NVPTXASMPRINTER_H
-
-#include "NVPTX.h"
-#include "NVPTXSubtarget.h"
-#include "NVPTXTargetMachine.h"
-#include "NVVMProperties.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/MachineJumpTableInfo.h"
-#include "llvm/CodeGen/MachineLoopInfo.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DebugLoc.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/GlobalAlias.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Value.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbol.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetMachine.h"
-#include <algorithm>
-#include <cassert>
-#include <map>
-#include <memory>
-#include <string>
-#include <type_traits>
-#include <vector>
-
-// The ptx syntax and format is very
diff erent from that usually seem in a .s
-// file,
-// therefore we are not able to use the MCAsmStreamer interface here.
-//
-// We are handcrafting the output method here.
-//
-// A better approach is to clone the MCAsmStreamer to a MCPTXAsmStreamer
-// (subclass of MCStreamer).
-
-namespace llvm {
-
-class MCOperand;
-class NVPTXTargetStreamer;
-
-class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
-
- class AggBuffer {
- // Used to buffer the emitted string for initializing global aggregates.
- //
- // Normally an aggregate (array, vector, or structure) is emitted as a u8[].
- // However, if either element/field of the aggregate is a non-NULL address,
- // and all such addresses are properly aligned, then the aggregate is
- // emitted as u32[] or u64[]. In the case of unaligned addresses, the
- // aggregate is emitted as u8[], and the mask() operator is used for all
- // pointers.
- //
- // We first layout the aggregate in 'buffer' in bytes, except for those
- // symbol addresses. For the i-th symbol address in the aggregate, its
- // corresponding 4-byte or 8-byte elements in 'buffer' are filled with 0s.
- // symbolPosInBuffer[i-1] records its position in 'buffer', and Symbols[i-1]
- // records the Value*.
- //
- // Once we have this AggBuffer setup, we can choose how to print it out.
- public:
- // number of symbol addresses
- unsigned numSymbols() const { return Symbols.size(); }
-
- bool allSymbolsAligned(unsigned ptrSize) const {
- return llvm::all_of(symbolPosInBuffer,
- [=](unsigned pos) { return pos % ptrSize == 0; });
- }
-
- private:
- const unsigned Size; // size of the buffer in bytes
- std::vector<unsigned char> buffer; // the buffer
- SmallVector<unsigned, 4> symbolPosInBuffer;
- SmallVector<const Value *, 4> Symbols;
- // SymbolsBeforeStripping[i] is the original form of Symbols[i] before
- // stripping pointer casts, i.e.,
- // Symbols[i] == SymbolsBeforeStripping[i]->stripPointerCasts().
- //
- // We need to keep these values because AggBuffer::print decides whether to
- // emit a "generic()" cast for Symbols[i] depending on the address space of
- // SymbolsBeforeStripping[i].
- SmallVector<const Value *, 4> SymbolsBeforeStripping;
- unsigned curpos;
- const NVPTXAsmPrinter &AP;
- const bool EmitGeneric;
-
- public:
- AggBuffer(unsigned Size, const NVPTXAsmPrinter &AP)
- : Size(Size), buffer(Size), curpos(0), AP(AP),
- EmitGeneric(AP.EmitGeneric) {}
-
- unsigned getBufferSize() const { return Size; }
-
- // Number of bytes written so far.
- unsigned getCurpos() const { return curpos; }
-
- // Copy Num bytes from Ptr.
- // if Bytes > Num, zero fill up to Bytes.
- void addBytes(const unsigned char *Ptr, unsigned Num, unsigned Bytes) {
- for (unsigned I : llvm::seq(Num))
- addByte(Ptr[I]);
- if (Bytes > Num)
- addZeros(Bytes - Num);
- }
-
- void addByte(uint8_t Byte) {
- assert(curpos < Size);
- buffer[curpos] = Byte;
- curpos++;
- }
-
- void addZeros(unsigned Num) {
- for ([[maybe_unused]] unsigned _ : llvm::seq(Num)) {
- addByte(0);
- }
- }
-
- void addSymbol(const Value *GVar, const Value *GVarBeforeStripping) {
- symbolPosInBuffer.push_back(curpos);
- Symbols.push_back(GVar);
- SymbolsBeforeStripping.push_back(GVarBeforeStripping);
- }
-
- void printBytes(raw_ostream &os);
- void printWords(raw_ostream &os);
-
- private:
- void printSymbol(unsigned nSym, raw_ostream &os);
- };
-
- friend class AggBuffer;
-
-public:
- static char ID;
-
- StringRef getPassName() const override { return "NVPTX Assembly Printer"; }
-
-private:
- const Function *F;
-
- NVPTXTargetStreamer *getTargetStreamer() const;
-
- void emitStartOfAsmFile(Module &M) override;
- void emitBasicBlockStart(const MachineBasicBlock &MBB) override;
- void emitFunctionEntryLabel() override;
- void emitFunctionBodyStart() override;
- void emitFunctionBodyEnd() override;
- void emitImplicitDef(const MachineInstr *MI) const override;
-
- void emitInstruction(const MachineInstr *) override;
- void lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
- MCOperand lowerOperand(const MachineOperand &MO);
- MCOperand GetSymbolRef(const MCSymbol *Symbol);
- MCRegister encodeVirtualRegister(Register Reg);
-
- /// The number \p Reg was assigned within its register class, as declared by
- /// this function's .reg directives.
- unsigned getVirtualRegisterNumber(Register Reg) const;
-
- void printMemOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O,
- const char *Modifier = nullptr);
- void printModuleLevelGV(const GlobalVariable *GVar, raw_ostream &O,
- bool processDemoted, const NVPTXSubtarget &STI);
- void emitGlobals(const Module &M);
- void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
- void emitHeader(Module &M, const NVPTXSubtarget &STI);
- void emitKernelFunctionDirectives(const Function &F, raw_ostream &O) const;
- void emitFunctionParamList(const Function *, raw_ostream &O);
- void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF);
- void encodeDebugInfoRegisterNumbers(const MachineFunction &MF);
- void printReturnValStr(const Function *, raw_ostream &O);
- void printReturnValStr(const MachineFunction &MF, raw_ostream &O);
- void emitCallPrototype(const CallBase &CB, unsigned UniqueCallSite,
- raw_ostream &O) const;
- void emitJumpTable(const MachineJumpTableEntry &MJT, unsigned MJTI) const;
-
- /// Should a .noreturn directive be emitted for \p V, which is either a
- /// function or a call site?
- template <typename T> bool shouldEmitPTXNoReturn(const T &V) const {
- static_assert(std::is_same_v<Function, T> || std::is_base_of_v<CallBase, T>,
- "expected a function or a call site");
-
- const auto &NTM = static_cast<const NVPTXTargetMachine &>(TM);
- if (!NTM.getSubtargetImpl()->hasNoReturn())
- return false;
-
- if (!V.doesNotReturn() || !V.getFunctionType()->getReturnType()->isVoidTy())
- return false;
-
- if constexpr (std::is_same_v<Function, T>)
- return !isKernelFunction(V);
- else
- return true;
- }
-
- bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
- const char *ExtraCode, raw_ostream &) override;
- void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
- bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
- const char *ExtraCode, raw_ostream &) override;
-
- const MCExpr *lowerConstantForGV(const Constant *CV,
- bool ProcessingGeneric) const;
- void printMCExpr(const MCExpr &Expr, raw_ostream &OS) const;
- /// Emit a blob of inline asm to the output streamer.
- void emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
- const MCTargetOptions &MCOptions, const MDNode *LocMDNode,
- InlineAsm::AsmDialect Dialect,
- const MachineInstr *MI) override;
-
-protected:
- bool doInitialization(Module &M) override;
- bool doFinalization(Module &M) override;
-
- /// Create NVPTX-specific DwarfDebug handler.
- DwarfDebug *createDwarfDebug() override;
-
-private:
- bool GlobalsEmitted;
-
- // This is specific per MachineFunction.
- const MachineRegisterInfo *MRI;
-
- // The number assigned to each virtual register within its class, populated
- // by setAndEmitFunctionVirtualRegisters and cleared between functions.
- using VRegMap = DenseMap<Register, unsigned>;
- using VRegRCMap = DenseMap<const TargetRegisterClass *, VRegMap>;
- VRegRCMap VRegMapping;
-
- // List of variables demoted to a function scope.
- std::map<const Function *, std::vector<const GlobalVariable *>> localDecls;
-
- void emitPTXGlobalVariable(const GlobalVariable *GVar, raw_ostream &O,
- const NVPTXSubtarget &STI);
- void emitPTXGlobalVariableDefinition(const GlobalVariable *GVar,
- raw_ostream &O,
- const NVPTXSubtarget &STI,
- bool EmitInitializer);
- void emitPTXAddressSpace(unsigned int AddressSpace, raw_ostream &O) const;
- std::string getPTXFundamentalTypeStr(Type *Ty, bool = true) const;
- void printScalarConstant(const Constant *CPV, raw_ostream &O);
- void printFPConstant(const ConstantFP *Fp, raw_ostream &O) const;
- void bufferLEByte(const Constant *CPV, int Bytes, AggBuffer *aggBuffer);
- void bufferAggregateConstant(const Constant *CV, AggBuffer *aggBuffer);
- void bufferAggregateConstVec(const ConstantVector *CV, AggBuffer *aggBuffer);
-
- void emitLinkageDirective(const GlobalValue *V, raw_ostream &O);
- void emitDeclarations(const Module &, raw_ostream &O);
- void emitDeclaration(const Function *, raw_ostream &O);
- void emitAliasDeclaration(const GlobalAlias *, raw_ostream &O);
- void emitDeclarationWithName(const Function *, MCSymbol *, raw_ostream &O);
- void emitDemotedVars(const Function *, raw_ostream &);
-
- bool isLoopHeaderOfNoUnroll(const MachineBasicBlock &MBB) const;
-
- // Used to control the need to emit .generic() in the initializer of
- // module scope variables.
- // Although ptx supports the hybrid mode like the following,
- // .global .u32 a;
- // .global .u32 b;
- // .global .u32 addr[] = {a, generic(b)}
- // we have
diff iculty representing the
diff erence in the NVVM IR.
- //
- // Since the address value should always be generic in CUDA C and always
- // be specific in OpenCL, we use this simple control here.
- //
- const bool EmitGeneric;
-
-public:
- NVPTXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
- : AsmPrinter(TM, std::move(Streamer), ID),
- EmitGeneric(static_cast<NVPTXTargetMachine &>(TM).getDrvInterface() ==
- NVPTX::CUDA) {}
-
- bool runOnMachineFunction(MachineFunction &F) override;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<MachineLoopInfoWrapperPass>();
- AsmPrinter::getAnalysisUsage(AU);
- }
-
- std::string getVirtualRegisterName(Register Reg) const;
-
- const MCSymbol *getFunctionFrameSymbol() const override;
-
- // Make emitGlobalVariable() no-op for NVPTX.
- // Global variables have been already emitted by the time the base AsmPrinter
- // attempts to do so in doFinalization() (see NVPTXAsmPrinter::emitGlobals()).
- void emitGlobalVariable(const GlobalVariable *GV) override {}
-};
-
-} // end namespace llvm
-
-#endif // LLVM_LIB_TARGET_NVPTX_NVPTXASMPRINTER_H
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index aa961ee6dd951..a0e3b1cc8e47f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -10,17 +10,25 @@
//
//===----------------------------------------------------------------------===//
-#include "NVPTXISelDAGToDAG.h"
+#include "MCTargetDesc/NVPTXBaseInfo.h"
#include "NVPTX.h"
+#include "NVPTXISelLowering.h"
+#include "NVPTXSelectionDAGInfo.h"
+#include "NVPTXTargetMachine.h"
#include "NVPTXUtilities.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAG.h"
+#include "llvm/CodeGen/SelectionDAGISel.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/CommandLine.h"
@@ -45,6 +53,104 @@ static cl::opt<bool> EnableMADWide("nvptx-mad-wide-opt", cl::init(false),
cl::Hidden,
cl::desc("Enable MAD wide optimization"));
+namespace {
+
+struct NVPTXScopes {
+ NVPTXScopes() = default;
+ NVPTXScopes(LLVMContext &C);
+ NVPTX::Scope operator[](SyncScope::ID ID) const;
+ bool empty() const;
+
+private:
+ SmallMapVector<SyncScope::ID, NVPTX::Scope, 8> Scopes{};
+ LLVMContext *Context = nullptr;
+};
+
+class NVPTXDAGToDAGISel : public SelectionDAGISel {
+ const NVPTXTargetMachine &TM;
+
+ NVPTX::DivPrecisionLevel getDivF32Level(const SDNode *N) const;
+ bool usePrecSqrtF32(const SDNode *N) const;
+ bool useF32FTZ() const;
+ bool allowFMA() const;
+ bool doRsqrtOpt() const;
+ bool doMADWideOpt() const;
+
+ NVPTXScopes Scopes{};
+
+public:
+ NVPTXDAGToDAGISel() = delete;
+
+ explicit NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, CodeGenOptLevel OptLevel);
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ const NVPTXSubtarget *Subtarget = nullptr;
+
+ bool SelectInlineAsmMemoryOperand(const SDValue &Op,
+ InlineAsm::ConstraintCode ConstraintID,
+ std::vector<SDValue> &OutOps) override;
+
+private:
+// Include the pieces autogenerated from the target description.
+#include "NVPTXGenDAGISel.inc"
+
+ void Select(SDNode *N) override;
+ bool tryIntrinsicChain(SDNode *N);
+ bool tryIntrinsicVoid(SDNode *N);
+ void SelectTexSurfHandle(SDNode *N);
+ bool tryLoad(SDNode *N);
+ bool tryLoadVector(SDNode *N);
+ bool tryLDU(SDNode *N);
+ bool tryLDG(MemSDNode *N);
+ bool tryStore(SDNode *N);
+ bool tryStoreVector(SDNode *N);
+ bool tryFence(SDNode *N);
+ bool tryBFE(SDNode *N);
+ bool tryBF16ArithToFMA(SDNode *N);
+ bool tryConstantFP(SDNode *N);
+ bool SelectSETP_F16X2(SDNode *N);
+ bool SelectSETP_BF16X2(SDNode *N);
+ bool tryUNPACK_VECTOR(SDNode *N);
+ bool tryEXTRACT_VECTOR_ELEMENT(SDNode *N);
+ void SelectV2I64toI128(SDNode *N);
+ void SelectI128toV2I64(SDNode *N);
+ void SelectCpAsyncBulkTensorReduceCommon(SDNode *N, unsigned RedOp,
+ bool IsIm2Col = false);
+ void SelectTcgen05Ld(SDNode *N, bool hasOffset = false);
+ void SelectTcgen05St(SDNode *N, bool hasOffset = false);
+ void selectAtomicSwap128(SDNode *N);
+
+ inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
+ return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
+ }
+ NVPTX::Ordering getMemOrder(const MemSDNode *N) const;
+ NVPTX::Scope getAtomicScope(const MemSDNode *N) const;
+
+ bool SelectADDR(SDValue Addr, SDValue &Base, SDValue &Offset);
+ SDValue getPTXCmpMode(const CondCodeSDNode &CondCode);
+ SDValue selectPossiblyImm(SDValue V);
+
+ // Returns the Memory Order and Scope that the PTX memory instruction should
+ // use, and inserts appropriate fence instruction before the memory
+ // instruction, if needed to implement the instructions memory order. Required
+ // fences after the instruction need to be handled elsewhere.
+ std::pair<NVPTX::Ordering, NVPTX::Scope>
+ insertMemoryInstructionFence(SDLoc DL, SDValue &Chain, MemSDNode *N);
+ NVPTX::Scope getOperationScope(MemSDNode *N, NVPTX::Ordering O) const;
+
+public:
+ static NVPTX::AddressSpace getAddrSpace(const MemSDNode *N);
+};
+
+class NVPTXDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
+public:
+ static char ID;
+ explicit NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm,
+ CodeGenOptLevel OptLevel);
+};
+
+} // end anonymous namespace
+
/// createNVPTXISelDag - This pass converts a legalized DAG into a
/// NVPTX-specific DAG, ready for instruction scheduling.
FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
@@ -1232,16 +1338,6 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
return true;
}
-unsigned NVPTXDAGToDAGISel::getFromTypeWidthForLoad(const MemSDNode *Mem) {
- auto TotalWidth = Mem->getMemoryVT().getSizeInBits();
- auto NumElts = Mem->getNumValues() - 1;
- auto ElementBitWidth = TotalWidth / NumElts;
- assert(isPowerOf2_32(ElementBitWidth) && ElementBitWidth >= 8 &&
- ElementBitWidth <= 128 && TotalWidth <= 256 &&
- "Invalid width for load");
- return ElementBitWidth;
-}
-
bool NVPTXDAGToDAGISel::tryLDU(SDNode *N) {
auto *LD = cast<MemSDNode>(N);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
deleted file mode 100644
index cfb65f9381b69..0000000000000
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ /dev/null
@@ -1,127 +0,0 @@
-//===-- NVPTXISelDAGToDAG.h - A dag to dag inst selector for NVPTX --------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines an instruction selector for the NVPTX target.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NVPTX_NVPTXISELDAGTODAG_H
-#define LLVM_LIB_TARGET_NVPTX_NVPTXISELDAGTODAG_H
-
-#include "MCTargetDesc/NVPTXBaseInfo.h"
-#include "NVPTX.h"
-#include "NVPTXISelLowering.h"
-#include "NVPTXRegisterInfo.h"
-#include "NVPTXSelectionDAGInfo.h"
-#include "NVPTXTargetMachine.h"
-#include "llvm/ADT/MapVector.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
-#include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/Support/Compiler.h"
-
-namespace llvm {
-
-struct NVPTXScopes {
- NVPTXScopes() = default;
- NVPTXScopes(LLVMContext &C);
- NVPTX::Scope operator[](SyncScope::ID ID) const;
- bool empty() const;
-
-private:
- SmallMapVector<SyncScope::ID, NVPTX::Scope, 8> Scopes{};
- LLVMContext *Context = nullptr;
-};
-
-class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
- const NVPTXTargetMachine &TM;
-
- NVPTX::DivPrecisionLevel getDivF32Level(const SDNode *N) const;
- bool usePrecSqrtF32(const SDNode *N) const;
- bool useF32FTZ() const;
- bool allowFMA() const;
- bool doRsqrtOpt() const;
- bool doMADWideOpt() const;
-
- NVPTXScopes Scopes{};
-
-public:
- NVPTXDAGToDAGISel() = delete;
-
- explicit NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, CodeGenOptLevel OptLevel);
-
- bool runOnMachineFunction(MachineFunction &MF) override;
- const NVPTXSubtarget *Subtarget = nullptr;
-
- bool SelectInlineAsmMemoryOperand(const SDValue &Op,
- InlineAsm::ConstraintCode ConstraintID,
- std::vector<SDValue> &OutOps) override;
-
-private:
-// Include the pieces autogenerated from the target description.
-#include "NVPTXGenDAGISel.inc"
-
- void Select(SDNode *N) override;
- bool tryIntrinsicChain(SDNode *N);
- bool tryIntrinsicVoid(SDNode *N);
- void SelectTexSurfHandle(SDNode *N);
- bool tryLoad(SDNode *N);
- bool tryLoadVector(SDNode *N);
- bool tryLDU(SDNode *N);
- bool tryLDG(MemSDNode *N);
- bool tryStore(SDNode *N);
- bool tryStoreVector(SDNode *N);
- bool tryFence(SDNode *N);
- bool tryBFE(SDNode *N);
- bool tryBF16ArithToFMA(SDNode *N);
- bool tryConstantFP(SDNode *N);
- bool SelectSETP_F16X2(SDNode *N);
- bool SelectSETP_BF16X2(SDNode *N);
- bool tryUNPACK_VECTOR(SDNode *N);
- bool tryEXTRACT_VECTOR_ELEMENT(SDNode *N);
- void SelectV2I64toI128(SDNode *N);
- void SelectI128toV2I64(SDNode *N);
- void SelectCpAsyncBulkTensorReduceCommon(SDNode *N, unsigned RedOp,
- bool IsIm2Col = false);
- void SelectTcgen05Ld(SDNode *N, bool hasOffset = false);
- void SelectTcgen05St(SDNode *N, bool hasOffset = false);
- void selectAtomicSwap128(SDNode *N);
-
- inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
- return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
- }
- NVPTX::Ordering getMemOrder(const MemSDNode *N) const;
- NVPTX::Scope getAtomicScope(const MemSDNode *N) const;
-
- bool SelectADDR(SDValue Addr, SDValue &Base, SDValue &Offset);
- SDValue getPTXCmpMode(const CondCodeSDNode &CondCode);
- SDValue selectPossiblyImm(SDValue V);
-
- // Returns the Memory Order and Scope that the PTX memory instruction should
- // use, and inserts appropriate fence instruction before the memory
- // instruction, if needed to implement the instructions memory order. Required
- // fences after the instruction need to be handled elsewhere.
- std::pair<NVPTX::Ordering, NVPTX::Scope>
- insertMemoryInstructionFence(SDLoc DL, SDValue &Chain, MemSDNode *N);
- NVPTX::Scope getOperationScope(MemSDNode *N, NVPTX::Ordering O) const;
-
-public:
- static NVPTX::AddressSpace getAddrSpace(const MemSDNode *N);
- static unsigned getFromTypeWidthForLoad(const MemSDNode *Mem);
-};
-
-class NVPTXDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
-public:
- static char ID;
- explicit NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm,
- CodeGenOptLevel OptLevel);
-};
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index f45fd02f406dd..29921ff86b352 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -14,7 +14,6 @@
#include "NVPTXISelLowering.h"
#include "MCTargetDesc/NVPTXBaseInfo.h"
#include "NVPTX.h"
-#include "NVPTXISelDAGToDAG.h"
#include "NVPTXMachineFunctionInfo.h"
#include "NVPTXSelectionDAGInfo.h"
#include "NVPTXSubtarget.h"
@@ -7779,7 +7778,7 @@ static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known) {
return;
assert(Known.getBitWidth() == DestVT.getSizeInBits());
- auto ElementBitWidth = NVPTXDAGToDAGISel::getFromTypeWidthForLoad(LD);
+ auto ElementBitWidth = getFromTypeWidthForLoad(LD);
Known.Zero.setHighBits(Known.getBitWidth() - ElementBitWidth);
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index b871de0e6ee70..453a63ef24b8b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -11,8 +11,8 @@
//===----------------------------------------------------------------------===//
#include "NVPTXUtilities.h"
-#include "NVPTX.h"
#include "NVVMProperties.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Function.h"
@@ -32,6 +32,16 @@ Function *llvm::getMaybeBitcastedCallee(const CallBase *CB) {
return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
}
+unsigned llvm::getFromTypeWidthForLoad(const MemSDNode *Mem) {
+ auto TotalWidth = Mem->getMemoryVT().getSizeInBits();
+ auto NumElts = Mem->getNumValues() - 1;
+ auto ElementBitWidth = TotalWidth / NumElts;
+ assert(isPowerOf2_32(ElementBitWidth) && ElementBitWidth >= 8 &&
+ ElementBitWidth <= 128 && TotalWidth <= 256 &&
+ "Invalid width for load");
+ return ElementBitWidth;
+}
+
Align llvm::getPTXParamTypeAlign(Type *ArgTy, const DataLayout &DL) {
// Capping the alignment to 128 bytes as that is the maximum alignment
// supported by PTX.
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index a190a3cba42d1..ae3692fecafae 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -14,11 +14,10 @@
#define LLVM_LIB_TARGET_NVPTX_NVPTXUTILITIES_H
#include "NVPTX.h"
-#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Function.h"
-#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Alignment.h"
#include "llvm/Support/FormatVariadic.h"
@@ -28,9 +27,14 @@
namespace llvm {
class DataLayout;
+class MemSDNode;
Function *getMaybeBitcastedCallee(const CallBase *CB);
+/// The bit-width of a single element loaded by \p Mem, i.e. the width used for
+/// the ".fromtype" part of the emitted PTX load.
+unsigned getFromTypeWidthForLoad(const MemSDNode *Mem);
+
/// ABI alignment of \p ArgTy in .param space, capped at the PTX maximum of 128.
Align getPTXParamTypeAlign(Type *ArgTy, const DataLayout &DL);
More information about the llvm-commits
mailing list