[llvm] [NVPTX] Rip out vestigial variadic support (NFC) (PR #202385)
Alex MacLean via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 09:40:27 PDT 2026
https://github.com/AlexMaclean created https://github.com/llvm/llvm-project/pull/202385
None
>From 824fab3c9523701d3c6078178dfe281baf71964a Mon Sep 17 00:00:00 2001
From: Alex Maclean <amaclean at nvidia.com>
Date: Mon, 8 Jun 2026 09:39:24 -0700
Subject: [PATCH] [NVPTX] Rip out vestigial variadic support (NFC)
---
llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 263 ++++++++--------
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 331 +++++---------------
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 13 +-
llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 8 -
4 files changed, 216 insertions(+), 399 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index d7b34abb0127c..cd8ab2b29d952 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1360,158 +1360,151 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F, raw_ostream &O) {
const NVPTXMachineFunctionInfo *MFI =
MF ? MF->getInfo<NVPTXMachineFunctionInfo>() : nullptr;
- bool IsFirst = true;
const bool IsKernelFunc = isKernelFunction(*F);
- if (F->arg_empty() && !F->isVarArg()) {
+ assert(!F->isVarArg() && "VarArg functions lowered in ExpandVariadics");
+
+ if (F->arg_empty()) {
O << "()";
return;
}
O << "(\n";
- for (const Argument &Arg : F->args()) {
- Type *Ty = Arg.getType();
- const std::string ParamSym = TLI->getParamName(F, Arg.getArgNo());
-
- if (!IsFirst)
- O << ",\n";
-
- IsFirst = false;
-
- // Handle image/sampler parameters
- if (IsKernelFunc) {
- const PTXOpaqueType ArgOpaqueType = getPTXOpaqueType(Arg);
- if (ArgOpaqueType != PTXOpaqueType::None) {
- const bool EmitImgPtr = !MFI || !MFI->checkImageHandleSymbol(ParamSym);
- O << "\t.param ";
- if (EmitImgPtr)
- O << ".u64 .ptr ";
+ interleave(
+ F->args(), O,
+ [&](const Argument &Arg) {
+ Type *Ty = Arg.getType();
+ const std::string ParamSym = TLI->getParamName(F, Arg.getArgNo());
+
+ // Handle image/sampler parameters
+ if (IsKernelFunc) {
+ const PTXOpaqueType ArgOpaqueType = getPTXOpaqueType(Arg);
+ if (ArgOpaqueType != PTXOpaqueType::None) {
+ const bool EmitImgPtr =
+ !MFI || !MFI->checkImageHandleSymbol(ParamSym);
+ O << "\t.param ";
+ if (EmitImgPtr)
+ O << ".u64 .ptr ";
+
+ switch (ArgOpaqueType) {
+ case PTXOpaqueType::Sampler:
+ O << ".samplerref ";
+ break;
+ case PTXOpaqueType::Texture:
+ O << ".texref ";
+ break;
+ case PTXOpaqueType::Surface:
+ O << ".surfref ";
+ break;
+ case PTXOpaqueType::None:
+ llvm_unreachable("handled above");
+ }
+ O << ParamSym;
+ return;
+ }
+ }
- switch (ArgOpaqueType) {
- case PTXOpaqueType::Sampler:
- O << ".samplerref ";
- break;
- case PTXOpaqueType::Texture:
- O << ".texref ";
- break;
- case PTXOpaqueType::Surface:
- O << ".surfref ";
- break;
- case PTXOpaqueType::None:
- llvm_unreachable("handled above");
+ auto GetOptimalAlignForParam = [&DL, F, &Arg](Type *Ty) -> Align {
+ if (MaybeAlign StackAlign =
+ getAlign(*F, Arg.getArgNo() + AttributeList::FirstArgIndex))
+ return StackAlign.value();
+
+ Align TypeAlign = getFunctionParamOptimizedAlign(F, Ty, DL);
+ MaybeAlign ParamAlign =
+ Arg.hasByValAttr() ? Arg.getParamAlign() : MaybeAlign();
+ return std::max(TypeAlign, ParamAlign.valueOrOne());
+ };
+
+ if (Arg.hasByValAttr()) {
+ // param has byVal attribute.
+ Type *ETy = Arg.getParamByValType();
+ assert(ETy && "Param should have byval type");
+
+ // Print .param .align <a> .b8 .param[size];
+ // <a> = optimal alignment for the element type; always multiple of
+ // PAL.getParamAlignment
+ // size = typeallocsize of element type
+ const Align OptimalAlign =
+ IsKernelFunc ? GetOptimalAlignForParam(ETy)
+ : getFunctionByValParamAlign(
+ F, ETy, Arg.getParamAlign().valueOrOne(), DL);
+
+ O << "\t.param .align " << OptimalAlign.value() << " .b8 " << ParamSym
+ << "[" << DL.getTypeAllocSize(ETy) << "]";
+ return;
}
- O << ParamSym;
- continue;
- }
- }
- auto GetOptimalAlignForParam = [&DL, F, &Arg](Type *Ty) -> Align {
- if (MaybeAlign StackAlign =
- getAlign(*F, Arg.getArgNo() + AttributeList::FirstArgIndex))
- return StackAlign.value();
-
- Align TypeAlign = getFunctionParamOptimizedAlign(F, Ty, DL);
- MaybeAlign ParamAlign =
- Arg.hasByValAttr() ? Arg.getParamAlign() : MaybeAlign();
- return std::max(TypeAlign, ParamAlign.valueOrOne());
- };
-
- if (Arg.hasByValAttr()) {
- // param has byVal attribute.
- Type *ETy = Arg.getParamByValType();
- assert(ETy && "Param should have byval type");
-
- // Print .param .align <a> .b8 .param[size];
- // <a> = optimal alignment for the element type; always multiple of
- // PAL.getParamAlignment
- // size = typeallocsize of element type
- const Align OptimalAlign =
- IsKernelFunc ? GetOptimalAlignForParam(ETy)
- : getFunctionByValParamAlign(
- F, ETy, Arg.getParamAlign().valueOrOne(), DL);
-
- O << "\t.param .align " << OptimalAlign.value() << " .b8 " << ParamSym
- << "[" << DL.getTypeAllocSize(ETy) << "]";
- continue;
- }
+ if (shouldPassAsArray(Ty)) {
+ // Just print .param .align <a> .b8 .param[size];
+ // <a> = optimal alignment for the element type; always multiple of
+ // PAL.getParamAlignment
+ // size = typeallocsize of element type
+ Align OptimalAlign = GetOptimalAlignForParam(Ty);
- if (shouldPassAsArray(Ty)) {
- // Just print .param .align <a> .b8 .param[size];
- // <a> = optimal alignment for the element type; always multiple of
- // PAL.getParamAlignment
- // size = typeallocsize of element type
- Align OptimalAlign = GetOptimalAlignForParam(Ty);
+ O << "\t.param .align " << OptimalAlign.value() << " .b8 " << ParamSym
+ << "[" << DL.getTypeAllocSize(Ty) << "]";
- O << "\t.param .align " << OptimalAlign.value() << " .b8 " << ParamSym
- << "[" << DL.getTypeAllocSize(Ty) << "]";
+ return;
+ }
+ // Just a scalar
+ auto *PTy = dyn_cast<PointerType>(Ty);
+ unsigned PTySizeInBits = 0;
+ if (PTy) {
+ PTySizeInBits =
+ TLI->getPointerTy(DL, PTy->getAddressSpace()).getSizeInBits();
+ assert(PTySizeInBits && "Invalid pointer size");
+ }
- continue;
- }
- // Just a scalar
- auto *PTy = dyn_cast<PointerType>(Ty);
- unsigned PTySizeInBits = 0;
- if (PTy) {
- PTySizeInBits =
- TLI->getPointerTy(DL, PTy->getAddressSpace()).getSizeInBits();
- assert(PTySizeInBits && "Invalid pointer size");
- }
+ if (IsKernelFunc) {
+ if (PTy) {
+ O << "\t.param .u" << PTySizeInBits << " .ptr";
+
+ switch (PTy->getAddressSpace()) {
+ default:
+ break;
+ case ADDRESS_SPACE_GLOBAL:
+ O << " .global";
+ break;
+ case ADDRESS_SPACE_SHARED:
+ O << " .shared";
+ break;
+ case ADDRESS_SPACE_CONST:
+ O << " .const";
+ break;
+ case ADDRESS_SPACE_LOCAL:
+ O << " .local";
+ break;
+ }
- if (IsKernelFunc) {
- if (PTy) {
- O << "\t.param .u" << PTySizeInBits << " .ptr";
+ O << " .align " << Arg.getParamAlign().valueOrOne().value() << " "
+ << ParamSym;
+ return;
+ }
- switch (PTy->getAddressSpace()) {
- default:
- break;
- case ADDRESS_SPACE_GLOBAL:
- O << " .global";
- break;
- case ADDRESS_SPACE_SHARED:
- O << " .shared";
- break;
- case ADDRESS_SPACE_CONST:
- O << " .const";
- break;
- case ADDRESS_SPACE_LOCAL:
- O << " .local";
- break;
+ // non-pointer scalar to kernel func
+ O << "\t.param .";
+ // Special case: predicate operands become .u8 types
+ if (Ty->isIntegerTy(1))
+ O << "u8";
+ else
+ O << getPTXFundamentalTypeStr(Ty);
+ O << " " << ParamSym;
+ return;
}
-
- O << " .align " << Arg.getParamAlign().valueOrOne().value() << " "
- << ParamSym;
- continue;
- }
-
- // non-pointer scalar to kernel func
- O << "\t.param .";
- // Special case: predicate operands become .u8 types
- if (Ty->isIntegerTy(1))
- O << "u8";
- else
- O << getPTXFundamentalTypeStr(Ty);
- O << " " << ParamSym;
- continue;
- }
- // Non-kernel function, just print .param .b<size> for ABI
- // and .reg .b<size> for non-ABI
- unsigned Size;
- if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
- Size = promoteScalarArgumentSize(ITy->getBitWidth());
- } else if (PTy) {
- assert(PTySizeInBits && "Invalid pointer size");
- Size = PTySizeInBits;
- } else
- Size = Ty->getPrimitiveSizeInBits();
- O << "\t.param .b" << Size << " " << ParamSym;
- }
-
- if (F->isVarArg()) {
- if (!IsFirst)
- O << ",\n";
- O << "\t.param .align " << STI.getMaxRequiredAlignment() << " .b8 "
- << TLI->getParamName(F, /* vararg */ -1) << "[]";
- }
+ // Non-kernel function, just print .param .b<size> for ABI
+ // and .reg .b<size> for non-ABI
+ unsigned Size;
+ if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
+ Size = promoteScalarArgumentSize(ITy->getBitWidth());
+ } else if (PTy) {
+ assert(PTySizeInBits && "Invalid pointer size");
+ Size = PTySizeInBits;
+ } else
+ Size = Ty->getPrimitiveSizeInBits();
+ O << "\t.param .b" << Size << " " << ParamSym;
+ },
+ ",\n");
O << "\n)";
}
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index e13911f87eed5..8d735fbee94cd 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -308,12 +308,10 @@ getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI,
static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
LLVMContext &Ctx, CallingConv::ID CallConv,
Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
- SmallVectorImpl<uint64_t> &Offsets,
- uint64_t StartingOffset = 0) {
+ SmallVectorImpl<TypeSize> &Offsets) {
SmallVector<EVT, 16> TempVTs;
- SmallVector<uint64_t, 16> TempOffsets;
- ComputeValueVTs(TLI, DL, Ty, TempVTs, /*MemVTs=*/nullptr, &TempOffsets,
- StartingOffset);
+ SmallVector<TypeSize, 16> TempOffsets;
+ ComputeValueVTs(TLI, DL, Ty, TempVTs, /*MemVTs=*/nullptr, &TempOffsets);
for (const auto [VT, Off] : zip(TempVTs, TempOffsets)) {
MVT RegisterVT = TLI.getRegisterTypeForCallingConv(Ctx, CallConv, VT);
@@ -428,10 +426,9 @@ static EVT promoteScalarIntegerPTX(const EVT VT) {
// parameter starting at index Idx using a single vectorized op of
// size AccessSize. If so, it returns the number of param pieces
// covered by the vector op. Otherwise, it returns 1.
-template <typename T>
static unsigned canMergeParamLoadStoresStartingAt(
unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs,
- const SmallVectorImpl<T> &Offsets, Align ParamAlignment) {
+ const SmallVectorImpl<TypeSize> &Offsets, Align ParamAlignment) {
// Can't vectorize if param alignment is not sufficient.
if (ParamAlignment < AccessSize)
@@ -481,17 +478,10 @@ static unsigned canMergeParamLoadStoresStartingAt(
// of the same size as ValueVTs indicating how each piece should be
// loaded/stored (i.e. as a scalar, or as part of a vector
// load/store).
-template <typename T>
static SmallVector<unsigned, 16>
VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs,
- const SmallVectorImpl<T> &Offsets, Align ParamAlignment,
- bool IsVAArg = false) {
- // Set vector size to match ValueVTs and mark all elements as
- // scalars by default.
-
- if (IsVAArg)
- return SmallVector<unsigned>(ValueVTs.size(), 1);
-
+ const SmallVectorImpl<TypeSize> &Offsets,
+ Align ParamAlignment) {
SmallVector<unsigned, 16> VectorInfo;
const auto GetNumElts = [&](unsigned I) -> unsigned {
@@ -806,12 +796,6 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
// DEBUGTRAP can be lowered to PTX brkpt
setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
- // Support varargs.
- setOperationAction(ISD::VASTART, MVT::Other, Custom);
- setOperationAction(ISD::VAARG, MVT::Other, Custom);
- setOperationAction(ISD::VACOPY, MVT::Other, Expand);
- setOperationAction(ISD::VAEND, MVT::Other, Expand);
-
setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX},
{MVT::i16, MVT::i32, MVT::i64}, Legal);
// PTX abs.s is undefined for INT_MIN, so ISD::ABS (which requires
@@ -1208,8 +1192,7 @@ static Align getArgumentAlignment(const CallBase *CB, Type *Ty, unsigned Idx,
std::string NVPTXTargetLowering::getPrototype(
const DataLayout &DL, Type *RetTy, const ArgListTy &Args,
- const SmallVectorImpl<ISD::OutputArg> &Outs,
- std::optional<unsigned> FirstVAArg, const CallBase &CB,
+ const SmallVectorImpl<ISD::OutputArg> &Outs, const CallBase &CB,
unsigned UniqueCallSite) const {
auto PtrVT = getPointerTy(DL);
@@ -1249,59 +1232,54 @@ std::string NVPTXTargetLowering::getPrototype(
}
O << "_ (";
- bool first = true;
-
- const unsigned NumArgs = FirstVAArg.value_or(Args.size());
auto AllOuts = ArrayRef(Outs);
- for (const unsigned I : llvm::seq(NumArgs)) {
- const auto ArgOuts =
- AllOuts.take_while([I](auto O) { return O.OrigArgIndex == I; });
- AllOuts = AllOuts.drop_front(ArgOuts.size());
+ interleave(
+ seq(Args.size()), O,
+ [&](const unsigned I) {
+ const auto ArgOuts =
+ AllOuts.take_while([I](auto O) { return O.OrigArgIndex == I; });
+ AllOuts = AllOuts.drop_front(ArgOuts.size());
+
+ Type *Ty = Args[I].Ty;
+
+ if (ArgOuts[0].Flags.isByVal()) {
+ // Indirect calls need strict ABI alignment so we disable
+ // optimizations by not providing a function to optimize.
+ Type *ETy = Args[I].IndirectType;
+ Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
+ Align ParamByValAlign =
+ getFunctionByValParamAlign(/*F=*/nullptr, ETy, InitialAlign, DL);
+
+ O << ".param .align " << ParamByValAlign.value() << " .b8 _["
+ << ArgOuts[0].Flags.getByValSize() << "]";
+ return;
+ }
- Type *Ty = Args[I].Ty;
- if (!first) {
- O << ", ";
- }
- first = false;
-
- if (ArgOuts[0].Flags.isByVal()) {
- // Indirect calls need strict ABI alignment so we disable optimizations by
- // not providing a function to optimize.
- Type *ETy = Args[I].IndirectType;
- Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
- Align ParamByValAlign =
- getFunctionByValParamAlign(/*F=*/nullptr, ETy, InitialAlign, DL);
-
- O << ".param .align " << ParamByValAlign.value() << " .b8 _["
- << ArgOuts[0].Flags.getByValSize() << "]";
- } else {
- if (shouldPassAsArray(Ty)) {
- Align ParamAlign =
- getArgumentAlignment(&CB, Ty, I + AttributeList::FirstArgIndex, DL);
- O << ".param .align " << ParamAlign.value() << " .b8 _["
- << DL.getTypeAllocSize(Ty) << "]";
- continue;
- }
- // i8 types in IR will be i16 types in SDAG
- assert((getValueType(DL, Ty) == ArgOuts[0].VT ||
- (getValueType(DL, Ty) == MVT::i8 && ArgOuts[0].VT == MVT::i16)) &&
- "type mismatch between callee prototype and arguments");
- // scalar type
- unsigned sz = 0;
- if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
- sz = promoteScalarArgumentSize(ITy->getBitWidth());
- } else if (isa<PointerType>(Ty)) {
- sz = PtrVT.getSizeInBits();
- } else {
- sz = Ty->getPrimitiveSizeInBits();
- }
- O << ".param .b" << sz << " _";
- }
- }
+ if (shouldPassAsArray(Ty)) {
+ Align ParamAlign = getArgumentAlignment(
+ &CB, Ty, I + AttributeList::FirstArgIndex, DL);
+ O << ".param .align " << ParamAlign.value() << " .b8 _["
+ << DL.getTypeAllocSize(Ty) << "]";
+ return;
+ }
+ // i8 types in IR will be i16 types in SDAG
+ assert(
+ (getValueType(DL, Ty) == ArgOuts[0].VT ||
+ (getValueType(DL, Ty) == MVT::i8 && ArgOuts[0].VT == MVT::i16)) &&
+ "type mismatch between callee prototype and arguments");
+ // scalar type
+ unsigned sz = 0;
+ if (auto *ITy = dyn_cast<IntegerType>(Ty)) {
+ sz = promoteScalarArgumentSize(ITy->getBitWidth());
+ } else if (isa<PointerType>(Ty)) {
+ sz = PtrVT.getSizeInBits();
+ } else {
+ sz = Ty->getPrimitiveSizeInBits();
+ }
+ O << ".param .b" << sz << " _";
+ },
+ ", ");
- if (FirstVAArg)
- O << (first ? "" : ",") << " .param .align "
- << STI.getMaxRequiredAlignment() << " .b8 _[]";
O << ")";
if (shouldEmitPTXNoReturn(&CB, *nvTM))
O << " .noreturn";
@@ -1391,10 +1369,7 @@ static SDValue correctParamType(SDValue V, EVT ExpectedVT,
SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
SmallVectorImpl<SDValue> &InVals) const {
- if (CLI.IsVarArg && (STI.getPTXVersion() < 60 || STI.getSmVersion() < 30))
- report_fatal_error(
- "Support for variadic functions (unsized array parameter) introduced "
- "in PTX ISA version 6.0 and requires target sm_30.");
+ assert(!CLI.IsVarArg && "Vararg functions lowered in ExpandVariadics");
SelectionDAG &DAG = CLI.DAG;
SDLoc dl = CLI.DL;
@@ -1440,32 +1415,11 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
return Declare;
};
- // Variadic arguments.
- //
- // Normally, for each argument, we declare a param scalar or a param
- // byte array in the .param space, and store the argument value to that
- // param scalar or array starting at offset 0.
- //
- // In the case of the first variadic argument, we declare a vararg byte array
- // with size 0. The exact size of this array isn't known at this point, so
- // it'll be patched later. All the variadic arguments will be stored to this
- // array at a certain offset (which gets tracked by 'VAOffset'). The offset is
- // initially set to 0, so it can be used for non-variadic arguments (which use
- // 0 offset) to simplify the code.
- //
- // After all vararg is processed, 'VAOffset' holds the size of the
- // vararg byte array.
- assert((CLI.IsVarArg || CLI.Args.size() == CLI.NumFixedArgs) &&
- "Non-VarArg function with extra arguments");
-
- const unsigned FirstVAArg = CLI.NumFixedArgs; // position of first variadic
- unsigned VAOffset = 0; // current offset in the param array
-
- const SDValue VADeclareParam =
- CLI.Args.size() > FirstVAArg
- ? MakeDeclareArrayParam(getCallParamSymbol(DAG, FirstVAArg, MVT::i32),
- Align(STI.getMaxRequiredAlignment()), 0)
- : SDValue();
+ // For each argument, we declare a param scalar or a param byte array in the
+ // .param space, and store the argument value to that param scalar or array
+ // starting at offset 0.
+ assert(CLI.Args.size() == CLI.NumFixedArgs &&
+ "function with extra arguments");
// Args.size() and Outs.size() need not match.
// Outs.size() will be larger
@@ -1491,11 +1445,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
AllOuts = AllOuts.drop_front(ArgOuts.size());
AllOutVals = AllOutVals.drop_front(ArgOuts.size());
- const bool IsVAArg = (ArgI >= FirstVAArg);
const bool IsByVal = Arg.IsByVal;
- const SDValue ParamSymbol =
- getCallParamSymbol(DAG, IsVAArg ? FirstVAArg : ArgI, MVT::i32);
+ const SDValue ParamSymbol = getCallParamSymbol(DAG, ArgI, MVT::i32);
assert((!IsByVal || Arg.IndirectType) &&
"byval arg must have indirect type");
@@ -1518,9 +1470,6 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
"type size mismatch");
const SDValue ArgDeclare = [&]() {
- if (IsVAArg)
- return VADeclareParam;
-
if (IsByVal || shouldPassAsArray(Arg.Ty))
return MakeDeclareArrayParam(ParamSymbol, ArgAlign, TySize);
@@ -1537,15 +1486,12 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
const auto PointerInfo = refinePtrAS(SrcPtr, DAG, DL, *this);
const Align BaseSrcAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
- if (IsVAArg)
- VAOffset = alignTo(VAOffset, ArgAlign);
-
SmallVector<EVT, 4> ValueVTs, MemVTs;
SmallVector<TypeSize, 4> Offsets;
ComputeValueVTs(*this, DL, ETy, ValueVTs, &MemVTs, &Offsets);
unsigned J = 0;
- const auto VI = VectorizePTXValueVTs(MemVTs, Offsets, ArgAlign, IsVAArg);
+ const auto VI = VectorizePTXValueVTs(MemVTs, Offsets, ArgAlign);
for (const unsigned NumElts : VI) {
EVT LoadVT = getVectorizedVT(MemVTs[J], NumElts, Ctx);
Align SrcAlign = commonAlignment(BaseSrcAlign, Offsets[J]);
@@ -1553,10 +1499,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
SDValue SrcLoad =
DAG.getLoad(LoadVT, dl, CallChain, SrcAddr, PointerInfo, SrcAlign);
- TypeSize ParamOffset = Offsets[J].getWithIncrement(VAOffset);
- Align ParamAlign = commonAlignment(ArgAlign, ParamOffset);
- SDValue ParamAddr =
- DAG.getObjectPtrOffset(dl, ParamSymbol, ParamOffset);
+ Align ParamAlign = commonAlignment(ArgAlign, Offsets[J]);
+ SDValue ParamAddr = DAG.getObjectPtrOffset(dl, ParamSymbol, Offsets[J]);
SDValue StoreParam = DAG.getStore(
ArgDeclare, dl, SrcLoad, ParamAddr,
MachinePointerInfo(NVPTX::AddressSpace::DeviceParam), ParamAlign);
@@ -1564,13 +1508,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
J += NumElts;
}
- if (IsVAArg)
- VAOffset += TySize;
} else {
SmallVector<EVT, 16> VTs;
- SmallVector<uint64_t, 16> Offsets;
- ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, Arg.Ty, VTs, Offsets,
- VAOffset);
+ SmallVector<TypeSize, 16> Offsets;
+ ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, Arg.Ty, VTs, Offsets);
assert(VTs.size() == Offsets.size() && "Size mismatch");
assert(VTs.size() == ArgOuts.size() && "Size mismatch");
@@ -1595,30 +1536,11 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
};
unsigned J = 0;
- const auto VI = VectorizePTXValueVTs(VTs, Offsets, ArgAlign, IsVAArg);
+ const auto VI = VectorizePTXValueVTs(VTs, Offsets, ArgAlign);
for (const unsigned NumElts : VI) {
- const EVT EltVT = promoteScalarIntegerPTX(VTs[J]);
-
- unsigned Offset;
- if (IsVAArg) {
- // TODO: We may need to support vector types that can be passed
- // as scalars in variadic arguments.
- assert(NumElts == 1 &&
- "Vectorization should be disabled for vaargs.");
+ TypeSize Offset = Offsets[J];
- // Align each part of the variadic argument to their type.
- VAOffset = alignTo(VAOffset, DAG.getEVTAlign(EltVT));
- Offset = VAOffset;
-
- const EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
- VAOffset += DL.getTypeAllocSize(TheStoreType.getTypeForEVT(Ctx));
- } else {
- assert(VAOffset == 0 && "VAOffset must be 0 for non-VA args");
- Offset = Offsets[J];
- }
-
- SDValue Ptr =
- DAG.getObjectPtrOffset(dl, ParamSymbol, TypeSize::getFixed(Offset));
+ SDValue Ptr = DAG.getObjectPtrOffset(dl, ParamSymbol, Offset);
const MaybeAlign CurrentAlign = ExtendIntegerParam
? MaybeAlign(std::nullopt)
@@ -1651,17 +1573,6 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
}
}
- // Set the size of the vararg param byte array if the callee is a variadic
- // function and the variadic part is not empty.
- if (VADeclareParam) {
- SDValue DeclareParamOps[] = {VADeclareParam.getOperand(0),
- VADeclareParam.getOperand(1),
- VADeclareParam.getOperand(2), GetI32(VAOffset),
- VADeclareParam.getOperand(4)};
- DAG.MorphNodeTo(VADeclareParam.getNode(), VADeclareParam.getOpcode(),
- VADeclareParam->getVTList(), DeclareParamOps);
- }
-
const auto *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
const auto *CalleeF = Func ? dyn_cast<Function>(Func->getGlobal()) : nullptr;
@@ -1695,11 +1606,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
// instruction.
// The prototype is embedded in a string and put as the operand for a
// CallPrototype SDNode which will print out to the value of the string.
- const bool HasVAArgs = CLI.IsVarArg && (CLI.Args.size() > CLI.NumFixedArgs);
std::string Proto =
- getPrototype(DL, RetTy, Args, CLI.Outs,
- HasVAArgs ? std::optional(FirstVAArg) : std::nullopt, *CB,
- UniqueCallSite);
+ getPrototype(DL, RetTy, Args, CLI.Outs, *CB, UniqueCallSite);
const char *ProtoStr = nvTM->getStrPool().save(Proto).data();
const SDValue PrototypeDeclare = DAG.getNode(
NVPTXISD::CallPrototype, dl, MVT::Other,
@@ -1733,7 +1641,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
SmallVector<SDValue, 16> ProxyRegOps;
if (!Ins.empty()) {
SmallVector<EVT, 16> VTs;
- SmallVector<uint64_t, 16> Offsets;
+ SmallVector<TypeSize, 16> Offsets;
ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, RetTy, VTs, Offsets);
assert(VTs.size() == Ins.size() && "Bad value decomposition");
@@ -1757,8 +1665,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
const EVT LoadVT =
ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
const EVT VecVT = getVectorizedVT(LoadVT, NumElts, Ctx);
- SDValue Ptr =
- DAG.getObjectPtrOffset(dl, RetSymbol, TypeSize::getFixed(Offsets[I]));
+ SDValue Ptr = DAG.getObjectPtrOffset(dl, RetSymbol, Offsets[I]);
SDValue R = DAG.getLoad(
VecVT, dl, Call, Ptr,
@@ -3509,10 +3416,6 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerFP_ROUND(Op, DAG);
case ISD::FP_EXTEND:
return LowerFP_EXTEND(Op, DAG);
- case ISD::VAARG:
- return LowerVAARG(Op, DAG);
- case ISD::VASTART:
- return LowerVASTART(Op, DAG);
case ISD::FSHL:
case ISD::FSHR:
return lowerFSH(Op, DAG);
@@ -3592,63 +3495,6 @@ SDValue NVPTXTargetLowering::LowerADDRSPACECAST(SDValue Op,
return Op;
}
-// This function is almost a copy of SelectionDAG::expandVAArg().
-// The only diff is that this one produces loads from local address space.
-SDValue NVPTXTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
- const TargetLowering *TLI = STI.getTargetLowering();
- SDLoc DL(Op);
-
- SDNode *Node = Op.getNode();
- const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
- EVT VT = Node->getValueType(0);
- auto *Ty = VT.getTypeForEVT(*DAG.getContext());
- SDValue Tmp1 = Node->getOperand(0);
- SDValue Tmp2 = Node->getOperand(1);
- const MaybeAlign MA(Node->getConstantOperandVal(3));
-
- SDValue VAListLoad = DAG.getLoad(TLI->getPointerTy(DAG.getDataLayout()), DL,
- Tmp1, Tmp2, MachinePointerInfo(V));
- SDValue VAList = VAListLoad;
-
- if (MA && *MA > TLI->getMinStackArgumentAlignment()) {
- VAList = DAG.getNode(
- ISD::ADD, DL, VAList.getValueType(), VAList,
- DAG.getConstant(MA->value() - 1, DL, VAList.getValueType()));
-
- VAList = DAG.getNode(ISD::AND, DL, VAList.getValueType(), VAList,
- DAG.getSignedConstant(-(int64_t)MA->value(), DL,
- VAList.getValueType()));
- }
-
- // Increment the pointer, VAList, to the next vaarg
- Tmp1 = DAG.getNode(ISD::ADD, DL, VAList.getValueType(), VAList,
- DAG.getConstant(DAG.getDataLayout().getTypeAllocSize(Ty),
- DL, VAList.getValueType()));
-
- // Store the incremented VAList to the legalized pointer
- Tmp1 = DAG.getStore(VAListLoad.getValue(1), DL, Tmp1, Tmp2,
- MachinePointerInfo(V));
-
- const Value *SrcV = Constant::getNullValue(
- PointerType::get(*DAG.getContext(), ADDRESS_SPACE_LOCAL));
-
- // Load the actual argument out of the pointer VAList
- return DAG.getLoad(VT, DL, Tmp1, VAList, MachinePointerInfo(SrcV));
-}
-
-SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
- const TargetLowering *TLI = STI.getTargetLowering();
- SDLoc DL(Op);
- EVT PtrVT = TLI->getPointerTy(DAG.getDataLayout());
-
- // Store the address of unsized array <function>_vararg[] in the ap object.
- SDValue VAReg = getParamSymbol(DAG, /* vararg */ -1, PtrVT);
-
- const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
- return DAG.getStore(Op.getOperand(0), DL, VAReg, Op.getOperand(1),
- MachinePointerInfo(SV));
-}
-
static std::pair<MemSDNode *, uint32_t>
convertMLOADToLoadWithUsedBytesMask(MemSDNode *N, SelectionDAG &DAG,
const NVPTXSubtarget &STI) {
@@ -4069,16 +3915,14 @@ bool NVPTXTargetLowering::splitValueIntoRegisterParts(
// This creates target external symbol for a function parameter.
// Name of the symbol is composed from its index and the function name.
-// Negative index corresponds to special parameter (unsized array) used for
-// passing variable arguments.
-SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int I,
+SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, unsigned I,
EVT T) const {
StringRef SavedStr = nvTM->getStrPool().save(
getParamName(&DAG.getMachineFunction().getFunction(), I));
return DAG.getExternalSymbol(SavedStr.data(), T);
}
-SDValue NVPTXTargetLowering::getCallParamSymbol(SelectionDAG &DAG, int I,
+SDValue NVPTXTargetLowering::getCallParamSymbol(SelectionDAG &DAG, unsigned I,
EVT T) const {
const StringRef SavedStr = nvTM->getStrPool().save("param" + Twine(I));
return DAG.getExternalSymbol(SavedStr.data(), T);
@@ -4088,6 +3932,8 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
+ assert(!isVarArg && "Vararg functions lowered in ExpandVariadics");
+
const DataLayout &DL = DAG.getDataLayout();
LLVMContext &Ctx = *DAG.getContext();
auto PtrVT = getPointerTy(DAG.getDataLayout());
@@ -4163,7 +4009,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
InVals.push_back(P);
} else {
SmallVector<EVT, 16> VTs;
- SmallVector<uint64_t, 16> Offsets;
+ SmallVector<TypeSize, 16> Offsets;
ComputePTXValueVTs(*this, DL, Ctx, CallConv, Ty, VTs, Offsets);
assert(VTs.size() == ArgIns.size() && "Size mismatch");
assert(VTs.size() == Offsets.size() && "Size mismatch");
@@ -4178,8 +4024,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];
const EVT VecVT = getVectorizedVT(LoadVT, NumElts, Ctx);
- SDValue VecAddr = DAG.getObjectPtrOffset(
- dl, ArgSymbol, TypeSize::getFixed(Offsets[I]));
+ SDValue VecAddr = DAG.getObjectPtrOffset(dl, ArgSymbol, Offsets[I]);
const Align PartAlign = commonAlignment(ArgAlign, Offsets[I]);
const unsigned AS = IsKernel ? NVPTX::AddressSpace::EntryParam
@@ -4213,6 +4058,8 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
const SDLoc &dl, SelectionDAG &DAG) const {
+ assert(!isVarArg && "Vararg functions lowered in ExpandVariadics");
+
const Function &F = DAG.getMachineFunction().getFunction();
Type *RetTy = F.getReturnType();
@@ -4234,7 +4081,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
SmallVector<EVT, 16> VTs;
- SmallVector<uint64_t, 16> Offsets;
+ SmallVector<TypeSize, 16> Offsets;
ComputePTXValueVTs(*this, DL, Ctx, CallConv, RetTy, VTs, Offsets);
assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
@@ -4260,8 +4107,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
SDValue Val = getBuildVectorizedValue(
NumElts, dl, DAG, [&](unsigned K) { return GetRetVal(I + K); });
- SDValue Ptr =
- DAG.getObjectPtrOffset(dl, RetSymbol, TypeSize::getFixed(Offsets[I]));
+ SDValue Ptr = DAG.getObjectPtrOffset(dl, RetSymbol, Offsets[I]);
Chain = DAG.getStore(Chain, dl, Val, Ptr,
MachinePointerInfo(NVPTX::AddressSpace::DeviceParam),
@@ -5568,20 +5414,11 @@ void NVPTXTargetLowering::getTgtMemIntrinsic(
}
// Helper for getting a function parameter name. Name is composed from
-// its index and the function name. Negative index corresponds to special
-// parameter (unsized array) used for passing variable arguments.
+// its index and the function name.
std::string NVPTXTargetLowering::getParamName(const Function *F,
- int Idx) const {
- std::string ParamName;
- raw_string_ostream ParamStr(ParamName);
-
- ParamStr << getTargetMachine().getSymbol(F)->getName();
- if (Idx < 0)
- ParamStr << "_vararg";
- else
- ParamStr << "_param_" << Idx;
-
- return ParamName;
+ unsigned Idx) const {
+ return (getTargetMachine().getSymbol(F)->getName() + "_param_" + Twine(Idx))
+ .str();
}
/// isLegalAddressingMode - Return true if the addressing mode represented
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 0e8dd6056af81..76892c229a842 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -37,9 +37,8 @@ class NVPTXTargetLowering : public TargetLowering {
unsigned Intrinsic) const override;
// Helper for getting a function parameter name. Name is composed from
- // its index and the function name. Negative index corresponds to special
- // parameter (unsized array) used for passing variable arguments.
- std::string getParamName(const Function *F, int Idx) const;
+ // its index and the function name.
+ std::string getParamName(const Function *F, unsigned Idx) const;
/// isLegalAddressingMode - Return true if the addressing mode represented
/// by AM is legal for this target, for a load/store of the specified type
@@ -85,7 +84,6 @@ class NVPTXTargetLowering : public TargetLowering {
std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
const SmallVectorImpl<ISD::OutputArg> &,
- std::optional<unsigned> FirstVAArg,
const CallBase &CB, unsigned UniqueCallSite) const;
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
@@ -194,8 +192,8 @@ class NVPTXTargetLowering : public TargetLowering {
const NVPTXSubtarget &STI; // cache the subtarget here
mutable unsigned GlobalUniqueCallSite;
- SDValue getParamSymbol(SelectionDAG &DAG, int I, EVT T) const;
- SDValue getCallParamSymbol(SelectionDAG &DAG, int I, EVT T) const;
+ SDValue getParamSymbol(SelectionDAG &DAG, unsigned I, EVT T) const;
+ SDValue getCallParamSymbol(SelectionDAG &DAG, unsigned I, EVT T) const;
SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
@@ -228,9 +226,6 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
- SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
- SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
-
SDValue LowerCopyToReg_128(SDValue Op, SelectionDAG &DAG) const;
unsigned getNumRegisters(LLVMContext &Context, EVT VT,
std::optional<MVT> RegisterVT) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 1df5d326f63a6..e632204a444d5 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -334,14 +334,6 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
bool hasNativeBF16Support(int Opcode) const;
- // Get maximum value of required alignments among the supported data types.
- // From the PTX ISA doc, section 8.2.3:
- // The memory consistency model relates operations executed on memory
- // locations with scalar data-types, which have a maximum size and alignment
- // of 64 bits. Memory operations with a vector data-type are modelled as a
- // set of equivalent memory operations with a scalar data-type, executed in
- // an unspecified order on the elements in the vector.
- unsigned getMaxRequiredAlignment() const { return 8; }
// Get the smallest cmpxchg word size that the hardware supports.
unsigned getMinCmpXchgSizeInBits() const { return 32; }
More information about the llvm-commits
mailing list