[llvm] 960d0e4 - Reland "[AMDGPU] Account for inline asm size in inst_pref_size calculation" (#197227)
via llvm-commits
llvm-commits at lists.llvm.org
Wed May 20 06:02:31 PDT 2026
Author: Adel Ejjeh
Date: 2026-05-20T08:02:24-05:00
New Revision: 960d0e4644464cf81a4ad2c90699d901e8adf650
URL: https://github.com/llvm/llvm-project/commit/960d0e4644464cf81a4ad2c90699d901e8adf650
DIFF: https://github.com/llvm/llvm-project/commit/960d0e4644464cf81a4ad2c90699d901e8adf650.diff
LOG: Reland "[AMDGPU] Account for inline asm size in inst_pref_size calculation" (#197227)
This relands commit 7ddee0b619f658cef905a69427ef9531fd1d229d (PR
#192306) which was reverted in 70a70e0ed664 (#197070) due to a missing
MC assembler parser case for the `instprefsize` MCExpr, breaking text
assembly roundtrip tests.
Fix:
- Add `"instprefsize"` to the `StringSwitch` in
`AMDGPUAsmParser::parsePrimaryExpr` so the MC assembler can parse
`instprefsize(...)` expressions emitted by `llc` in text assembly mode.
- Add roundtrip lit tests (`llc -filetype=asm | llvm-mc -filetype=obj |
llvm-objdump`) for both GFX11 and GFX12 to prevent regressions.
Confirmed by compiling the new lit test using the original commit that
it was failing and passes now.
_Original PR description_
> `SIProgramInfo::getFunctionCodeSize()` with `IsLowerBound=true` was
completely skipping inline assembly instructions, treating them as zero
bytes. This caused `amdhsa_inst_pref_size` to be severely underestimated
for kernels containing inline asm, defeating instruction prefetch on
gfx11+.
>
> Use MCExpr label subtraction (`.Lfunc_end - func_sym`) to compute
exact function code size, resolved at assembly time. Add a new
`AMDGPUMCExpr` variant (`AGVK_InstPrefSize`) to compute
`min(divideCeil(codeSize, cacheLineSize), maxFieldVal)`.
>
> Remove the `IsLowerBound` parameter from `getFunctionCodeSize()` as it
is no longer needed.
Assisted-by: Claude Code Opus.
Added:
llvm/test/CodeGen/AMDGPU/inst-prefetch-inline-asm.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
llvm/lib/Target/AMDGPU/GCNSubtarget.h
llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
llvm/lib/Target/AMDGPU/SIProgramInfo.h
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
llvm/test/CodeGen/AMDGPU/inst-prefetch-hint.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 9ae6e69c5fb79..f1c015312d9a7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -234,6 +234,18 @@ void AMDGPUAsmPrinter::emitFunctionBodyStart() {
HSAMetadataStream->emitKernel(*MF, CurrentProgramInfo);
}
+/// Set bits in a kernel descriptor MCExpr field:
+/// return ((Dst & ~Mask) | (Value << Shift))
+static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
+ uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
+ const auto *Shft = MCConstantExpr::create(Shift, Ctx);
+ const auto *Msk = MCConstantExpr::create(Mask, Ctx);
+ Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx);
+ Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Shft, Ctx),
+ Ctx);
+ return Dst;
+}
+
void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
if (!MFI.isEntryFunction())
@@ -241,6 +253,29 @@ void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
+ const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
+ MCContext &Ctx = MF->getContext();
+
+ AMDGPU::MCKernelDescriptor KD =
+ getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo);
+
+ // Compute inst_pref_size using MCExpr label subtraction for exact code
+ // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
+ // right after the function code, so (Lfunc_end - func_sym) gives the
+ // exact function code size in bytes.
+ if (STM.hasInstPrefSize()) {
+ const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
+ MCSymbolRefExpr::create(getFunctionEnd(), OutContext),
+ MCSymbolRefExpr::create(CurrentFnSym, OutContext), OutContext);
+
+ uint32_t Mask, Shift, Width, CacheLineSize;
+ STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
+ const MCExpr *InstPrefSize =
+ AMDGPUMCExpr::createInstPrefSize(CodeSizeExpr, Ctx);
+ KD.compute_pgm_rsrc3 =
+ setBits(KD.compute_pgm_rsrc3, InstPrefSize, Mask, Shift, Ctx);
+ }
+
auto &Streamer = getTargetStreamer()->getStreamer();
auto &Context = Streamer.getContext();
auto &ObjectFileInfo = *Context.getObjectFileInfo();
@@ -254,13 +289,10 @@ void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
Streamer.emitValueToAlignment(Align(64), 0, 1, 0);
ReadOnlySection.ensureMinAlignment(Align(64));
- const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
-
SmallString<128> KernelName;
getNameWithPrefix(KernelName, &MF->getFunction());
getTargetStreamer()->EmitAmdhsaKernelDescriptor(
- STM, KernelName, getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo),
- CurrentProgramInfo.NumVGPRsForWavesPerEU,
+ STM, KernelName, KD, CurrentProgramInfo.NumVGPRsForWavesPerEU,
MCBinaryExpr::createSub(
CurrentProgramInfo.NumSGPRsForWavesPerEU,
AMDGPUMCExpr::createExtraSGPRs(
@@ -1438,33 +1470,22 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
ProgInfo.EXCPEnable = 0;
- // return ((Dst & ~Mask) | (Value << Shift))
- auto SetBits = [&Ctx](const MCExpr *Dst, const MCExpr *Value, uint32_t Mask,
- uint32_t Shift) {
- const auto *Shft = MCConstantExpr::create(Shift, Ctx);
- const auto *Msk = MCConstantExpr::create(Mask, Ctx);
- Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx);
- Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Shft, Ctx),
- Ctx);
- return Dst;
- };
-
if (STM.hasGFX90AInsts()) {
ProgInfo.ComputePGMRSrc3 =
- SetBits(ProgInfo.ComputePGMRSrc3, ProgInfo.AccumOffset,
+ setBits(ProgInfo.ComputePGMRSrc3, ProgInfo.AccumOffset,
amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
- amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT);
+ amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
ProgInfo.ComputePGMRSrc3 =
- SetBits(ProgInfo.ComputePGMRSrc3, CreateExpr(ProgInfo.TgSplit),
+ setBits(ProgInfo.ComputePGMRSrc3, CreateExpr(ProgInfo.TgSplit),
amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
- amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT);
+ amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
}
if (STM.hasGFX1250Insts())
ProgInfo.ComputePGMRSrc3 =
- SetBits(ProgInfo.ComputePGMRSrc3, ProgInfo.NamedBarCnt,
+ setBits(ProgInfo.ComputePGMRSrc3, ProgInfo.NamedBarCnt,
amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
- amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT);
+ amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
ProgInfo.Occupancy = createOccupancy(
STM.computeOccupancy(F, ProgInfo.LDSSize).second,
@@ -1483,26 +1504,6 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
", final occupancy is " + Twine(Occupancy));
F.getContext().diagnose(Diag);
}
-
- if (isGFX11Plus(STM)) {
- uint32_t CodeSizeInBytes = (uint32_t)std::min(
- ProgInfo.getFunctionCodeSize(MF, true /* IsLowerBound */),
- (uint64_t)std::numeric_limits<uint32_t>::max());
- uint32_t CodeSizeInLines = divideCeil(CodeSizeInBytes, 128);
- uint32_t Field, Shift, Width;
- if (isGFX11(STM)) {
- Field = amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE;
- Shift = amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_SHIFT;
- Width = amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_WIDTH;
- } else {
- Field = amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE;
- Shift = amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_SHIFT;
- Width = amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_WIDTH;
- }
- uint64_t InstPrefSize = std::min(CodeSizeInLines, (1u << Width) - 1);
- ProgInfo.ComputePGMRSrc3 = SetBits(ProgInfo.ComputePGMRSrc3,
- CreateExpr(InstPrefSize), Field, Shift);
- }
}
static unsigned getRsrcReg(CallingConv::ID CallConv) {
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index b573ff6eec429..bcfb105137af8 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -9358,6 +9358,7 @@ bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
.Case("totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
.Case("alignto", AGVK::AGVK_AlignTo)
.Case("occupancy", AGVK::AGVK_Occupancy)
+ .Case("instprefsize", AGVK::AGVK_InstPrefSize)
.Default(AGVK::AGVK_None);
if (VK != AGVK::AGVK_None && peekToken().is(AsmToken::LParen)) {
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 62476719500fe..fa7a623ec993a 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -21,6 +21,7 @@
#include "SIISelLowering.h"
#include "SIInstrInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/Support/AMDHSAKernelDescriptor.h"
#include "llvm/Support/ErrorHandling.h"
#define GET_SUBTARGETINFO_HEADER
@@ -425,6 +426,23 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool hasPrefetch() const { return HasGFX12Insts; }
+ bool hasInstPrefSize() const { return isGFX11Plus(); }
+
+ void getInstPrefSizeArgs(uint32_t &Mask, uint32_t &Shift, uint32_t &Width,
+ uint32_t &CacheLineSize) const {
+ assert(isGFX11Plus());
+ CacheLineSize = getInstCacheLineSize();
+ if (getGeneration() == GFX11) {
+ Mask = amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE;
+ Shift = amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_SHIFT;
+ Width = amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_WIDTH;
+ } else {
+ Mask = amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE;
+ Shift = amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_SHIFT;
+ Width = amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_WIDTH;
+ }
+ }
+
// Has s_cmpk_* instructions.
bool hasSCmpK() const { return getGeneration() < GFX12; }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
index 01c68d3b061d0..08871f43a42b5 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
@@ -12,9 +12,12 @@
#include "llvm/MC/MCAssembler.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/MC/MCValue.h"
+#include "llvm/Support/AMDHSAKernelDescriptor.h"
#include "llvm/Support/KnownBits.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <functional>
#include <optional>
@@ -74,6 +77,9 @@ void AMDGPUMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
case AGVK_Occupancy:
OS << "occupancy(";
break;
+ case AGVK_InstPrefSize:
+ OS << "instprefsize(";
+ break;
case AGVK_Lit:
OS << "lit(";
break;
@@ -182,6 +188,27 @@ bool AMDGPUMCExpr::evaluateOccupancy(MCValue &Res,
return true;
}
+/// Get the inst_pref_size field width for the given subtarget.
+static unsigned getInstPrefSizeFieldWidth(const MCSubtargetInfo &STI) {
+ if (AMDGPU::isGFX12Plus(STI))
+ return amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_WIDTH;
+ return amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_WIDTH;
+}
+
+bool AMDGPUMCExpr::evaluateInstPrefSize(MCValue &Res,
+ const MCAssembler *Asm) const {
+ uint64_t CodeSizeInBytes = 0;
+ if (!evaluateMCExprs(Args, Asm, {CodeSizeInBytes}))
+ return false;
+ const MCSubtargetInfo *STI = Ctx.getSubtargetInfo();
+ unsigned FieldWidth = getInstPrefSizeFieldWidth(*STI);
+ unsigned CacheLineSize = AMDGPU::IsaInfo::getInstCacheLineSize(*STI);
+ uint64_t CodeSizeInLines = divideCeil(CodeSizeInBytes, CacheLineSize);
+ uint64_t MaxVal = (1u << FieldWidth) - 1;
+ Res = MCValue::get(std::min(CodeSizeInLines, MaxVal));
+ return true;
+}
+
bool AMDGPUMCExpr::isSymbolUsedInExpression(const MCSymbol *Sym,
const MCExpr *E) {
switch (E->getKind()) {
@@ -227,6 +254,8 @@ bool AMDGPUMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
return evaluateTotalNumVGPR(Res, Asm);
case AGVK_Occupancy:
return evaluateOccupancy(Res, Asm);
+ case AGVK_InstPrefSize:
+ return evaluateInstPrefSize(Res, Asm);
case AGVK_Lit:
case AGVK_Lit64:
return Args[0]->evaluateAsRelocatable(Res, Asm);
@@ -279,6 +308,11 @@ const AMDGPUMCExpr *AMDGPUMCExpr::createTotalNumVGPR(const MCExpr *NumAGPR,
return create(AGVK_TotalNumVGPRs, {NumAGPR, NumVGPR}, Ctx);
}
+const AMDGPUMCExpr *
+AMDGPUMCExpr::createInstPrefSize(const MCExpr *CodeSizeBytes, MCContext &Ctx) {
+ return create(AGVK_InstPrefSize, {CodeSizeBytes}, Ctx);
+}
+
const AMDGPUMCExpr *AMDGPUMCExpr::createLit(LitModifier Lit, int64_t Value,
MCContext &Ctx) {
assert(Lit == LitModifier::Lit || Lit == LitModifier::Lit64);
@@ -469,6 +503,7 @@ static void targetOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
case AMDGPUMCExpr::VariantKind::AGVK_TotalNumVGPRs:
case AMDGPUMCExpr::VariantKind::AGVK_AlignTo:
case AMDGPUMCExpr::VariantKind::AGVK_Occupancy:
+ case AMDGPUMCExpr::VariantKind::AGVK_InstPrefSize:
case AMDGPUMCExpr::VariantKind::AGVK_Lit:
case AMDGPUMCExpr::VariantKind::AGVK_Lit64: {
int64_t Val;
@@ -477,6 +512,16 @@ static void targetOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
KBM[Expr] = KnownBits::makeConstant(APValue);
return;
}
+ if (AGVK->getKind() == AMDGPUMCExpr::VariantKind::AGVK_InstPrefSize) {
+ // The result is clamped to (1 << FieldWidth) - 1, so upper bits are
+ // known zero. FieldWidth is derived from the subtarget.
+ const MCSubtargetInfo *STI = AGVK->getCtx().getSubtargetInfo();
+ unsigned FieldWidth = getInstPrefSizeFieldWidth(*STI);
+ KnownBits KB(BitWidth);
+ KB.Zero.setBitsFrom(FieldWidth);
+ KBM[Expr] = KB;
+ return;
+ }
KBM[Expr] = KnownBits(BitWidth);
return;
}
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
index 8bc4a2f9c0e36..66b6fdb4b0042 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
@@ -38,6 +38,7 @@ class AMDGPUMCExpr : public MCTargetExpr {
AGVK_TotalNumVGPRs,
AGVK_AlignTo,
AGVK_Occupancy,
+ AGVK_InstPrefSize,
AGVK_Lit,
AGVK_Lit64,
};
@@ -69,6 +70,7 @@ class AMDGPUMCExpr : public MCTargetExpr {
bool evaluateTotalNumVGPR(MCValue &Res, const MCAssembler *Asm) const;
bool evaluateAlignTo(MCValue &Res, const MCAssembler *Asm) const;
bool evaluateOccupancy(MCValue &Res, const MCAssembler *Asm) const;
+ bool evaluateInstPrefSize(MCValue &Res, const MCAssembler *Asm) const;
public:
static const AMDGPUMCExpr *
@@ -97,11 +99,18 @@ class AMDGPUMCExpr : public MCTargetExpr {
return create(VariantKind::AGVK_AlignTo, {Value, Align}, Ctx);
}
+ /// Create an expression for instruction prefetch size computation:
+ /// min(divideCeil(CodeSizeBytes, CacheLineSize), (1 << FieldWidth) - 1)
+ /// FieldWidth and CacheLineSize are derived from the subtarget.
+ static const AMDGPUMCExpr *createInstPrefSize(const MCExpr *CodeSizeBytes,
+ MCContext &Ctx);
+
static const AMDGPUMCExpr *createLit(LitModifier Lit, int64_t Value,
MCContext &Ctx);
ArrayRef<const MCExpr *> getArgs() const { return Args; }
VariantKind getKind() const { return Kind; }
+ MCContext &getCtx() const { return Ctx; }
const MCExpr *getSubExpr(size_t Index) const;
void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
index 99255e4060886..27cef7a1b9158 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
@@ -215,9 +215,8 @@ const MCExpr *SIProgramInfo::getPGMRSrc2(CallingConv::ID CC,
return MCConstantExpr::create(0, Ctx);
}
-uint64_t SIProgramInfo::getFunctionCodeSize(const MachineFunction &MF,
- bool IsLowerBound) {
- if (!IsLowerBound && CodeSizeInBytes.has_value())
+uint64_t SIProgramInfo::getFunctionCodeSize(const MachineFunction &MF) {
+ if (CodeSizeInBytes.has_value())
return *CodeSizeInBytes;
const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
@@ -226,12 +225,7 @@ uint64_t SIProgramInfo::getFunctionCodeSize(const MachineFunction &MF,
uint64_t CodeSize = 0;
for (const MachineBasicBlock &MBB : MF) {
- // The amount of padding to align code can be both underestimated and
- // overestimated. In case of inline asm used getInstSizeInBytes() will
- // return a maximum size of a single instruction, where the real size may
- //
diff er. At this point CodeSize may be already off.
- if (!IsLowerBound)
- CodeSize = alignTo(CodeSize, MBB.getAlignment());
+ CodeSize = alignTo(CodeSize, MBB.getAlignment());
for (const MachineInstr &MI : MBB) {
// TODO: CodeSize should account for multiple functions.
@@ -239,11 +233,6 @@ uint64_t SIProgramInfo::getFunctionCodeSize(const MachineFunction &MF,
if (MI.isMetaInstruction())
continue;
- // We cannot properly estimate inline asm size. It can be as small as zero
- // if that is just a comment.
- if (IsLowerBound && MI.isInlineAsm())
- continue;
-
CodeSize += TII->getInstSizeInBytes(MI);
}
}
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.h b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
index 947b473142a1f..fb56ebf88c96f 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
@@ -105,10 +105,7 @@ struct LLVM_EXTERNAL_VISIBILITY SIProgramInfo {
void reset(const MachineFunction &MF);
// Get function code size and cache the value.
- // If \p IsLowerBound is set it returns a minimal code size which is safe
- // to address.
- uint64_t getFunctionCodeSize(const MachineFunction &MF,
- bool IsLowerBound = false);
+ uint64_t getFunctionCodeSize(const MachineFunction &MF);
/// Compute the value of the ComputePGMRsrc1 register.
const MCExpr *getComputePGMRSrc1(const GCNSubtarget &ST,
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index b7c46d70ac685..44c207505d872 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1186,6 +1186,14 @@ std::string AMDGPUTargetID::toString() const {
return Str;
}
+unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
+ if (STI.getFeatureBits().test(FeatureInstCacheLineSize128))
+ return 128;
+ if (STI.getFeatureBits().test(FeatureInstCacheLineSize64))
+ return 64;
+ return 64;
+}
+
unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
if (STI.getFeatureBits().test(FeatureWavefrontSize16))
return 16;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 66c83683a1fff..d761da4d0d804 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -233,6 +233,9 @@ inline raw_ostream &operator<<(raw_ostream &OS,
return OS;
}
+/// \returns Instruction cache line size in bytes for given subtarget \p STI.
+unsigned getInstCacheLineSize(const MCSubtargetInfo &STI);
+
/// \returns Wavefront size for given subtarget \p STI.
unsigned getWavefrontSize(const MCSubtargetInfo &STI);
diff --git a/llvm/test/CodeGen/AMDGPU/inst-prefetch-hint.ll b/llvm/test/CodeGen/AMDGPU/inst-prefetch-hint.ll
index 580167076e1f0..1509eb16d8627 100644
--- a/llvm/test/CodeGen/AMDGPU/inst-prefetch-hint.ll
+++ b/llvm/test/CodeGen/AMDGPU/inst-prefetch-hint.ll
@@ -1,11 +1,36 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 --amdgpu-memcpy-loop-unroll=100000 < %s | FileCheck --check-prefixes=GCN,GFX11 %s
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 --amdgpu-memcpy-loop-unroll=100000 < %s | FileCheck --check-prefixes=GCN,GFX12 %s
+;; Verify that inst_pref_size resolves to the correct value in the object file.
+;; COMPUTE_PGM_RSRC3 is at offset 0x2C in each 64-byte kernel descriptor.
+;; inst_pref_size is bits [9:4] on GFX11 (6-bit) and bits [11:4] on GFX12+ (8-bit).
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 --amdgpu-memcpy-loop-unroll=100000 -filetype=obj < %s -o %t.gfx11.o
+; RUN: llvm-objdump -s -j .rodata %t.gfx11.o | FileCheck --check-prefix=OBJ-GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 --amdgpu-memcpy-loop-unroll=100000 -filetype=obj < %s -o %t.gfx12.o
+; RUN: llvm-objdump -s -j .rodata %t.gfx12.o | FileCheck --check-prefix=OBJ-GFX12 %s
+;; Verify text assembly round-trips through the MC assembler (instprefsize MCExpr parsing).
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 --amdgpu-memcpy-loop-unroll=100000 < %s | llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj -o %t.rt.gfx11.o
+; RUN: llvm-objdump -s -j .rodata %t.rt.gfx11.o | FileCheck --check-prefix=OBJ-GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 --amdgpu-memcpy-loop-unroll=100000 < %s | llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj -o %t.rt.gfx12.o
+; RUN: llvm-objdump -s -j .rodata %t.rt.gfx12.o | FileCheck --check-prefix=OBJ-GFX12 %s
+
+; The inst_pref_size is computed via MCExpr label subtraction, resolved at
+; assembly/link time. In text output it appears as:
+; ((instprefsize(<code_size>)<<Shift)&Mask)>>Shift
+; where:
+; <code_size> = .Lfunc_endN - func_sym (exact function code size in bytes)
+; instprefsize = min(divideCeil(code_size, cache_line_size), (1 << field_width) - 1)
+; field_width and cache_line_size are derived from the subtarget
+
; GCN-LABEL: .amdhsa_kernel large
-; GFX11: .amdhsa_inst_pref_size 3
-; GFX11: codeLenInByte = 3{{[0-9][0-9]$}}
-; GFX12: .amdhsa_inst_pref_size 4
-; GFX12: codeLenInByte = 4{{[0-9][0-9]$}}
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end0-large)<<4)&1008)>>4
+; GFX11: codeLenInByte = {{[0-9]+}}
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end0-large)<<4)&4080)>>4
+; GFX12: codeLenInByte = {{[0-9]+}}
+;; Object: kernel descriptor at 0x00, COMPUTE_PGM_RSRC3 at 0x2C:
+;; gfx11 pref=3 (0x30), gfx12 pref=4 (0x40)
+; OBJ-GFX11: 0020 {{.*}}30000000
+; OBJ-GFX12: 0020 {{.*}}40000000
define amdgpu_kernel void @large(ptr addrspace(1) %out, ptr addrspace(1) %in) {
bb:
call void @llvm.memcpy.p1.p3.i32(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 256, i1 false)
@@ -13,18 +38,30 @@ bb:
}
; GCN-LABEL: .amdhsa_kernel small
-; GCN: .amdhsa_inst_pref_size 1
-; GCN: codeLenInByte = {{[0-9]$}}
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end1-small)<<4)&1008)>>4
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end1-small)<<4)&4080)>>4
+; GCN: codeLenInByte = {{[0-9]+}}
+;; Object: kernel descriptor at 0x40, COMPUTE_PGM_RSRC3 at 0x6C:
+;; pref=1 (0x10) for both
+; OBJ-GFX11: 0060 {{.*}}10000000
+; OBJ-GFX12: 0060 {{.*}}10000000
define amdgpu_kernel void @small() {
bb:
ret void
}
-; Ignore inline asm in size calculation
+; Inline asm is accounted for via MCExpr label subtraction (exact code size).
+; The MCExpr resolves to the correct inst_pref_size at assembly time.
; GCN-LABEL: .amdhsa_kernel inline_asm
-; GCN: .amdhsa_inst_pref_size 1
-; GCN: codeLenInByte = {{[0-9]$}}
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end2-inline_asm)<<4)&1008)>>4
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end2-inline_asm)<<4)&4080)>>4
+; GCN: codeLenInByte = {{[0-9]+}}
+;; Object: kernel descriptor at 0x80, COMPUTE_PGM_RSRC3 at 0xAC:
+;; pref=9 (0x90) for both
+;; (.fill 256, 4, 0 = 1024 bytes + 4 s_endpgm = 1028 -> divideCeil(1028,128) = 9)
+; OBJ-GFX11: 00a0 {{.*}}90000000
+; OBJ-GFX12: 00a0 {{.*}}90000000
define amdgpu_kernel void @inline_asm() {
bb:
call void asm sideeffect ".fill 256, 4, 0", ""()
diff --git a/llvm/test/CodeGen/AMDGPU/inst-prefetch-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/inst-prefetch-inline-asm.ll
new file mode 100644
index 0000000000000..f1c7d39f2b9c3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inst-prefetch-inline-asm.ll
@@ -0,0 +1,159 @@
+;; Verify that inline assembly is correctly accounted for in the
+;; inst_pref_size calculation. The inst_pref_size is computed via MCExpr
+;; label subtraction (.Lfunc_end - func_sym), giving exact code size.
+;; See inst-prefetch-hint.ll for explanation of the instprefsize expression.
+
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj < %s -o %t.gfx11.o
+; RUN: llvm-objdump -s -j .rodata %t.gfx11.o | FileCheck --check-prefix=OBJ-GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj < %s -o %t.gfx12.o
+; RUN: llvm-objdump -s -j .rodata %t.gfx12.o | FileCheck --check-prefix=OBJ-GFX12 %s
+;; Verify text assembly round-trips through the MC assembler.
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj -o %t.rt.gfx11.o
+; RUN: llvm-objdump -s -j .rodata %t.rt.gfx11.o | FileCheck --check-prefix=OBJ-GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj -o %t.rt.gfx12.o
+; RUN: llvm-objdump -s -j .rodata %t.rt.gfx12.o | FileCheck --check-prefix=OBJ-GFX12 %s
+
+;; --- .fill directive: .fill 256, 4, 0 => 1024 bytes + 4 (s_endpgm) = 1028 ---
+;; pref_size = divideCeil(1028, 128) = 9
+
+; GFX11-LABEL: .amdhsa_kernel test_fill
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end0-test_fill)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_fill
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end0-test_fill)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x00, COMPUTE_PGM_RSRC3 at 0x2C:
+;; pref_size=9 -> 9<<4 = 0x90
+; OBJ-GFX11: 0020 {{.*}}90000000
+; OBJ-GFX12: 0020 {{.*}}90000000
+
+define amdgpu_kernel void @test_fill() {
+ call void asm sideeffect ".fill 256, 4, 0", ""()
+ ret void
+}
+
+;; --- .space directive: .space 1024 => 1024 bytes + 4 = 1028 ---
+;; pref_size = 9
+
+; GFX11-LABEL: .amdhsa_kernel test_space
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end1-test_space)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_space
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end1-test_space)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x40, COMPUTE_PGM_RSRC3 at 0x6C:
+;; pref_size=9 -> 9<<4 = 0x90
+; OBJ-GFX11: 0060 {{.*}}90000000
+; OBJ-GFX12: 0060 {{.*}}90000000
+
+define amdgpu_kernel void @test_space() {
+ call void asm sideeffect ".space 1024", ""()
+ ret void
+}
+
+;; --- Instructions: 32 x s_nop (4 bytes each) = 128 + 4 = 132 ---
+;; pref_size = divideCeil(132, 128) = 2
+
+; GFX11-LABEL: .amdhsa_kernel test_instructions
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end2-test_instructions)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_instructions
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end2-test_instructions)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x80, COMPUTE_PGM_RSRC3 at 0xAC:
+;; pref_size=2 -> 2<<4 = 0x20
+; OBJ-GFX11: 00a0 {{.*}}20000000
+; OBJ-GFX12: 00a0 {{.*}}20000000
+
+define amdgpu_kernel void @test_instructions() {
+ call void asm sideeffect "s_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0\0As_nop 0", ""()
+ ret void
+}
+
+;; --- Comments emit no bytes: only s_endpgm = 4 bytes ---
+;; pref_size = 1
+
+; GFX11-LABEL: .amdhsa_kernel test_comments
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end3-test_comments)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_comments
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end3-test_comments)<<4)&4080)>>4
+;; Object: kernel descriptor at 0xC0, COMPUTE_PGM_RSRC3 at 0xEC:
+;; pref_size=1 -> 1<<4 = 0x10
+; OBJ-GFX11: 00e0 {{.*}}10000000
+; OBJ-GFX12: 00e0 {{.*}}10000000
+
+define amdgpu_kernel void @test_comments() {
+ call void asm sideeffect "; comment 1\0A; comment 2\0A; comment 3", ""()
+ ret void
+}
+
+;; --- Empty inline asm: only s_endpgm = 4 bytes ---
+;; pref_size = 1
+
+; GFX11-LABEL: .amdhsa_kernel test_empty_asm
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end4-test_empty_asm)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_empty_asm
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end4-test_empty_asm)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x100, COMPUTE_PGM_RSRC3 at 0x12C:
+;; pref_size=1 -> 1<<4 = 0x10
+; OBJ-GFX11: 0120 {{.*}}10000000
+; OBJ-GFX12: 0120 {{.*}}10000000
+
+define amdgpu_kernel void @test_empty_asm() {
+ call void asm sideeffect "", ""()
+ ret void
+}
+
+;; --- Multiple inline asm blocks: .fill (512) + .space (512) + s_endpgm (4) = 1028 ---
+;; pref_size = divideCeil(1028, 128) = 9
+
+; GFX11-LABEL: .amdhsa_kernel test_multiple_asm
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end5-test_multiple_asm)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_multiple_asm
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end5-test_multiple_asm)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x140, COMPUTE_PGM_RSRC3 at 0x16C:
+;; pref_size=9 -> 9<<4 = 0x90
+; OBJ-GFX11: 0160 {{.*}}90000000
+; OBJ-GFX12: 0160 {{.*}}90000000
+
+define amdgpu_kernel void @test_multiple_asm() {
+ call void asm sideeffect ".fill 128, 4, 0", ""()
+ call void asm sideeffect ".space 512", ""()
+ ret void
+}
+
+;; --- Large function that exceeds GFX11 6-bit field max (63) ---
+;; .fill 2048, 4, 0 = 8192 bytes + 4 = 8196 bytes
+;; divideCeil(8196, 128) = 65, but GFX11 max = (1<<6)-1 = 63
+;; pref_size should clamp to 63
+
+; GFX11-LABEL: .amdhsa_kernel test_clamping
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end6-test_clamping)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_clamping
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end6-test_clamping)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x180, COMPUTE_PGM_RSRC3 at 0x1AC:
+;; gfx11: clamped to 63 -> 63<<4 = 0x3F0
+;; gfx12: no clamping, 65 -> 65<<4 = 0x410
+; OBJ-GFX11: 01a0 {{.*}}f0030000
+; OBJ-GFX12: 01a0 {{.*}}10040000
+
+define amdgpu_kernel void @test_clamping() {
+ call void asm sideeffect ".fill 2048, 4, 0", ""()
+ ret void
+}
+
+;; --- Large function that exceeds both GFX11 and GFX12 field max ---
+;; .fill 8192, 4, 0 = 32768 bytes + 4 = 32772 bytes
+;; divideCeil(32772, 128) = 257
+;; GFX11 max = 63, GFX12 max = 255 -> both clamp
+
+; GFX11-LABEL: .amdhsa_kernel test_clamping_both
+; GFX11: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end7-test_clamping_both)<<4)&1008)>>4
+; GFX12-LABEL: .amdhsa_kernel test_clamping_both
+; GFX12: .amdhsa_inst_pref_size ((instprefsize(.Lfunc_end7-test_clamping_both)<<4)&4080)>>4
+;; Object: kernel descriptor at 0x1C0, COMPUTE_PGM_RSRC3 at 0x1EC:
+;; gfx11: clamped to 63 -> 63<<4 = 0x3F0
+;; gfx12: clamped to 255 -> 255<<4 = 0xFF0
+; OBJ-GFX11: 01e0 {{.*}}f0030000
+; OBJ-GFX12: 01e0 {{.*}}f00f0000
+
+define amdgpu_kernel void @test_clamping_both() {
+ call void asm sideeffect ".fill 8192, 4, 0", ""()
+ ret void
+}
More information about the llvm-commits
mailing list