[llvm] AMDGPU: share LDS budget logic and add experimental LDS buffering pass (PR #166388)
Yaxun Liu via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 6 15:53:13 PST 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/166388
>From 5d01732237cf90919c577b607babd24596dabe27 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 6 Jan 2026 16:41:48 -0500
Subject: [PATCH] AMDGPU: add LDS buffering pass and shared LDS budget helper
Introduce an experimental per-thread LDS buffering IR pass that converts
eligible global load/store pairs into global<->LDS memcpy pairs, using only
leftover LDS budget computed with the same occupancy-tier constraints as alloca
promotion.
Add a shared LDS budget utility, wire the pass as a parameterized pipeline
pass, gate default-pipeline insertion behind a hidden flag on AMDGCN, and add
consolidated CodeGen coverage (positive, complexity, rocRand-inspired, and
negative cases).
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 17 ++
llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp | 272 ++++++++++++++++++
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 10 +-
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 122 +-------
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 37 +++
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
.../Target/AMDGPU/Utils/AMDGPULDSUtils.cpp | 235 +++++++++++++++
llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h | 53 ++++
llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt | 1 +
llvm/test/CodeGen/AMDGPU/lds-buffering.ll | 183 ++++++++++++
10 files changed, 816 insertions(+), 115 deletions(-)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
create mode 100644 llvm/test/CodeGen/AMDGPU/lds-buffering.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5df11a45b4889..24c86b791dc39 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -270,6 +270,23 @@ struct AMDGPUPromoteAllocaToVectorPass
TargetMachine &TM;
};
+// Buffer selected per-thread global memory through LDS to improve
+// performance in memory-bound kernels. This runs late and is separate
+// from alloca promotion.
+struct AMDGPULDSBufferingPass : PassInfoMixin<AMDGPULDSBufferingPass> {
+ AMDGPULDSBufferingPass(const AMDGPUTargetMachine &TM, unsigned MaxBytes = 64)
+ : TM(TM), MaxBytes(MaxBytes) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+private:
+ const AMDGPUTargetMachine &TM;
+ unsigned MaxBytes;
+};
+
+// Legacy PM wrapper for LDS buffering
+FunctionPass *createAMDGPULDSBufferingLegacyPass();
+void initializeAMDGPULDSBufferingLegacyPass(PassRegistry &);
+
struct AMDGPUAtomicOptimizerPass : PassInfoMixin<AMDGPUAtomicOptimizerPass> {
AMDGPUAtomicOptimizerPass(TargetMachine &TM, ScanOptions ScanImpl)
: TM(TM), ScanImpl(ScanImpl) {}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp b/llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp
new file mode 100644
index 0000000000000..a4c022e240e25
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp
@@ -0,0 +1,272 @@
+//===-- AMDGPULDSBuffering.cpp - Per-thread LDS buffering -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass buffers selected per-thread global memory accesses through LDS
+// (addrspace(3)) to improve performance in memory-bound kernels.
+//
+// This is intended to help cases where a value is loaded from global memory and
+// later stored back to (the same) global location, but intervening memory
+// operations (e.g. potentially-aliasing accesses) inhibit load/store
+// forwarding, keeping the load and store "live" and increasing global traffic
+// and cache pressure.
+//
+// The pass runs late in the pipeline, after SROA and AMDGPUPromoteAlloca,
+// using only leftover LDS budget to avoid interfering with other LDS
+// optimizations. It respects the same LDS budget constraints as
+// AMDGPUPromoteAlloca, ensuring that LDS usage remains within occupancy
+// tier limits.
+//
+// Current implementation handles the simplest pattern: a load from global
+// memory whose only use is a store back to the same pointer. This pattern is
+// transformed into a pair of memcpy operations (global->LDS and LDS->global),
+// effectively moving the value through LDS instead of accessing global memory
+// directly.
+//
+// This pass was inspired by finding that some rocrand performance tests
+// show better performance when global memory is buffered through LDS
+// instead of being loaded/stored to registers directly. This optimization
+// is experimental and must be enabled in the default pipeline via the
+// -amdgpu-enable-lds-buffering flag (or explicitly scheduled via
+// -passes='amdgpu-lds-buffering<...>').
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUTargetMachine.h"
+#include "GCNSubtarget.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPULDSUtils.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/Debug.h"
+#include <algorithm>
+
+#define DEBUG_TYPE "amdgpu-lds-buffering"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPULDSBufferingImpl {
+ const AMDGPUTargetMachine &TM;
+ unsigned MaxBytes;
+ Module *Mod = nullptr;
+ const DataLayout *DL = nullptr;
+
+public:
+ AMDGPULDSBufferingImpl(const AMDGPUTargetMachine &TM, unsigned MaxBytes)
+ : TM(TM), MaxBytes(MaxBytes) {}
+
+ bool run(Function &F) {
+ LLVM_DEBUG(dbgs() << "[LDSBuffer] Visit function: " << F.getName() << '\n');
+ if (!AMDGPU::isEntryFunctionCC(F.getCallingConv()))
+ return false;
+
+ Mod = F.getParent();
+ DL = &Mod->getDataLayout();
+
+ auto Budget = AMDGPU::computeLDSBudget(F, TM);
+ if (!Budget.promotable)
+ return false;
+ uint32_t localUsage = Budget.currentUsage;
+ uint32_t localLimit = Budget.limit;
+
+ const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
+ unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(F).second;
+
+ bool Changed = false;
+ unsigned NumTransformed = 0;
+
+ // Minimal pattern: a load from AS(1) whose only use is a store back to the
+ // exact same pointer later. Replace with global<->LDS memcpy pair to
+ // shorten the live range and free VGPRs.
+ SmallVector<Instruction *> ToErase;
+ for (BasicBlock &BB : F) {
+ for (Instruction &I : llvm::make_early_inc_range(BB)) {
+ auto *LI = dyn_cast<LoadInst>(&I);
+ if (!LI || LI->isVolatile() || LI->isAtomic())
+ continue;
+
+ Type *ValTy = LI->getType();
+ if (!ValTy->isFirstClassType())
+ continue;
+
+ Value *Ptr = LI->getPointerOperand();
+ auto *PtrTy = cast<PointerType>(Ptr->getType());
+ if (PtrTy->getAddressSpace() != AMDGPUAS::GLOBAL_ADDRESS)
+ continue;
+
+ if (!LI->hasOneUse())
+ continue;
+ auto *SI = dyn_cast<StoreInst>(LI->user_back());
+ if (!SI || SI->isVolatile() || SI->isAtomic())
+ continue;
+ if (SI->getValueOperand() != LI)
+ continue;
+
+ Value *SPtr = SI->getPointerOperand();
+ if (SPtr != Ptr)
+ continue;
+
+ TypeSize TS = DL->getTypeStoreSize(ValTy);
+ if (TS.isScalable())
+ continue;
+ uint64_t Size = TS.getFixedValue();
+ if (Size == 0 || Size > MaxBytes)
+ continue;
+ Align MinAlign = Align(16);
+ Align LoadAlign = LI->getAlign();
+ Align StoreAlign = SI->getAlign();
+ Align Alignment = std::min(LoadAlign, StoreAlign);
+ if (Alignment < MinAlign)
+ continue;
+
+ // Create LDS slot near the load and emit memcpy global->LDS.
+ LLVM_DEBUG({
+ dbgs() << "[LDSBuffer] Candidate found: load->store same ptr in "
+ << F.getName() << '\n';
+ dbgs() << " size=" << Size
+ << "B, loadAlign=" << LoadAlign.value()
+ << ", storeAlign=" << StoreAlign.value()
+ << ", chosenAlign=" << Alignment.value()
+ << ", ptr AS=" << PtrTy->getAddressSpace() << "\n";
+ });
+ IRBuilder<> BLoad(LI);
+
+ // Ensure LDS budget allows allocating a per-thread slot.
+ uint32_t NewSize = alignTo(localUsage, Alignment);
+ NewSize += WorkGroupSize * static_cast<uint32_t>(Size);
+ if (NewSize > localLimit)
+ continue;
+ localUsage = NewSize;
+ auto [GV, SlotPtr] =
+ createLDSGlobalAndThreadSlot(F, ValTy, Alignment, "ldsbuf", BLoad);
+ // memcpy p3 <- p1
+ LLVM_DEBUG(dbgs() << "[LDSBuffer] Insert memcpy global->LDS: "
+ << GV->getName() << ", bytes=" << Size
+ << ", align=" << Alignment.value() << '\n');
+ BLoad.CreateMemCpy(SlotPtr, Alignment, Ptr, LoadAlign, TS);
+
+ // Replace the final store with memcpy LDS->global.
+ IRBuilder<> BStore(SI);
+ LLVM_DEBUG(dbgs() << "[LDSBuffer] Insert memcpy LDS->global: "
+ << GV->getName() << ", bytes=" << Size
+ << ", align=" << Alignment.value() << '\n');
+ BStore.CreateMemCpy(SPtr, StoreAlign, SlotPtr, Alignment, TS);
+
+ ToErase.push_back(SI);
+ ToErase.push_back(LI);
+ LLVM_DEBUG(dbgs() << "[LDSBuffer] Erase original load/store pair\n");
+ Changed = true;
+ ++NumTransformed;
+ }
+ }
+
+ for (Instruction *E : ToErase)
+ E->eraseFromParent();
+
+ LLVM_DEBUG(dbgs() << "[LDSBuffer] Transformations applied: "
+ << NumTransformed << "\n");
+
+ return Changed;
+ }
+
+private:
+ // Create an LDS array [WGSize x ElemTy] and return pointer to per-thread
+ // slot.
+ std::pair<GlobalVariable *, Value *>
+ createLDSGlobalAndThreadSlot(Function &F, Type *ElemTy, Align Alignment,
+ StringRef BaseName, IRBuilder<> &Builder) {
+ const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
+ unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(F).second;
+ Type *ArrTy = ArrayType::get(ElemTy, WorkGroupSize);
+ GlobalVariable *GV = new GlobalVariable(
+ *Mod, ArrTy, /*isConstant=*/false, GlobalValue::InternalLinkage,
+ PoisonValue::get(ArrTy), (F.getName() + "." + BaseName).str(), nullptr,
+ GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS);
+ GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+ GV->setAlignment(Alignment);
+
+ LLVM_DEBUG({
+ dbgs() << "[LDSBuffer] Create LDS global: name=" << GV->getName()
+ << ", elemTy=" << *ElemTy << ", WGSize=" << WorkGroupSize
+ << ", align=" << Alignment.value() << '\n';
+ });
+
+ Value *LinearTID =
+ AMDGPU::buildLinearThreadId(Builder, *Mod, ST);
+ LLVMContext &Ctx = Mod->getContext();
+ Value *Indices[] = {Constant::getNullValue(Type::getInt32Ty(Ctx)),
+ LinearTID};
+ Value *SlotPtr = Builder.CreateInBoundsGEP(ArrTy, GV, Indices);
+ return {GV, SlotPtr};
+ }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses AMDGPULDSBufferingPass::run(Function &F,
+ FunctionAnalysisManager &AM) {
+ bool Changed = AMDGPULDSBufferingImpl(TM, MaxBytes).run(F);
+ if (!Changed)
+ return PreservedAnalyses::all();
+
+ PreservedAnalyses PA;
+ PA.preserveSet<CFGAnalyses>();
+ return PA;
+}
+
+//===----------------------------------------------------------------------===//
+// Legacy PM wrapper
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class AMDGPULDSBufferingLegacy : public FunctionPass {
+public:
+ static char ID;
+ AMDGPULDSBufferingLegacy() : FunctionPass(ID) {}
+
+ StringRef getPassName() const override { return "AMDGPU LDS Buffering"; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ FunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnFunction(Function &F) override {
+ if (skipFunction(F))
+ return false;
+ if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>())
+ return AMDGPULDSBufferingImpl(TPC->getTM<AMDGPUTargetMachine>(),
+ /*MaxBytes=*/64)
+ .run(F);
+ return false;
+ }
+};
+
+} // end anonymous namespace
+
+char AMDGPULDSBufferingLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AMDGPULDSBufferingLegacy, DEBUG_TYPE,
+ "AMDGPU per-thread LDS buffering", false, false)
+INITIALIZE_PASS_END(AMDGPULDSBufferingLegacy, DEBUG_TYPE,
+ "AMDGPU per-thread LDS buffering", false, false)
+
+FunctionPass *llvm::createAMDGPULDSBufferingLegacyPass() {
+ return new AMDGPULDSBufferingLegacy();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index f464fbf31c754..e3320d609f9a5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -71,11 +71,13 @@ FUNCTION_PASS("amdgpu-promote-kernel-arguments",
AMDGPUPromoteKernelArgumentsPass())
FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass())
FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
+FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes",
AMDGPUUnifyDivergentExitNodesPass())
FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass())
-FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this)))
-FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
+FUNCTION_PASS("si-annotate-control-flow",
+ SIAnnotateControlFlowPass(
+ *static_cast<const GCNTargetMachine *>(this)))
#undef FUNCTION_PASS
#ifndef FUNCTION_ANALYSIS
@@ -93,6 +95,10 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
#ifndef FUNCTION_PASS_WITH_PARAMS
#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
#endif
+FUNCTION_PASS_WITH_PARAMS(
+ "amdgpu-lds-buffering", "AMDGPULDSBufferingPass",
+ [=](unsigned MaxBytes) { return AMDGPULDSBufferingPass(*this, MaxBytes); },
+ parseAMDGPULDSBufferingMaxBytes, "max-bytes")
FUNCTION_PASS_WITH_PARAMS(
"amdgpu-atomic-optimizer",
"AMDGPUAtomicOptimizerPass",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index d0bc36c2f8501..866ae62d9e5d3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -28,6 +28,7 @@
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPULDSUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/CaptureTracking.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
@@ -1460,129 +1461,24 @@ void AMDGPUPromoteAllocaImpl::analyzePromoteToLDS(AllocaAnalysis &AA) const {
}
bool AMDGPUPromoteAllocaImpl::hasSufficientLocalMem(const Function &F) {
-
- FunctionType *FTy = F.getFunctionType();
- const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
-
- // If the function has any arguments in the local address space, then it's
- // possible these arguments require the entire local memory space, so
- // we cannot use local memory in the pass.
- for (Type *ParamTy : FTy->params()) {
- PointerType *PtrTy = dyn_cast<PointerType>(ParamTy);
- if (PtrTy && PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
- LocalMemLimit = 0;
+ AMDGPU::AMDGPULDSBudget Budget = AMDGPU::computeLDSBudget(F, TM);
+ CurrentLocalMemUsage = Budget.currentUsage;
+ LocalMemLimit = Budget.limit;
+ if (!Budget.promotable) {
+ if (Budget.disabledDueToLocalArg) {
LLVM_DEBUG(dbgs() << "Function has local memory argument. Promoting to "
"local memory disabled.\n");
- return false;
- }
- }
-
- LocalMemLimit = ST.getAddressableLocalMemorySize();
- if (LocalMemLimit == 0)
- return false;
-
- SmallVector<const Constant *, 16> Stack;
- SmallPtrSet<const Constant *, 8> VisitedConstants;
- SmallPtrSet<const GlobalVariable *, 8> UsedLDS;
-
- auto visitUsers = [&](const GlobalVariable *GV, const Constant *Val) -> bool {
- for (const User *U : Val->users()) {
- if (const Instruction *Use = dyn_cast<Instruction>(U)) {
- if (Use->getFunction() == &F)
- return true;
- } else {
- const Constant *C = cast<Constant>(U);
- if (VisitedConstants.insert(C).second)
- Stack.push_back(C);
- }
- }
-
- return false;
- };
-
- for (GlobalVariable &GV : Mod->globals()) {
- if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
- continue;
-
- if (visitUsers(&GV, &GV)) {
- UsedLDS.insert(&GV);
- Stack.clear();
- continue;
}
-
- // For any ConstantExpr uses, we need to recursively search the users until
- // we see a function.
- while (!Stack.empty()) {
- const Constant *C = Stack.pop_back_val();
- if (visitUsers(&GV, C)) {
- UsedLDS.insert(&GV);
- Stack.clear();
- break;
- }
- }
- }
-
- const DataLayout &DL = Mod->getDataLayout();
- SmallVector<std::pair<uint64_t, Align>, 16> AllocatedSizes;
- AllocatedSizes.reserve(UsedLDS.size());
-
- for (const GlobalVariable *GV : UsedLDS) {
- Align Alignment =
- DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());
- uint64_t AllocSize = DL.getTypeAllocSize(GV->getValueType());
-
- // HIP uses an extern unsized array in local address space for dynamically
- // allocated shared memory. In that case, we have to disable the promotion.
- if (GV->hasExternalLinkage() && AllocSize == 0) {
- LocalMemLimit = 0;
+ if (Budget.disabledDueToExternDynShared) {
LLVM_DEBUG(dbgs() << "Function has a reference to externally allocated "
"local memory. Promoting to local memory "
"disabled.\n");
- return false;
}
-
- AllocatedSizes.emplace_back(AllocSize, Alignment);
- }
-
- // Sort to try to estimate the worst case alignment padding
- //
- // FIXME: We should really do something to fix the addresses to a more optimal
- // value instead
- llvm::sort(AllocatedSizes, llvm::less_second());
-
- // Check how much local memory is being used by global objects
- CurrentLocalMemUsage = 0;
-
- // FIXME: Try to account for padding here. The real padding and address is
- // currently determined from the inverse order of uses in the function when
- // legalizing, which could also potentially change. We try to estimate the
- // worst case here, but we probably should fix the addresses earlier.
- for (auto Alloc : AllocatedSizes) {
- CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alloc.second);
- CurrentLocalMemUsage += Alloc.first;
- }
-
- unsigned MaxOccupancy =
- ST.getWavesPerEU(ST.getFlatWorkGroupSizes(F), CurrentLocalMemUsage, F)
- .second;
-
- // Round up to the next tier of usage.
- unsigned MaxSizeWithWaveCount =
- ST.getMaxLocalMemSizeWithWaveCount(MaxOccupancy, F);
-
- // Program may already use more LDS than is usable at maximum occupancy.
- if (CurrentLocalMemUsage > MaxSizeWithWaveCount)
return false;
-
- LocalMemLimit = MaxSizeWithWaveCount;
+ }
LLVM_DEBUG(dbgs() << F.getName() << " uses " << CurrentLocalMemUsage
- << " bytes of LDS\n"
- << " Rounding size to " << MaxSizeWithWaveCount
- << " with a maximum occupancy of " << MaxOccupancy << '\n'
- << " and " << (LocalMemLimit - CurrentLocalMemUsage)
- << " available for promotion\n");
-
+ << " bytes of LDS\n");
return true;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d2a2f81255344..438bc8040fcd7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -499,6 +499,12 @@ static cl::opt<bool> EnableImageIntrinsicOptimizer(
cl::desc("Enable image intrinsic optimizer pass"), cl::init(true),
cl::Hidden);
+// Gate insertion of the AMDGPU LDS Buffering pass into the default pipeline.
+static cl::opt<bool> EnableLDSBuffering(
+ "amdgpu-enable-lds-buffering",
+ cl::desc("Enable AMDGPU LDS Buffering pass in the default pipeline"),
+ cl::init(false), cl::Hidden);
+
static cl::opt<bool>
EnableLoopPrefetch("amdgpu-loop-prefetch",
cl::desc("Enable loop data prefetch on AMDGPU"),
@@ -588,6 +594,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUPreLegalizerCombinerPass(*PR);
initializeAMDGPURegBankCombinerPass(*PR);
initializeAMDGPUPromoteAllocaPass(*PR);
+ initializeAMDGPULDSBufferingLegacyPass(*PR);
initializeAMDGPUCodeGenPreparePass(*PR);
initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
@@ -833,6 +840,30 @@ parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
return make_error<StringError>("invalid parameter", inconvertibleErrorCode());
}
+static Expected<unsigned> parseAMDGPULDSBufferingMaxBytes(StringRef Params) {
+ unsigned Result = 64;
+ while (!Params.empty()) {
+ StringRef Param;
+ std::tie(Param, Params) = Params.split(';');
+ if (Param.empty())
+ continue;
+ if (!Param.consume_front("max-bytes=")) {
+ return make_error<StringError>(
+ formatv("invalid AMDGPULDSBuffering pass parameter '{0}' ", Param)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ unsigned Parsed = 0;
+ if (Param.getAsInteger(10, Parsed)) {
+ return make_error<StringError>(
+ formatv("invalid AMDGPULDSBuffering max-bytes '{0}' ", Param).str(),
+ inconvertibleErrorCode());
+ }
+ Result = Parsed;
+ }
+ return Result;
+}
+
Expected<AMDGPUAttributorOptions>
parseAMDGPUAttributorPassOptions(StringRef Params) {
AMDGPUAttributorOptions Result;
@@ -1364,6 +1395,9 @@ void AMDGPUPassConfig::addIRPasses() {
if (TM.getOptLevel() > CodeGenOptLevel::None) {
addPass(createAMDGPUPromoteAlloca());
+ // Run per-thread LDS buffering after promote-alloca to use leftover LDS.
+ if (TM.getTargetTriple().isAMDGCN() && EnableLDSBuffering)
+ addPass(createAMDGPULDSBufferingLegacyPass());
if (isPassEnabled(EnableScalarIRPasses))
addStraightLineScalarOptimizationPasses();
@@ -2148,6 +2182,9 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
if (TM.getOptLevel() > CodeGenOptLevel::None) {
addFunctionPass(AMDGPUPromoteAllocaPass(TM), PMW);
+ // Run per-thread LDS buffering after promote-alloca to use leftover LDS.
+ if (TM.getTargetTriple().isAMDGCN() && EnableLDSBuffering)
+ addFunctionPass(AMDGPULDSBufferingPass(TM), PMW);
if (isPassEnabled(EnableScalarIRPasses))
addStraightLineScalarOptimizationPasses(PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 782cbfa76e6e9..034e1be4de924 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -77,6 +77,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPULowerKernelArguments.cpp
AMDGPULowerKernelAttributes.cpp
AMDGPULowerModuleLDSPass.cpp
+ AMDGPULDSBuffering.cpp
AMDGPUPrepareAGPRAlloc.cpp
AMDGPULowerExecSync.cpp
AMDGPUSwLowerLDS.cpp
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
new file mode 100644
index 0000000000000..3daadc544c20a
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
@@ -0,0 +1,235 @@
+//===-- AMDGPULDSUtils.cpp - AMDGPU LDS utilities ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Shared helpers for computing LDS usage and limits for an AMDGPU function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Utils/AMDGPULDSUtils.h"
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Alignment.h"
+
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+// Work-group / work-item query IR helpers
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+// Read local size Y/Z from the HSA dispatch packet.
+static std::pair<Value *, Value *>
+getLocalSizeYZFromDispatch(IRBuilderBase &Builder, Module &M,
+ const AMDGPUSubtarget &ST) {
+ Function &F = *Builder.GetInsertBlock()->getParent();
+
+ CallInst *DispatchPtr =
+ cast<CallInst>(Builder.CreateIntrinsic(Intrinsic::amdgcn_dispatch_ptr, {}));
+ DispatchPtr->addRetAttr(Attribute::NoAlias);
+ DispatchPtr->addRetAttr(Attribute::NonNull);
+ F.removeFnAttr("amdgpu-no-dispatch-ptr");
+ DispatchPtr->addDereferenceableRetAttr(64);
+
+ Type *I32Ty = Type::getInt32Ty(M.getContext());
+ Value *GEPXY = Builder.CreateConstInBoundsGEP1_64(I32Ty, DispatchPtr, 1);
+ LoadInst *LoadXY = Builder.CreateAlignedLoad(I32Ty, GEPXY, Align(4));
+ Value *GEPZU = Builder.CreateConstInBoundsGEP1_64(I32Ty, DispatchPtr, 2);
+ LoadInst *LoadZU = Builder.CreateAlignedLoad(I32Ty, GEPZU, Align(4));
+
+ MDNode *MD = MDNode::get(M.getContext(), {});
+ LoadXY->setMetadata(LLVMContext::MD_invariant_load, MD);
+ LoadZU->setMetadata(LLVMContext::MD_invariant_load, MD);
+ ST.makeLIDRangeMetadata(LoadZU);
+
+ Value *Y = Builder.CreateLShr(LoadXY, 16);
+ return {Y, LoadZU};
+}
+
+} // end anonymous namespace
+
+Value *AMDGPU::getWorkitemID(IRBuilderBase &Builder, Module &M,
+ const AMDGPUSubtarget &ST, unsigned N) {
+ Function *F = Builder.GetInsertBlock()->getParent();
+ Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
+ StringRef AttrName;
+
+ switch (N) {
+ case 0:
+ IntrID = Intrinsic::amdgcn_workitem_id_x;
+ AttrName = "amdgpu-no-workitem-id-x";
+ break;
+ case 1:
+ IntrID = Intrinsic::amdgcn_workitem_id_y;
+ AttrName = "amdgpu-no-workitem-id-y";
+ break;
+ case 2:
+ IntrID = Intrinsic::amdgcn_workitem_id_z;
+ AttrName = "amdgpu-no-workitem-id-z";
+ break;
+ default:
+ llvm_unreachable("invalid dimension");
+ }
+
+ Function *WorkitemIdFn = Intrinsic::getOrInsertDeclaration(&M, IntrID);
+ CallInst *CI = cast<CallInst>(Builder.CreateCall(WorkitemIdFn));
+ ST.makeLIDRangeMetadata(CI);
+ F->removeFnAttr(AttrName);
+ return CI;
+}
+
+Value *AMDGPU::buildLinearThreadId(IRBuilderBase &Builder, Module &M,
+ const AMDGPUSubtarget &ST) {
+ Value *TCntY = nullptr;
+ Value *TCntZ = nullptr;
+ std::tie(TCntY, TCntZ) = getLocalSizeYZFromDispatch(Builder, M, ST);
+ Value *TIdX = getWorkitemID(Builder, M, ST, 0);
+ Value *TIdY = getWorkitemID(Builder, M, ST, 1);
+ Value *TIdZ = getWorkitemID(Builder, M, ST, 2);
+
+ Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ, "", true, true);
+ Tmp0 = Builder.CreateMul(Tmp0, TIdX);
+ Value *Tmp1 = Builder.CreateMul(TIdY, TCntZ, "", true, true);
+ Value *TID = Builder.CreateAdd(Tmp0, Tmp1);
+ TID = Builder.CreateAdd(TID, TIdZ);
+ return TID;
+}
+
+//===----------------------------------------------------------------------===//
+// LDS budget computation
+//===----------------------------------------------------------------------===//
+
+AMDGPU::AMDGPULDSBudget AMDGPU::computeLDSBudget(const Function &F,
+ const TargetMachine &TM) {
+ AMDGPU::AMDGPULDSBudget Result;
+
+ const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
+ const Module *M = F.getParent();
+ const DataLayout &DL = M->getDataLayout();
+
+ // If the function has any arguments in the local address space, then it's
+ // possible these arguments require the entire local memory space, so
+ // we cannot use local memory.
+ FunctionType *FTy = F.getFunctionType();
+ for (Type *ParamTy : FTy->params()) {
+ PointerType *PtrTy = dyn_cast<PointerType>(ParamTy);
+ if (PtrTy && PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
+ Result.limit = 0;
+ Result.promotable = false;
+ Result.disabledDueToLocalArg = true;
+ return Result;
+ }
+ }
+
+ uint32_t LocalMemLimit = ST.getAddressableLocalMemorySize();
+ if (LocalMemLimit == 0) {
+ Result.limit = 0;
+ Result.promotable = false;
+ return Result;
+ }
+
+ SmallVector<const Constant *, 16> Stack;
+ SmallPtrSet<const Constant *, 8> VisitedConstants;
+ SmallPtrSet<const GlobalVariable *, 8> UsedLDS;
+
+ auto visitUsers = [&](const GlobalVariable *GV, const Constant *Val) -> bool {
+ for (const User *U : Val->users()) {
+ if (const Instruction *Use = dyn_cast<Instruction>(U)) {
+ if (Use->getParent()->getParent() == &F)
+ return true;
+ } else {
+ const Constant *C = cast<Constant>(U);
+ if (VisitedConstants.insert(C).second)
+ Stack.push_back(C);
+ }
+ }
+ return false;
+ };
+
+ for (const GlobalVariable &GV : M->globals()) {
+ if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
+ continue;
+
+ if (visitUsers(&GV, &GV)) {
+ UsedLDS.insert(&GV);
+ Stack.clear();
+ continue;
+ }
+
+ while (!Stack.empty()) {
+ const Constant *C = Stack.pop_back_val();
+ if (visitUsers(&GV, C)) {
+ UsedLDS.insert(&GV);
+ Stack.clear();
+ break;
+ }
+ }
+ }
+
+ SmallVector<std::pair<uint64_t, Align>, 16> AllocatedSizes;
+ AllocatedSizes.reserve(UsedLDS.size());
+
+ for (const GlobalVariable *GV : UsedLDS) {
+ Align Alignment =
+ DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());
+ uint64_t AllocSize = DL.getTypeAllocSize(GV->getValueType());
+
+ // HIP uses an extern unsized array in local address space for dynamically
+ // allocated shared memory.
+ if (GV->hasExternalLinkage() && AllocSize == 0) {
+ Result.limit = 0;
+ Result.promotable = false;
+ Result.disabledDueToExternDynShared = true;
+ return Result;
+ }
+
+ AllocatedSizes.emplace_back(AllocSize, Alignment);
+ }
+
+ // Sort to try to estimate the worst case alignment padding.
+ llvm::sort(AllocatedSizes, llvm::less_second());
+
+ uint32_t CurrentLocalMemUsage = 0;
+ for (auto Alloc : AllocatedSizes) {
+ CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alloc.second);
+ CurrentLocalMemUsage += Alloc.first;
+ }
+
+ unsigned MaxOccupancy =
+ ST.getWavesPerEU(ST.getFlatWorkGroupSizes(F), CurrentLocalMemUsage, F)
+ .second;
+
+ unsigned MaxSizeWithWaveCount =
+ ST.getMaxLocalMemSizeWithWaveCount(MaxOccupancy, F);
+
+ if (CurrentLocalMemUsage > MaxSizeWithWaveCount) {
+ Result.currentUsage = CurrentLocalMemUsage;
+ Result.limit = MaxSizeWithWaveCount;
+ Result.maxOccupancy = MaxOccupancy;
+ Result.promotable = false;
+ return Result;
+ }
+
+ Result.currentUsage = CurrentLocalMemUsage;
+ Result.limit = MaxSizeWithWaveCount;
+ Result.maxOccupancy = MaxOccupancy;
+ Result.promotable = true;
+ return Result;
+}
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
new file mode 100644
index 0000000000000..18690a77647c1
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
@@ -0,0 +1,53 @@
+//===-- AMDGPULDSUtils.h - AMDGPU LDS utilities ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Shared helpers for computing LDS usage and limits for an AMDGPU function.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPULDSUTILS_H
+#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPULDSUTILS_H
+
+#include <cstdint>
+#include <utility>
+
+namespace llvm {
+
+class AMDGPUSubtarget;
+class Function;
+class IRBuilderBase;
+class Module;
+class TargetMachine;
+class Value;
+
+namespace AMDGPU {
+
+/// Get workitem id for dimension N (0,1,2).
+Value *getWorkitemID(IRBuilderBase &Builder, Module &M,
+ const AMDGPUSubtarget &ST, unsigned N);
+
+/// Compute linear thread id within a workgroup.
+Value *buildLinearThreadId(IRBuilderBase &Builder, Module &M,
+ const AMDGPUSubtarget &ST);
+
+struct AMDGPULDSBudget {
+ uint32_t currentUsage = 0;
+ uint32_t limit = 0;
+ unsigned maxOccupancy = 0;
+ bool promotable = false;
+ bool disabledDueToLocalArg = false;
+ bool disabledDueToExternDynShared = false;
+};
+
+AMDGPULDSBudget computeLDSBudget(const Function &F, const TargetMachine &TM);
+
+} // end namespace AMDGPU
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPULDSUTILS_H
diff --git a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
index 7b2200d8bc488..4633763d23f22 100644
--- a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -4,6 +4,7 @@ add_llvm_component_library(LLVMAMDGPUUtils
AMDGPUDelayedMCExpr.cpp
AMDGPUPALMetadata.cpp
AMDKernelCodeTUtils.cpp
+ AMDGPULDSUtils.cpp
LINK_COMPONENTS
Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/lds-buffering.ll b/llvm/test/CodeGen/AMDGPU/lds-buffering.ll
new file mode 100644
index 0000000000000..64165c3cb1e40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-buffering.ll
@@ -0,0 +1,183 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes='amdgpu-lds-buffering<max-bytes=64>' -S %s | FileCheck %s
+
+%state = type { <4 x i32>, <4 x i32> }
+
+;===---------------------------------------------------------------------===//
+; Positive cases
+;===---------------------------------------------------------------------===//
+
+; CHECK: @ldsbuf_test.ldsbuf = internal unnamed_addr addrspace(3) global
+; CHECK: @ldsbuf_complex.ldsbuf = internal unnamed_addr addrspace(3) global
+; CHECK: @ldsbuf_rocrand_preserved_subsequence.ldsbuf = internal unnamed_addr addrspace(3) global
+
+; Basic positive case: a single global load whose only use is a store back to
+; the same pointer. Include an intervening (potentially-aliasing) global store
+; to keep the value live and avoid trivially folding away.
+; CHECK-LABEL: @ldsbuf_test(
+; CHECK: %[[SLOT_BASIC:[^ ]+]] = getelementptr inbounds {{.*}}, ptr addrspace(3) @ldsbuf_test.ldsbuf, i32 0, i32 %
+; CHECK: call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3){{.*}}%[[SLOT_BASIC]], ptr addrspace(1){{.*}}%p, i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1){{.*}}%p, ptr addrspace(3){{.*}}%[[SLOT_BASIC]], i64 16, i1 false)
+define amdgpu_kernel void @ldsbuf_test(ptr addrspace(1) %p, ptr addrspace(1) %q) #0 {
+entry:
+ %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+ store i32 0, ptr addrspace(1) %q, align 4
+ store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+ ret void
+}
+
+; "Complexity" positive case: keep the loaded value live across control flow
+; and additional global memory operations, then store back to the same pointer.
+; CHECK-LABEL: @ldsbuf_complex(
+; CHECK: %[[SLOT_COMPLEX:[^ ]+]] = getelementptr inbounds {{.*}}, ptr addrspace(3) @ldsbuf_complex.ldsbuf, i32 0, i32 %
+; CHECK: call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3){{.*}}%[[SLOT_COMPLEX]], ptr addrspace(1){{.*}}%p, i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1){{.*}}%p, ptr addrspace(3){{.*}}%[[SLOT_COMPLEX]], i64 16, i1 false)
+define amdgpu_kernel void @ldsbuf_complex(ptr addrspace(1) %p, ptr addrspace(1) %q, i1 %c) #0 {
+entry:
+ %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+ br i1 %c, label %then, label %else
+
+then:
+ store i32 1, ptr addrspace(1) %q, align 4
+ br label %merge
+
+else:
+ store i32 2, ptr addrspace(1) %q, align 4
+ br label %merge
+
+merge:
+ store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+ ret void
+}
+
+; rocRand-inspired positive case: model a per-thread RNG state where part of
+; the state (a preserved 16B "subsequence" field) is loaded and stored back
+; unchanged, but kept live across a non-trivial loop body.
+; CHECK-LABEL: @ldsbuf_rocrand_preserved_subsequence(
+; CHECK: %[[TAILPTR:[^ ]+]] = getelementptr inbounds %state, ptr addrspace(1) %state_ptr, i32 0, i32 1
+; CHECK: %[[SLOT_TAIL:[^ ]+]] = getelementptr inbounds {{.*}}, ptr addrspace(3) @ldsbuf_rocrand_preserved_subsequence.ldsbuf, i32 0, i32 %
+; CHECK: call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3){{.*}}%[[SLOT_TAIL]], ptr addrspace(1){{.*}}%[[TAILPTR]], i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1){{.*}}%[[TAILPTR]], ptr addrspace(3){{.*}}%[[SLOT_TAIL]], i64 16, i1 false)
+define amdgpu_kernel void @ldsbuf_rocrand_preserved_subsequence(ptr addrspace(1) %state_ptr,
+ ptr addrspace(1) %out) #0 {
+entry:
+ %tailptr = getelementptr inbounds %state, ptr addrspace(1) %state_ptr, i32 0, i32 1
+ %tail = load <4 x i32>, ptr addrspace(1) %tailptr, align 16
+
+ br label %loop
+
+loop:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi i32 [ 1, %entry ], [ %acc.next, %loop ]
+
+ %shl = shl i32 %acc, 6
+ %xor = xor i32 %shl, %acc
+ %shr = lshr i32 %xor, 13
+ %acc.next = xor i32 %shr, 1234567
+
+ %idx = zext i32 %i to i64
+ %outp = getelementptr inbounds float, ptr addrspace(1) %out, i64 %idx
+ %f = uitofp i32 %acc.next to float
+ store float %f, ptr addrspace(1) %outp, align 4
+
+ %i.next = add nuw i32 %i, 1
+ %done = icmp eq i32 %i.next, 8
+ br i1 %done, label %exit, label %loop
+
+exit:
+ store <4 x i32> %tail, ptr addrspace(1) %tailptr, align 16
+ ret void
+}
+
+;===---------------------------------------------------------------------===//
+; Negative coverage: patterns that must NOT be transformed.
+;===---------------------------------------------------------------------===//
+
+; Negative: atomic operations are excluded (must not transform).
+; CHECK-LABEL: @ldsbuf_atomic(
+; CHECK: load atomic <4 x i32>, ptr addrspace(1) %p unordered, align 16
+; CHECK: store atomic <4 x i32> {{.*}}, ptr addrspace(1) %p unordered, align 16
+define amdgpu_kernel void @ldsbuf_atomic(ptr addrspace(1) %p) #0 {
+entry:
+ %ld = load atomic <4 x i32>, ptr addrspace(1) %p unordered, align 16
+ store atomic <4 x i32> %ld, ptr addrspace(1) %p unordered, align 16
+ ret void
+}
+
+; Negative: volatile operations are excluded (must not transform).
+; CHECK-LABEL: @ldsbuf_volatile(
+; CHECK: load volatile <4 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: store volatile <4 x i32> {{.*}}, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_volatile(ptr addrspace(1) %p) #0 {
+entry:
+ %ld = load volatile <4 x i32>, ptr addrspace(1) %p, align 16
+ store volatile <4 x i32> %ld, ptr addrspace(1) %p, align 16
+ ret void
+}
+
+; Negative: alignment requirement (min 16) must be met.
+; CHECK-LABEL: @ldsbuf_misaligned(
+; CHECK: load <4 x i32>, ptr addrspace(1) %p, align 8
+; CHECK: store <4 x i32> {{.*}}, ptr addrspace(1) %p, align 8
+define amdgpu_kernel void @ldsbuf_misaligned(ptr addrspace(1) %p) #0 {
+entry:
+ %ld = load <4 x i32>, ptr addrspace(1) %p, align 8
+ store <4 x i32> %ld, ptr addrspace(1) %p, align 8
+ ret void
+}
+
+; Negative: size limit (max-bytes=64) must be respected.
+; CHECK-LABEL: @ldsbuf_too_large(
+; CHECK: load <20 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: store <20 x i32> {{.*}}, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_too_large(ptr addrspace(1) %p) #0 {
+entry:
+ %ld = load <20 x i32>, ptr addrspace(1) %p, align 16
+ store <20 x i32> %ld, ptr addrspace(1) %p, align 16
+ ret void
+}
+
+; Negative: non-global pointers are excluded (only addrspace(1) is supported).
+; CHECK-LABEL: @ldsbuf_non_global_ptr(
+; CHECK: load <4 x i32>, ptr %p, align 16
+; CHECK: store <4 x i32> {{.*}}, ptr %p, align 16
+define amdgpu_kernel void @ldsbuf_non_global_ptr(ptr %p) #0 {
+entry:
+ %ld = load <4 x i32>, ptr %p, align 16
+ store <4 x i32> %ld, ptr %p, align 16
+ ret void
+}
+
+; Negative: the load must have exactly one use (the final store).
+; CHECK-LABEL: @ldsbuf_multiple_uses(
+; CHECK: %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: extractelement <4 x i32> %ld, i32 0
+; CHECK: store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_multiple_uses(ptr addrspace(1) %p, ptr addrspace(1) %q) #0 {
+entry:
+ %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+ %e = extractelement <4 x i32> %ld, i32 0
+ store i32 %e, ptr addrspace(1) %q, align 4
+ store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+ ret void
+}
+
+; Negative: budget rejection. With large pre-existing LDS usage and large
+; workgroup size, per-thread slots should be rejected by the LDS budget check.
+ at ldsbuf_budget_reject.big = internal addrspace(3) global [8192 x i32] zeroinitializer, align 16
+
+; CHECK-LABEL: @ldsbuf_budget_reject(
+; CHECK: load <16 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: store <16 x i32> {{.*}}, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_budget_reject(ptr addrspace(1) %p) #1 {
+entry:
+ %x = load i32, ptr addrspace(3) getelementptr inbounds ([8192 x i32], ptr addrspace(3) @ldsbuf_budget_reject.big, i32 0, i32 0), align 16
+ call void @llvm.donothing()
+ %ld = load <16 x i32>, ptr addrspace(1) %p, align 16
+ store <16 x i32> %ld, ptr addrspace(1) %p, align 16
+ ret void
+}
+
+declare void @llvm.donothing()
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "uniform-work-group-size"="true" }
+attributes #1 = { "amdgpu-flat-work-group-size"="1024,1024" "uniform-work-group-size"="true" }
More information about the llvm-commits
mailing list