[llvm] AMDGPU: share LDS budget logic and add experimental LDS buffering pass (PR #166388)

Yaxun Liu via llvm-commits llvm-commits at lists.llvm.org
Tue Jan 6 15:53:13 PST 2026


https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/166388

>From 5d01732237cf90919c577b607babd24596dabe27 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Tue, 6 Jan 2026 16:41:48 -0500
Subject: [PATCH] AMDGPU: add LDS buffering pass and shared LDS budget helper

Introduce an experimental per-thread LDS buffering IR pass that converts
eligible global load/store pairs into global<->LDS memcpy pairs, using only
leftover LDS budget computed with the same occupancy-tier constraints as alloca
promotion.

Add a shared LDS budget utility, wire the pass as a parameterized pipeline
pass, gate default-pipeline insertion behind a hidden flag on AMDGCN, and add
consolidated CodeGen coverage (positive, complexity, rocRand-inspired, and
negative cases).
---
 llvm/lib/Target/AMDGPU/AMDGPU.h               |  17 ++
 llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp | 272 ++++++++++++++++++
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |  10 +-
 .../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 122 +-------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  37 +++
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 +
 .../Target/AMDGPU/Utils/AMDGPULDSUtils.cpp    | 235 +++++++++++++++
 llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h |  53 ++++
 llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt   |   1 +
 llvm/test/CodeGen/AMDGPU/lds-buffering.ll     | 183 ++++++++++++
 10 files changed, 816 insertions(+), 115 deletions(-)
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
 create mode 100644 llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
 create mode 100644 llvm/test/CodeGen/AMDGPU/lds-buffering.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5df11a45b4889..24c86b791dc39 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -270,6 +270,23 @@ struct AMDGPUPromoteAllocaToVectorPass
   TargetMachine &TM;
 };
 
+// Buffer selected per-thread global memory through LDS to improve
+// performance in memory-bound kernels. This runs late and is separate
+// from alloca promotion.
+struct AMDGPULDSBufferingPass : PassInfoMixin<AMDGPULDSBufferingPass> {
+  AMDGPULDSBufferingPass(const AMDGPUTargetMachine &TM, unsigned MaxBytes = 64)
+      : TM(TM), MaxBytes(MaxBytes) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+private:
+  const AMDGPUTargetMachine &TM;
+  unsigned MaxBytes;
+};
+
+// Legacy PM wrapper for LDS buffering
+FunctionPass *createAMDGPULDSBufferingLegacyPass();
+void initializeAMDGPULDSBufferingLegacyPass(PassRegistry &);
+
 struct AMDGPUAtomicOptimizerPass : PassInfoMixin<AMDGPUAtomicOptimizerPass> {
   AMDGPUAtomicOptimizerPass(TargetMachine &TM, ScanOptions ScanImpl)
       : TM(TM), ScanImpl(ScanImpl) {}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp b/llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp
new file mode 100644
index 0000000000000..a4c022e240e25
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPULDSBuffering.cpp
@@ -0,0 +1,272 @@
+//===-- AMDGPULDSBuffering.cpp - Per-thread LDS buffering -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass buffers selected per-thread global memory accesses through LDS
+// (addrspace(3)) to improve performance in memory-bound kernels.
+//
+// This is intended to help cases where a value is loaded from global memory and
+// later stored back to (the same) global location, but intervening memory
+// operations (e.g. potentially-aliasing accesses) inhibit load/store
+// forwarding, keeping the load and store "live" and increasing global traffic
+// and cache pressure.
+//
+// The pass runs late in the pipeline, after SROA and AMDGPUPromoteAlloca,
+// using only leftover LDS budget to avoid interfering with other LDS
+// optimizations. It respects the same LDS budget constraints as
+// AMDGPUPromoteAlloca, ensuring that LDS usage remains within occupancy
+// tier limits.
+//
+// Current implementation handles the simplest pattern: a load from global
+// memory whose only use is a store back to the same pointer. This pattern is
+// transformed into a pair of memcpy operations (global->LDS and LDS->global),
+// effectively moving the value through LDS instead of accessing global memory
+// directly.
+//
+// This pass was inspired by finding that some rocrand performance tests
+// show better performance when global memory is buffered through LDS
+// instead of being loaded/stored to registers directly. This optimization
+// is experimental and must be enabled in the default pipeline via the
+// -amdgpu-enable-lds-buffering flag (or explicitly scheduled via
+// -passes='amdgpu-lds-buffering<...>').
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUTargetMachine.h"
+#include "GCNSubtarget.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPULDSUtils.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/Debug.h"
+#include <algorithm>
+
+#define DEBUG_TYPE "amdgpu-lds-buffering"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPULDSBufferingImpl {
+  const AMDGPUTargetMachine &TM;
+  unsigned MaxBytes;
+  Module *Mod = nullptr;
+  const DataLayout *DL = nullptr;
+
+public:
+  AMDGPULDSBufferingImpl(const AMDGPUTargetMachine &TM, unsigned MaxBytes)
+      : TM(TM), MaxBytes(MaxBytes) {}
+
+  bool run(Function &F) {
+    LLVM_DEBUG(dbgs() << "[LDSBuffer] Visit function: " << F.getName() << '\n');
+    if (!AMDGPU::isEntryFunctionCC(F.getCallingConv()))
+      return false;
+
+    Mod = F.getParent();
+    DL = &Mod->getDataLayout();
+
+    auto Budget = AMDGPU::computeLDSBudget(F, TM);
+    if (!Budget.promotable)
+      return false;
+    uint32_t localUsage = Budget.currentUsage;
+    uint32_t localLimit = Budget.limit;
+
+    const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
+    unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(F).second;
+
+    bool Changed = false;
+    unsigned NumTransformed = 0;
+
+    // Minimal pattern: a load from AS(1) whose only use is a store back to the
+    // exact same pointer later. Replace with global<->LDS memcpy pair to
+    // shorten the live range and free VGPRs.
+    SmallVector<Instruction *> ToErase;
+    for (BasicBlock &BB : F) {
+      for (Instruction &I : llvm::make_early_inc_range(BB)) {
+        auto *LI = dyn_cast<LoadInst>(&I);
+        if (!LI || LI->isVolatile() || LI->isAtomic())
+          continue;
+
+        Type *ValTy = LI->getType();
+        if (!ValTy->isFirstClassType())
+          continue;
+
+        Value *Ptr = LI->getPointerOperand();
+        auto *PtrTy = cast<PointerType>(Ptr->getType());
+        if (PtrTy->getAddressSpace() != AMDGPUAS::GLOBAL_ADDRESS)
+          continue;
+
+        if (!LI->hasOneUse())
+          continue;
+        auto *SI = dyn_cast<StoreInst>(LI->user_back());
+        if (!SI || SI->isVolatile() || SI->isAtomic())
+          continue;
+        if (SI->getValueOperand() != LI)
+          continue;
+
+        Value *SPtr = SI->getPointerOperand();
+        if (SPtr != Ptr)
+          continue;
+
+        TypeSize TS = DL->getTypeStoreSize(ValTy);
+        if (TS.isScalable())
+          continue;
+        uint64_t Size = TS.getFixedValue();
+        if (Size == 0 || Size > MaxBytes)
+          continue;
+        Align MinAlign = Align(16);
+        Align LoadAlign = LI->getAlign();
+        Align StoreAlign = SI->getAlign();
+        Align Alignment = std::min(LoadAlign, StoreAlign);
+        if (Alignment < MinAlign)
+          continue;
+
+        // Create LDS slot near the load and emit memcpy global->LDS.
+        LLVM_DEBUG({
+          dbgs() << "[LDSBuffer] Candidate found: load->store same ptr in "
+                 << F.getName() << '\n';
+          dbgs() << "            size=" << Size
+                 << "B, loadAlign=" << LoadAlign.value()
+                 << ", storeAlign=" << StoreAlign.value()
+                 << ", chosenAlign=" << Alignment.value()
+                 << ", ptr AS=" << PtrTy->getAddressSpace() << "\n";
+        });
+        IRBuilder<> BLoad(LI);
+
+        // Ensure LDS budget allows allocating a per-thread slot.
+        uint32_t NewSize = alignTo(localUsage, Alignment);
+        NewSize += WorkGroupSize * static_cast<uint32_t>(Size);
+        if (NewSize > localLimit)
+          continue;
+        localUsage = NewSize;
+        auto [GV, SlotPtr] =
+            createLDSGlobalAndThreadSlot(F, ValTy, Alignment, "ldsbuf", BLoad);
+        // memcpy p3 <- p1
+        LLVM_DEBUG(dbgs() << "[LDSBuffer] Insert memcpy global->LDS: "
+                          << GV->getName() << ", bytes=" << Size
+                          << ", align=" << Alignment.value() << '\n');
+        BLoad.CreateMemCpy(SlotPtr, Alignment, Ptr, LoadAlign, TS);
+
+        // Replace the final store with memcpy LDS->global.
+        IRBuilder<> BStore(SI);
+        LLVM_DEBUG(dbgs() << "[LDSBuffer] Insert memcpy LDS->global: "
+                          << GV->getName() << ", bytes=" << Size
+                          << ", align=" << Alignment.value() << '\n');
+        BStore.CreateMemCpy(SPtr, StoreAlign, SlotPtr, Alignment, TS);
+
+        ToErase.push_back(SI);
+        ToErase.push_back(LI);
+        LLVM_DEBUG(dbgs() << "[LDSBuffer] Erase original load/store pair\n");
+        Changed = true;
+        ++NumTransformed;
+      }
+    }
+
+    for (Instruction *E : ToErase)
+      E->eraseFromParent();
+
+    LLVM_DEBUG(dbgs() << "[LDSBuffer] Transformations applied: "
+                      << NumTransformed << "\n");
+
+    return Changed;
+  }
+
+private:
+  // Create an LDS array [WGSize x ElemTy] and return pointer to per-thread
+  // slot.
+  std::pair<GlobalVariable *, Value *>
+  createLDSGlobalAndThreadSlot(Function &F, Type *ElemTy, Align Alignment,
+                               StringRef BaseName, IRBuilder<> &Builder) {
+    const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
+    unsigned WorkGroupSize = ST.getFlatWorkGroupSizes(F).second;
+    Type *ArrTy = ArrayType::get(ElemTy, WorkGroupSize);
+    GlobalVariable *GV = new GlobalVariable(
+        *Mod, ArrTy, /*isConstant=*/false, GlobalValue::InternalLinkage,
+        PoisonValue::get(ArrTy), (F.getName() + "." + BaseName).str(), nullptr,
+        GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS);
+    GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+    GV->setAlignment(Alignment);
+
+    LLVM_DEBUG({
+      dbgs() << "[LDSBuffer] Create LDS global: name=" << GV->getName()
+             << ", elemTy=" << *ElemTy << ", WGSize=" << WorkGroupSize
+             << ", align=" << Alignment.value() << '\n';
+    });
+
+    Value *LinearTID =
+        AMDGPU::buildLinearThreadId(Builder, *Mod, ST);
+    LLVMContext &Ctx = Mod->getContext();
+    Value *Indices[] = {Constant::getNullValue(Type::getInt32Ty(Ctx)),
+                        LinearTID};
+    Value *SlotPtr = Builder.CreateInBoundsGEP(ArrTy, GV, Indices);
+    return {GV, SlotPtr};
+  }
+};
+
+} // end anonymous namespace
+
+PreservedAnalyses AMDGPULDSBufferingPass::run(Function &F,
+                                              FunctionAnalysisManager &AM) {
+  bool Changed = AMDGPULDSBufferingImpl(TM, MaxBytes).run(F);
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA;
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
+
+//===----------------------------------------------------------------------===//
+// Legacy PM wrapper
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class AMDGPULDSBufferingLegacy : public FunctionPass {
+public:
+  static char ID;
+  AMDGPULDSBufferingLegacy() : FunctionPass(ID) {}
+
+  StringRef getPassName() const override { return "AMDGPU LDS Buffering"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    FunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (skipFunction(F))
+      return false;
+    if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>())
+      return AMDGPULDSBufferingImpl(TPC->getTM<AMDGPUTargetMachine>(),
+                                    /*MaxBytes=*/64)
+          .run(F);
+    return false;
+  }
+};
+
+} // end anonymous namespace
+
+char AMDGPULDSBufferingLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(AMDGPULDSBufferingLegacy, DEBUG_TYPE,
+                      "AMDGPU per-thread LDS buffering", false, false)
+INITIALIZE_PASS_END(AMDGPULDSBufferingLegacy, DEBUG_TYPE,
+                    "AMDGPU per-thread LDS buffering", false, false)
+
+FunctionPass *llvm::createAMDGPULDSBufferingLegacyPass() {
+  return new AMDGPULDSBufferingLegacy();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index f464fbf31c754..e3320d609f9a5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -71,11 +71,13 @@ FUNCTION_PASS("amdgpu-promote-kernel-arguments",
               AMDGPUPromoteKernelArgumentsPass())
 FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass())
 FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass())
+FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
 FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes",
               AMDGPUUnifyDivergentExitNodesPass())
 FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass())
-FUNCTION_PASS("si-annotate-control-flow", SIAnnotateControlFlowPass(*static_cast<const GCNTargetMachine *>(this)))
-FUNCTION_PASS("amdgpu-uniform-intrinsic-combine", AMDGPUUniformIntrinsicCombinePass())
+FUNCTION_PASS("si-annotate-control-flow",
+              SIAnnotateControlFlowPass(
+                  *static_cast<const GCNTargetMachine *>(this)))
 #undef FUNCTION_PASS
 
 #ifndef FUNCTION_ANALYSIS
@@ -93,6 +95,10 @@ FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA())
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
+FUNCTION_PASS_WITH_PARAMS(
+    "amdgpu-lds-buffering", "AMDGPULDSBufferingPass",
+    [=](unsigned MaxBytes) { return AMDGPULDSBufferingPass(*this, MaxBytes); },
+    parseAMDGPULDSBufferingMaxBytes, "max-bytes")
 FUNCTION_PASS_WITH_PARAMS(
     "amdgpu-atomic-optimizer",
     "AMDGPUAtomicOptimizerPass",
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index d0bc36c2f8501..866ae62d9e5d3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -28,6 +28,7 @@
 #include "AMDGPU.h"
 #include "GCNSubtarget.h"
 #include "Utils/AMDGPUBaseInfo.h"
+#include "Utils/AMDGPULDSUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
@@ -1460,129 +1461,24 @@ void AMDGPUPromoteAllocaImpl::analyzePromoteToLDS(AllocaAnalysis &AA) const {
 }
 
 bool AMDGPUPromoteAllocaImpl::hasSufficientLocalMem(const Function &F) {
-
-  FunctionType *FTy = F.getFunctionType();
-  const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
-
-  // If the function has any arguments in the local address space, then it's
-  // possible these arguments require the entire local memory space, so
-  // we cannot use local memory in the pass.
-  for (Type *ParamTy : FTy->params()) {
-    PointerType *PtrTy = dyn_cast<PointerType>(ParamTy);
-    if (PtrTy && PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
-      LocalMemLimit = 0;
+  AMDGPU::AMDGPULDSBudget Budget = AMDGPU::computeLDSBudget(F, TM);
+  CurrentLocalMemUsage = Budget.currentUsage;
+  LocalMemLimit = Budget.limit;
+  if (!Budget.promotable) {
+    if (Budget.disabledDueToLocalArg) {
       LLVM_DEBUG(dbgs() << "Function has local memory argument. Promoting to "
                            "local memory disabled.\n");
-      return false;
-    }
-  }
-
-  LocalMemLimit = ST.getAddressableLocalMemorySize();
-  if (LocalMemLimit == 0)
-    return false;
-
-  SmallVector<const Constant *, 16> Stack;
-  SmallPtrSet<const Constant *, 8> VisitedConstants;
-  SmallPtrSet<const GlobalVariable *, 8> UsedLDS;
-
-  auto visitUsers = [&](const GlobalVariable *GV, const Constant *Val) -> bool {
-    for (const User *U : Val->users()) {
-      if (const Instruction *Use = dyn_cast<Instruction>(U)) {
-        if (Use->getFunction() == &F)
-          return true;
-      } else {
-        const Constant *C = cast<Constant>(U);
-        if (VisitedConstants.insert(C).second)
-          Stack.push_back(C);
-      }
-    }
-
-    return false;
-  };
-
-  for (GlobalVariable &GV : Mod->globals()) {
-    if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
-      continue;
-
-    if (visitUsers(&GV, &GV)) {
-      UsedLDS.insert(&GV);
-      Stack.clear();
-      continue;
     }
-
-    // For any ConstantExpr uses, we need to recursively search the users until
-    // we see a function.
-    while (!Stack.empty()) {
-      const Constant *C = Stack.pop_back_val();
-      if (visitUsers(&GV, C)) {
-        UsedLDS.insert(&GV);
-        Stack.clear();
-        break;
-      }
-    }
-  }
-
-  const DataLayout &DL = Mod->getDataLayout();
-  SmallVector<std::pair<uint64_t, Align>, 16> AllocatedSizes;
-  AllocatedSizes.reserve(UsedLDS.size());
-
-  for (const GlobalVariable *GV : UsedLDS) {
-    Align Alignment =
-        DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());
-    uint64_t AllocSize = DL.getTypeAllocSize(GV->getValueType());
-
-    // HIP uses an extern unsized array in local address space for dynamically
-    // allocated shared memory.  In that case, we have to disable the promotion.
-    if (GV->hasExternalLinkage() && AllocSize == 0) {
-      LocalMemLimit = 0;
+    if (Budget.disabledDueToExternDynShared) {
       LLVM_DEBUG(dbgs() << "Function has a reference to externally allocated "
                            "local memory. Promoting to local memory "
                            "disabled.\n");
-      return false;
     }
-
-    AllocatedSizes.emplace_back(AllocSize, Alignment);
-  }
-
-  // Sort to try to estimate the worst case alignment padding
-  //
-  // FIXME: We should really do something to fix the addresses to a more optimal
-  // value instead
-  llvm::sort(AllocatedSizes, llvm::less_second());
-
-  // Check how much local memory is being used by global objects
-  CurrentLocalMemUsage = 0;
-
-  // FIXME: Try to account for padding here. The real padding and address is
-  // currently determined from the inverse order of uses in the function when
-  // legalizing, which could also potentially change. We try to estimate the
-  // worst case here, but we probably should fix the addresses earlier.
-  for (auto Alloc : AllocatedSizes) {
-    CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alloc.second);
-    CurrentLocalMemUsage += Alloc.first;
-  }
-
-  unsigned MaxOccupancy =
-      ST.getWavesPerEU(ST.getFlatWorkGroupSizes(F), CurrentLocalMemUsage, F)
-          .second;
-
-  // Round up to the next tier of usage.
-  unsigned MaxSizeWithWaveCount =
-      ST.getMaxLocalMemSizeWithWaveCount(MaxOccupancy, F);
-
-  // Program may already use more LDS than is usable at maximum occupancy.
-  if (CurrentLocalMemUsage > MaxSizeWithWaveCount)
     return false;
-
-  LocalMemLimit = MaxSizeWithWaveCount;
+  }
 
   LLVM_DEBUG(dbgs() << F.getName() << " uses " << CurrentLocalMemUsage
-                    << " bytes of LDS\n"
-                    << "  Rounding size to " << MaxSizeWithWaveCount
-                    << " with a maximum occupancy of " << MaxOccupancy << '\n'
-                    << " and " << (LocalMemLimit - CurrentLocalMemUsage)
-                    << " available for promotion\n");
-
+                    << " bytes of LDS\n");
   return true;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index d2a2f81255344..438bc8040fcd7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -499,6 +499,12 @@ static cl::opt<bool> EnableImageIntrinsicOptimizer(
     cl::desc("Enable image intrinsic optimizer pass"), cl::init(true),
     cl::Hidden);
 
+// Gate insertion of the AMDGPU LDS Buffering pass into the default pipeline.
+static cl::opt<bool> EnableLDSBuffering(
+    "amdgpu-enable-lds-buffering",
+    cl::desc("Enable AMDGPU LDS Buffering pass in the default pipeline"),
+    cl::init(false), cl::Hidden);
+
 static cl::opt<bool>
     EnableLoopPrefetch("amdgpu-loop-prefetch",
                        cl::desc("Enable loop data prefetch on AMDGPU"),
@@ -588,6 +594,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUPreLegalizerCombinerPass(*PR);
   initializeAMDGPURegBankCombinerPass(*PR);
   initializeAMDGPUPromoteAllocaPass(*PR);
+  initializeAMDGPULDSBufferingLegacyPass(*PR);
   initializeAMDGPUCodeGenPreparePass(*PR);
   initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
   initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
@@ -833,6 +840,30 @@ parseAMDGPUAtomicOptimizerStrategy(StringRef Params) {
   return make_error<StringError>("invalid parameter", inconvertibleErrorCode());
 }
 
+static Expected<unsigned> parseAMDGPULDSBufferingMaxBytes(StringRef Params) {
+  unsigned Result = 64;
+  while (!Params.empty()) {
+    StringRef Param;
+    std::tie(Param, Params) = Params.split(';');
+    if (Param.empty())
+      continue;
+    if (!Param.consume_front("max-bytes=")) {
+      return make_error<StringError>(
+          formatv("invalid AMDGPULDSBuffering pass parameter '{0}' ", Param)
+              .str(),
+          inconvertibleErrorCode());
+    }
+    unsigned Parsed = 0;
+    if (Param.getAsInteger(10, Parsed)) {
+      return make_error<StringError>(
+          formatv("invalid AMDGPULDSBuffering max-bytes '{0}' ", Param).str(),
+          inconvertibleErrorCode());
+    }
+    Result = Parsed;
+  }
+  return Result;
+}
+
 Expected<AMDGPUAttributorOptions>
 parseAMDGPUAttributorPassOptions(StringRef Params) {
   AMDGPUAttributorOptions Result;
@@ -1364,6 +1395,9 @@ void AMDGPUPassConfig::addIRPasses() {
 
   if (TM.getOptLevel() > CodeGenOptLevel::None) {
     addPass(createAMDGPUPromoteAlloca());
+    // Run per-thread LDS buffering after promote-alloca to use leftover LDS.
+    if (TM.getTargetTriple().isAMDGCN() && EnableLDSBuffering)
+      addPass(createAMDGPULDSBufferingLegacyPass());
 
     if (isPassEnabled(EnableScalarIRPasses))
       addStraightLineScalarOptimizationPasses();
@@ -2148,6 +2182,9 @@ void AMDGPUCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
 
   if (TM.getOptLevel() > CodeGenOptLevel::None) {
     addFunctionPass(AMDGPUPromoteAllocaPass(TM), PMW);
+    // Run per-thread LDS buffering after promote-alloca to use leftover LDS.
+    if (TM.getTargetTriple().isAMDGCN() && EnableLDSBuffering)
+      addFunctionPass(AMDGPULDSBufferingPass(TM), PMW);
     if (isPassEnabled(EnableScalarIRPasses))
       addStraightLineScalarOptimizationPasses(PMW);
 
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 782cbfa76e6e9..034e1be4de924 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -77,6 +77,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPULowerKernelArguments.cpp
   AMDGPULowerKernelAttributes.cpp
   AMDGPULowerModuleLDSPass.cpp
+  AMDGPULDSBuffering.cpp
   AMDGPUPrepareAGPRAlloc.cpp
   AMDGPULowerExecSync.cpp
   AMDGPUSwLowerLDS.cpp
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
new file mode 100644
index 0000000000000..3daadc544c20a
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp
@@ -0,0 +1,235 @@
+//===-- AMDGPULDSUtils.cpp - AMDGPU LDS utilities ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Shared helpers for computing LDS usage and limits for an AMDGPU function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Utils/AMDGPULDSUtils.h"
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Alignment.h"
+
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+// Work-group / work-item query IR helpers
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+// Read local size Y/Z from the HSA dispatch packet.
+static std::pair<Value *, Value *>
+getLocalSizeYZFromDispatch(IRBuilderBase &Builder, Module &M,
+                           const AMDGPUSubtarget &ST) {
+  Function &F = *Builder.GetInsertBlock()->getParent();
+
+  CallInst *DispatchPtr =
+      cast<CallInst>(Builder.CreateIntrinsic(Intrinsic::amdgcn_dispatch_ptr, {}));
+  DispatchPtr->addRetAttr(Attribute::NoAlias);
+  DispatchPtr->addRetAttr(Attribute::NonNull);
+  F.removeFnAttr("amdgpu-no-dispatch-ptr");
+  DispatchPtr->addDereferenceableRetAttr(64);
+
+  Type *I32Ty = Type::getInt32Ty(M.getContext());
+  Value *GEPXY = Builder.CreateConstInBoundsGEP1_64(I32Ty, DispatchPtr, 1);
+  LoadInst *LoadXY = Builder.CreateAlignedLoad(I32Ty, GEPXY, Align(4));
+  Value *GEPZU = Builder.CreateConstInBoundsGEP1_64(I32Ty, DispatchPtr, 2);
+  LoadInst *LoadZU = Builder.CreateAlignedLoad(I32Ty, GEPZU, Align(4));
+
+  MDNode *MD = MDNode::get(M.getContext(), {});
+  LoadXY->setMetadata(LLVMContext::MD_invariant_load, MD);
+  LoadZU->setMetadata(LLVMContext::MD_invariant_load, MD);
+  ST.makeLIDRangeMetadata(LoadZU);
+
+  Value *Y = Builder.CreateLShr(LoadXY, 16);
+  return {Y, LoadZU};
+}
+
+} // end anonymous namespace
+
+Value *AMDGPU::getWorkitemID(IRBuilderBase &Builder, Module &M,
+                             const AMDGPUSubtarget &ST, unsigned N) {
+  Function *F = Builder.GetInsertBlock()->getParent();
+  Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
+  StringRef AttrName;
+
+  switch (N) {
+  case 0:
+    IntrID = Intrinsic::amdgcn_workitem_id_x;
+    AttrName = "amdgpu-no-workitem-id-x";
+    break;
+  case 1:
+    IntrID = Intrinsic::amdgcn_workitem_id_y;
+    AttrName = "amdgpu-no-workitem-id-y";
+    break;
+  case 2:
+    IntrID = Intrinsic::amdgcn_workitem_id_z;
+    AttrName = "amdgpu-no-workitem-id-z";
+    break;
+  default:
+    llvm_unreachable("invalid dimension");
+  }
+
+  Function *WorkitemIdFn = Intrinsic::getOrInsertDeclaration(&M, IntrID);
+  CallInst *CI = cast<CallInst>(Builder.CreateCall(WorkitemIdFn));
+  ST.makeLIDRangeMetadata(CI);
+  F->removeFnAttr(AttrName);
+  return CI;
+}
+
+Value *AMDGPU::buildLinearThreadId(IRBuilderBase &Builder, Module &M,
+                                  const AMDGPUSubtarget &ST) {
+  Value *TCntY = nullptr;
+  Value *TCntZ = nullptr;
+  std::tie(TCntY, TCntZ) = getLocalSizeYZFromDispatch(Builder, M, ST);
+  Value *TIdX = getWorkitemID(Builder, M, ST, 0);
+  Value *TIdY = getWorkitemID(Builder, M, ST, 1);
+  Value *TIdZ = getWorkitemID(Builder, M, ST, 2);
+
+  Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ, "", true, true);
+  Tmp0 = Builder.CreateMul(Tmp0, TIdX);
+  Value *Tmp1 = Builder.CreateMul(TIdY, TCntZ, "", true, true);
+  Value *TID = Builder.CreateAdd(Tmp0, Tmp1);
+  TID = Builder.CreateAdd(TID, TIdZ);
+  return TID;
+}
+
+//===----------------------------------------------------------------------===//
+// LDS budget computation
+//===----------------------------------------------------------------------===//
+
+AMDGPU::AMDGPULDSBudget AMDGPU::computeLDSBudget(const Function &F,
+                                                 const TargetMachine &TM) {
+  AMDGPU::AMDGPULDSBudget Result;
+
+  const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, F);
+  const Module *M = F.getParent();
+  const DataLayout &DL = M->getDataLayout();
+
+  // If the function has any arguments in the local address space, then it's
+  // possible these arguments require the entire local memory space, so
+  // we cannot use local memory.
+  FunctionType *FTy = F.getFunctionType();
+  for (Type *ParamTy : FTy->params()) {
+    PointerType *PtrTy = dyn_cast<PointerType>(ParamTy);
+    if (PtrTy && PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
+      Result.limit = 0;
+      Result.promotable = false;
+      Result.disabledDueToLocalArg = true;
+      return Result;
+    }
+  }
+
+  uint32_t LocalMemLimit = ST.getAddressableLocalMemorySize();
+  if (LocalMemLimit == 0) {
+    Result.limit = 0;
+    Result.promotable = false;
+    return Result;
+  }
+
+  SmallVector<const Constant *, 16> Stack;
+  SmallPtrSet<const Constant *, 8> VisitedConstants;
+  SmallPtrSet<const GlobalVariable *, 8> UsedLDS;
+
+  auto visitUsers = [&](const GlobalVariable *GV, const Constant *Val) -> bool {
+    for (const User *U : Val->users()) {
+      if (const Instruction *Use = dyn_cast<Instruction>(U)) {
+        if (Use->getParent()->getParent() == &F)
+          return true;
+      } else {
+        const Constant *C = cast<Constant>(U);
+        if (VisitedConstants.insert(C).second)
+          Stack.push_back(C);
+      }
+    }
+    return false;
+  };
+
+  for (const GlobalVariable &GV : M->globals()) {
+    if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
+      continue;
+
+    if (visitUsers(&GV, &GV)) {
+      UsedLDS.insert(&GV);
+      Stack.clear();
+      continue;
+    }
+
+    while (!Stack.empty()) {
+      const Constant *C = Stack.pop_back_val();
+      if (visitUsers(&GV, C)) {
+        UsedLDS.insert(&GV);
+        Stack.clear();
+        break;
+      }
+    }
+  }
+
+  SmallVector<std::pair<uint64_t, Align>, 16> AllocatedSizes;
+  AllocatedSizes.reserve(UsedLDS.size());
+
+  for (const GlobalVariable *GV : UsedLDS) {
+    Align Alignment =
+        DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());
+    uint64_t AllocSize = DL.getTypeAllocSize(GV->getValueType());
+
+    // HIP uses an extern unsized array in local address space for dynamically
+    // allocated shared memory.
+    if (GV->hasExternalLinkage() && AllocSize == 0) {
+      Result.limit = 0;
+      Result.promotable = false;
+      Result.disabledDueToExternDynShared = true;
+      return Result;
+    }
+
+    AllocatedSizes.emplace_back(AllocSize, Alignment);
+  }
+
+  // Sort to try to estimate the worst case alignment padding.
+  llvm::sort(AllocatedSizes, llvm::less_second());
+
+  uint32_t CurrentLocalMemUsage = 0;
+  for (auto Alloc : AllocatedSizes) {
+    CurrentLocalMemUsage = alignTo(CurrentLocalMemUsage, Alloc.second);
+    CurrentLocalMemUsage += Alloc.first;
+  }
+
+  unsigned MaxOccupancy =
+      ST.getWavesPerEU(ST.getFlatWorkGroupSizes(F), CurrentLocalMemUsage, F)
+          .second;
+
+  unsigned MaxSizeWithWaveCount =
+      ST.getMaxLocalMemSizeWithWaveCount(MaxOccupancy, F);
+
+  if (CurrentLocalMemUsage > MaxSizeWithWaveCount) {
+    Result.currentUsage = CurrentLocalMemUsage;
+    Result.limit = MaxSizeWithWaveCount;
+    Result.maxOccupancy = MaxOccupancy;
+    Result.promotable = false;
+    return Result;
+  }
+
+  Result.currentUsage = CurrentLocalMemUsage;
+  Result.limit = MaxSizeWithWaveCount;
+  Result.maxOccupancy = MaxOccupancy;
+  Result.promotable = true;
+  return Result;
+}
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
new file mode 100644
index 0000000000000..18690a77647c1
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h
@@ -0,0 +1,53 @@
+//===-- AMDGPULDSUtils.h - AMDGPU LDS utilities ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Shared helpers for computing LDS usage and limits for an AMDGPU function.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPULDSUTILS_H
+#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPULDSUTILS_H
+
+#include <cstdint>
+#include <utility>
+
+namespace llvm {
+
+class AMDGPUSubtarget;
+class Function;
+class IRBuilderBase;
+class Module;
+class TargetMachine;
+class Value;
+
+namespace AMDGPU {
+
+/// Get workitem id for dimension N (0,1,2).
+Value *getWorkitemID(IRBuilderBase &Builder, Module &M,
+                     const AMDGPUSubtarget &ST, unsigned N);
+
+/// Compute linear thread id within a workgroup.
+Value *buildLinearThreadId(IRBuilderBase &Builder, Module &M,
+                           const AMDGPUSubtarget &ST);
+
+struct AMDGPULDSBudget {
+  uint32_t currentUsage = 0;
+  uint32_t limit = 0;
+  unsigned maxOccupancy = 0;
+  bool promotable = false;
+  bool disabledDueToLocalArg = false;
+  bool disabledDueToExternDynShared = false;
+};
+
+AMDGPULDSBudget computeLDSBudget(const Function &F, const TargetMachine &TM);
+
+} // end namespace AMDGPU
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPULDSUTILS_H
diff --git a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
index 7b2200d8bc488..4633763d23f22 100644
--- a/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/Utils/CMakeLists.txt
@@ -4,6 +4,7 @@ add_llvm_component_library(LLVMAMDGPUUtils
   AMDGPUDelayedMCExpr.cpp
   AMDGPUPALMetadata.cpp
   AMDKernelCodeTUtils.cpp
+  AMDGPULDSUtils.cpp
 
   LINK_COMPONENTS
   Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/lds-buffering.ll b/llvm/test/CodeGen/AMDGPU/lds-buffering.ll
new file mode 100644
index 0000000000000..64165c3cb1e40
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-buffering.ll
@@ -0,0 +1,183 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes='amdgpu-lds-buffering<max-bytes=64>' -S %s | FileCheck %s
+
+%state = type { <4 x i32>, <4 x i32> }
+
+;===---------------------------------------------------------------------===//
+; Positive cases
+;===---------------------------------------------------------------------===//
+
+; CHECK: @ldsbuf_test.ldsbuf = internal unnamed_addr addrspace(3) global
+; CHECK: @ldsbuf_complex.ldsbuf = internal unnamed_addr addrspace(3) global
+; CHECK: @ldsbuf_rocrand_preserved_subsequence.ldsbuf = internal unnamed_addr addrspace(3) global
+
+; Basic positive case: a single global load whose only use is a store back to
+; the same pointer. Include an intervening (potentially-aliasing) global store
+; to keep the value live and avoid trivially folding away.
+; CHECK-LABEL: @ldsbuf_test(
+; CHECK: %[[SLOT_BASIC:[^ ]+]] = getelementptr inbounds {{.*}}, ptr addrspace(3) @ldsbuf_test.ldsbuf, i32 0, i32 %
+; CHECK: call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3){{.*}}%[[SLOT_BASIC]], ptr addrspace(1){{.*}}%p, i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1){{.*}}%p, ptr addrspace(3){{.*}}%[[SLOT_BASIC]], i64 16, i1 false)
+define amdgpu_kernel void @ldsbuf_test(ptr addrspace(1) %p, ptr addrspace(1) %q) #0 {
+entry:
+  %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+  store i32 0, ptr addrspace(1) %q, align 4
+  store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+  ret void
+}
+
+; "Complexity" positive case: keep the loaded value live across control flow
+; and additional global memory operations, then store back to the same pointer.
+; CHECK-LABEL: @ldsbuf_complex(
+; CHECK: %[[SLOT_COMPLEX:[^ ]+]] = getelementptr inbounds {{.*}}, ptr addrspace(3) @ldsbuf_complex.ldsbuf, i32 0, i32 %
+; CHECK: call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3){{.*}}%[[SLOT_COMPLEX]], ptr addrspace(1){{.*}}%p, i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1){{.*}}%p, ptr addrspace(3){{.*}}%[[SLOT_COMPLEX]], i64 16, i1 false)
+define amdgpu_kernel void @ldsbuf_complex(ptr addrspace(1) %p, ptr addrspace(1) %q, i1 %c) #0 {
+entry:
+  %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+  br i1 %c, label %then, label %else
+
+then:
+  store i32 1, ptr addrspace(1) %q, align 4
+  br label %merge
+
+else:
+  store i32 2, ptr addrspace(1) %q, align 4
+  br label %merge
+
+merge:
+  store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+  ret void
+}
+
+; rocRand-inspired positive case: model a per-thread RNG state where part of
+; the state (a preserved 16B "subsequence" field) is loaded and stored back
+; unchanged, but kept live across a non-trivial loop body.
+; CHECK-LABEL: @ldsbuf_rocrand_preserved_subsequence(
+; CHECK: %[[TAILPTR:[^ ]+]] = getelementptr inbounds %state, ptr addrspace(1) %state_ptr, i32 0, i32 1
+; CHECK: %[[SLOT_TAIL:[^ ]+]] = getelementptr inbounds {{.*}}, ptr addrspace(3) @ldsbuf_rocrand_preserved_subsequence.ldsbuf, i32 0, i32 %
+; CHECK: call void @llvm.memcpy.p3.p1.i64(ptr addrspace(3){{.*}}%[[SLOT_TAIL]], ptr addrspace(1){{.*}}%[[TAILPTR]], i64 16, i1 false)
+; CHECK: call void @llvm.memcpy.p1.p3.i64(ptr addrspace(1){{.*}}%[[TAILPTR]], ptr addrspace(3){{.*}}%[[SLOT_TAIL]], i64 16, i1 false)
+define amdgpu_kernel void @ldsbuf_rocrand_preserved_subsequence(ptr addrspace(1) %state_ptr,
+                                                                ptr addrspace(1) %out) #0 {
+entry:
+  %tailptr = getelementptr inbounds %state, ptr addrspace(1) %state_ptr, i32 0, i32 1
+  %tail = load <4 x i32>, ptr addrspace(1) %tailptr, align 16
+
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %acc = phi i32 [ 1, %entry ], [ %acc.next, %loop ]
+
+  %shl = shl i32 %acc, 6
+  %xor = xor i32 %shl, %acc
+  %shr = lshr i32 %xor, 13
+  %acc.next = xor i32 %shr, 1234567
+
+  %idx = zext i32 %i to i64
+  %outp = getelementptr inbounds float, ptr addrspace(1) %out, i64 %idx
+  %f = uitofp i32 %acc.next to float
+  store float %f, ptr addrspace(1) %outp, align 4
+
+  %i.next = add nuw i32 %i, 1
+  %done = icmp eq i32 %i.next, 8
+  br i1 %done, label %exit, label %loop
+
+exit:
+  store <4 x i32> %tail, ptr addrspace(1) %tailptr, align 16
+  ret void
+}
+
+;===---------------------------------------------------------------------===//
+; Negative coverage: patterns that must NOT be transformed.
+;===---------------------------------------------------------------------===//
+
+; Negative: atomic operations are excluded (must not transform).
+; CHECK-LABEL: @ldsbuf_atomic(
+; CHECK: load atomic <4 x i32>, ptr addrspace(1) %p unordered, align 16
+; CHECK: store atomic <4 x i32> {{.*}}, ptr addrspace(1) %p unordered, align 16
+define amdgpu_kernel void @ldsbuf_atomic(ptr addrspace(1) %p) #0 {
+entry:
+  %ld = load atomic <4 x i32>, ptr addrspace(1) %p unordered, align 16
+  store atomic <4 x i32> %ld, ptr addrspace(1) %p unordered, align 16
+  ret void
+}
+
+; Negative: volatile operations are excluded (must not transform).
+; CHECK-LABEL: @ldsbuf_volatile(
+; CHECK: load volatile <4 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: store volatile <4 x i32> {{.*}}, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_volatile(ptr addrspace(1) %p) #0 {
+entry:
+  %ld = load volatile <4 x i32>, ptr addrspace(1) %p, align 16
+  store volatile <4 x i32> %ld, ptr addrspace(1) %p, align 16
+  ret void
+}
+
+; Negative: alignment requirement (min 16) must be met.
+; CHECK-LABEL: @ldsbuf_misaligned(
+; CHECK: load <4 x i32>, ptr addrspace(1) %p, align 8
+; CHECK: store <4 x i32> {{.*}}, ptr addrspace(1) %p, align 8
+define amdgpu_kernel void @ldsbuf_misaligned(ptr addrspace(1) %p) #0 {
+entry:
+  %ld = load <4 x i32>, ptr addrspace(1) %p, align 8
+  store <4 x i32> %ld, ptr addrspace(1) %p, align 8
+  ret void
+}
+
+; Negative: size limit (max-bytes=64) must be respected.
+; CHECK-LABEL: @ldsbuf_too_large(
+; CHECK: load <20 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: store <20 x i32> {{.*}}, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_too_large(ptr addrspace(1) %p) #0 {
+entry:
+  %ld = load <20 x i32>, ptr addrspace(1) %p, align 16
+  store <20 x i32> %ld, ptr addrspace(1) %p, align 16
+  ret void
+}
+
+; Negative: non-global pointers are excluded (only addrspace(1) is supported).
+; CHECK-LABEL: @ldsbuf_non_global_ptr(
+; CHECK: load <4 x i32>, ptr %p, align 16
+; CHECK: store <4 x i32> {{.*}}, ptr %p, align 16
+define amdgpu_kernel void @ldsbuf_non_global_ptr(ptr %p) #0 {
+entry:
+  %ld = load <4 x i32>, ptr %p, align 16
+  store <4 x i32> %ld, ptr %p, align 16
+  ret void
+}
+
+; Negative: the load must have exactly one use (the final store).
+; CHECK-LABEL: @ldsbuf_multiple_uses(
+; CHECK: %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: extractelement <4 x i32> %ld, i32 0
+; CHECK: store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_multiple_uses(ptr addrspace(1) %p, ptr addrspace(1) %q) #0 {
+entry:
+  %ld = load <4 x i32>, ptr addrspace(1) %p, align 16
+  %e = extractelement <4 x i32> %ld, i32 0
+  store i32 %e, ptr addrspace(1) %q, align 4
+  store <4 x i32> %ld, ptr addrspace(1) %p, align 16
+  ret void
+}
+
+; Negative: budget rejection. With large pre-existing LDS usage and large
+; workgroup size, per-thread slots should be rejected by the LDS budget check.
+ at ldsbuf_budget_reject.big = internal addrspace(3) global [8192 x i32] zeroinitializer, align 16
+
+; CHECK-LABEL: @ldsbuf_budget_reject(
+; CHECK: load <16 x i32>, ptr addrspace(1) %p, align 16
+; CHECK: store <16 x i32> {{.*}}, ptr addrspace(1) %p, align 16
+define amdgpu_kernel void @ldsbuf_budget_reject(ptr addrspace(1) %p) #1 {
+entry:
+  %x = load i32, ptr addrspace(3) getelementptr inbounds ([8192 x i32], ptr addrspace(3) @ldsbuf_budget_reject.big, i32 0, i32 0), align 16
+  call void @llvm.donothing()
+  %ld = load <16 x i32>, ptr addrspace(1) %p, align 16
+  store <16 x i32> %ld, ptr addrspace(1) %p, align 16
+  ret void
+}
+
+declare void @llvm.donothing()
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "uniform-work-group-size"="true" }
+attributes #1 = { "amdgpu-flat-work-group-size"="1024,1024" "uniform-work-group-size"="true" }



More information about the llvm-commits mailing list