[llvm-branch-commits] [llvm] [AMDGPU] Use shared memory metadata propagation (PR #243680)

Yonah Goldberg via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 18 01:13:39 PST 2026


https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/243680

>From f8bc5876ea439f4d27973635746baef30f1cd927 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:10:19 +0000
Subject: [PATCH 1/2] [AMDGPU] Add memory metadata tests

---
 ...wer-buffer-fat-pointers-memory-metadata.ll | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll

diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
new file mode 100644
index 0000000000000..c4ee77dda50ed
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s
+
+target triple = "amdgpu9.00--"
+
+; Splitting an aggregate containing a fat pointer preserves access metadata.
+define {ptr addrspace(7), i32} @load_fat_pointer_aggregate(ptr %p) {
+; CHECK-LABEL: define { { ptr addrspace(8), i32 }, i32 } @load_fat_pointer_aggregate(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V_0:%.*]] = load i160, ptr [[P]], align 32, !nontemporal [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i160 [[V_0]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT:    [[V_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT:    [[V_0_PTR_OFF:%.*]] = trunc i160 [[V_0]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[V_0_PTR_RSRC]], 0
+; CHECK-NEXT:    [[V_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[V_0_PTR_OFF]], 1
+; CHECK-NEXT:    [[V_0_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } poison, { ptr addrspace(8), i32 } [[V_0_PTR]], 0
+; CHECK-NEXT:    [[P_OFF_32:%.*]] = getelementptr nuw i8, ptr [[P]], i64 32
+; CHECK-NEXT:    [[V_1:%.*]] = load i32, ptr [[P_OFF_32]], align 32, !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT:    [[V_1_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } [[V_0_AGG]], i32 [[V_1]], 1
+; CHECK-NEXT:    ret { { ptr addrspace(8), i32 }, i32 } [[V_1_AGG]]
+;
+  %v = load {ptr addrspace(7), i32}, ptr %p, align 32, !nontemporal !0, !mem.cache_hint !1, !annotation !3, !pcsections !4
+  ret {ptr addrspace(7), i32} %v
+}
+
+; Legalizing a wide buffer load preserves metadata on each resulting access.
+define <8 x i32> @load_wide_buffer(ptr addrspace(7) %p) {
+; CHECK-LABEL: define <8 x i32> @load_wide_buffer(
+; CHECK-SAME: { ptr addrspace(8), i32 } [[P:%.*]]) {
+; CHECK-NEXT:    [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0
+; CHECK-NEXT:    [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1
+; CHECK-NEXT:    [[V_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT:    [[V_EXT_0:%.*]] = shufflevector <4 x i32> [[V_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[V_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[V_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[P_OFF_PTR_16:%.*]] = add i32 [[P_OFF]], 16
+; CHECK-NEXT:    [[V_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF_PTR_16]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT:    [[V_EXT_4:%.*]] = shufflevector <4 x i32> [[V_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[V:%.*]] = shufflevector <8 x i32> [[V_PARTS_0]], <8 x i32> [[V_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    ret <8 x i32> [[V]]
+;
+  %v = load <8 x i32>, ptr addrspace(7) %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !3, !pcsections !4
+  ret <8 x i32> %v
+}
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"test.cache", i32 1}
+!3 = !{!"memory access"}
+!4 = !{!"metadata_section"}
+;.
+; CHECK: [[META0]] = !{i32 1}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"test.cache", i32 1}
+;.

>From f373158016fef5b98f1855b60ed9cdc85ef1c9da Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:10:25 +0000
Subject: [PATCH 2/2] [AMDGPU] Use shared memory metadata propagation

---
 .../Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp |  5 +++--
 .../lower-buffer-fat-pointers-memory-metadata.ll   | 14 ++++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 89f024ca8a5dc..1e1a01be30e94 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -225,6 +225,7 @@
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/MetadataUtils.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -586,7 +587,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitLoadInst(LoadInst &LI) {
           LoadInst *NewLI = IRB.CreateAlignedLoad(
               IntLeafTy, Ptr, commonAlignment(LI.getAlign(), Off), Name);
           NewLI->setVolatile(LI.isVolatile());
-          copyMetadataForLoad(*NewLI, LI);
+          copyMetadataForMemoryAccess(*NewLI, LI);
           NewLI->setAAMetadata(AATags.adjustForAccess(Off, IntLeafTy, DL));
           Value *V = NewLI;
           if (LeafTy != IntLeafTy)
@@ -1192,7 +1193,7 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
       LoadInst *NewLI = IRB.CreateAlignedLoad(
           LoadableType, NewPtr, commonAlignment(OrigLI.getAlign(), ByteOffset),
           Name + ".off." + Twine(ByteOffset));
-      copyMetadataForLoad(*NewLI, OrigLI);
+      copyMetadataForMemoryAccess(*NewLI, OrigLI);
       NewLI->setAAMetadata(
           AANodes.adjustForAccess(ByteOffset, LoadableType, DL));
       NewLI->setAtomic(OrigLI.getOrdering(), OrigLI.getSyncScopeID());
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
index c4ee77dda50ed..0464d85b961d5 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
@@ -7,7 +7,7 @@ target triple = "amdgpu9.00--"
 define {ptr addrspace(7), i32} @load_fat_pointer_aggregate(ptr %p) {
 ; CHECK-LABEL: define { { ptr addrspace(8), i32 }, i32 } @load_fat_pointer_aggregate(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:    [[V_0:%.*]] = load i160, ptr [[P]], align 32, !nontemporal [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT:    [[V_0:%.*]] = load i160, ptr [[P]], align 32, !nontemporal [[META0:![0-9]+]], !annotation [[META1:![0-9]+]], !pcsections [[META2:![0-9]+]], !mem.cache_hint [[META3:![0-9]+]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i160 [[V_0]], 32
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
 ; CHECK-NEXT:    [[V_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
@@ -16,7 +16,7 @@ define {ptr addrspace(7), i32} @load_fat_pointer_aggregate(ptr %p) {
 ; CHECK-NEXT:    [[V_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[V_0_PTR_OFF]], 1
 ; CHECK-NEXT:    [[V_0_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } poison, { ptr addrspace(8), i32 } [[V_0_PTR]], 0
 ; CHECK-NEXT:    [[P_OFF_32:%.*]] = getelementptr nuw i8, ptr [[P]], i64 32
-; CHECK-NEXT:    [[V_1:%.*]] = load i32, ptr [[P_OFF_32]], align 32, !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT:    [[V_1:%.*]] = load i32, ptr [[P_OFF_32]], align 32, !nontemporal [[META0]], !annotation [[META1]], !pcsections [[META2]], !mem.cache_hint [[META3]]
 ; CHECK-NEXT:    [[V_1_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } [[V_0_AGG]], i32 [[V_1]], 1
 ; CHECK-NEXT:    ret { { ptr addrspace(8), i32 }, i32 } [[V_1_AGG]]
 ;
@@ -30,11 +30,11 @@ define <8 x i32> @load_wide_buffer(ptr addrspace(7) %p) {
 ; CHECK-SAME: { ptr addrspace(8), i32 } [[P:%.*]]) {
 ; CHECK-NEXT:    [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0
 ; CHECK-NEXT:    [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1
-; CHECK-NEXT:    [[V_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT:    [[V_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0), !nontemporal [[META0]], !annotation [[META1]], !pcsections [[META2]], !mem.cache_hint [[META3]]
 ; CHECK-NEXT:    [[V_EXT_0:%.*]] = shufflevector <4 x i32> [[V_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[V_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[V_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[P_OFF_PTR_16:%.*]] = add i32 [[P_OFF]], 16
-; CHECK-NEXT:    [[V_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF_PTR_16]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT:    [[V_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF_PTR_16]], i32 0, i32 0), !nontemporal [[META0]], !annotation [[META1]], !pcsections [[META2]], !mem.cache_hint [[META3]]
 ; CHECK-NEXT:    [[V_EXT_4:%.*]] = shufflevector <4 x i32> [[V_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[V:%.*]] = shufflevector <8 x i32> [[V_PARTS_0]], <8 x i32> [[V_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <8 x i32> [[V]]
@@ -50,6 +50,8 @@ define <8 x i32> @load_wide_buffer(ptr addrspace(7) %p) {
 !4 = !{!"metadata_section"}
 ;.
 ; CHECK: [[META0]] = !{i32 1}
-; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
-; CHECK: [[META2]] = !{!"test.cache", i32 1}
+; CHECK: [[META1]] = !{!"memory access"}
+; CHECK: [[META2]] = !{!"metadata_section"}
+; CHECK: [[META3]] = !{i32 0, [[META4:![0-9]+]]}
+; CHECK: [[META4]] = !{!"test.cache", i32 1}
 ;.



More information about the llvm-branch-commits mailing list