[llvm-branch-commits] [llvm] [AMDGPU] Use shared memory metadata propagation (PR #243680)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 18 01:13:36 PST 2026
https://github.com/YonahGoldberg updated https://github.com/llvm/llvm-project/pull/243680
>From f8bc5876ea439f4d27973635746baef30f1cd927 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:10:19 +0000
Subject: [PATCH 1/2] [AMDGPU] Add memory metadata tests
---
...wer-buffer-fat-pointers-memory-metadata.ll | 55 +++++++++++++++++++
1 file changed, 55 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
new file mode 100644
index 0000000000000..c4ee77dda50ed
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=amdgpu-lower-buffer-fat-pointers < %s | FileCheck %s
+
+target triple = "amdgpu9.00--"
+
+; Splitting an aggregate containing a fat pointer preserves access metadata.
+define {ptr addrspace(7), i32} @load_fat_pointer_aggregate(ptr %p) {
+; CHECK-LABEL: define { { ptr addrspace(8), i32 }, i32 } @load_fat_pointer_aggregate(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[V_0:%.*]] = load i160, ptr [[P]], align 32, !nontemporal [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[V_0]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
+; CHECK-NEXT: [[V_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
+; CHECK-NEXT: [[V_0_PTR_OFF:%.*]] = trunc i160 [[V_0]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = insertvalue { ptr addrspace(8), i32 } poison, ptr addrspace(8) [[V_0_PTR_RSRC]], 0
+; CHECK-NEXT: [[V_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[V_0_PTR_OFF]], 1
+; CHECK-NEXT: [[V_0_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } poison, { ptr addrspace(8), i32 } [[V_0_PTR]], 0
+; CHECK-NEXT: [[P_OFF_32:%.*]] = getelementptr nuw i8, ptr [[P]], i64 32
+; CHECK-NEXT: [[V_1:%.*]] = load i32, ptr [[P_OFF_32]], align 32, !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT: [[V_1_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } [[V_0_AGG]], i32 [[V_1]], 1
+; CHECK-NEXT: ret { { ptr addrspace(8), i32 }, i32 } [[V_1_AGG]]
+;
+ %v = load {ptr addrspace(7), i32}, ptr %p, align 32, !nontemporal !0, !mem.cache_hint !1, !annotation !3, !pcsections !4
+ ret {ptr addrspace(7), i32} %v
+}
+
+; Legalizing a wide buffer load preserves metadata on each resulting access.
+define <8 x i32> @load_wide_buffer(ptr addrspace(7) %p) {
+; CHECK-LABEL: define <8 x i32> @load_wide_buffer(
+; CHECK-SAME: { ptr addrspace(8), i32 } [[P:%.*]]) {
+; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0
+; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1
+; CHECK-NEXT: [[V_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT: [[V_EXT_0:%.*]] = shufflevector <4 x i32> [[V_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[V_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[V_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[P_OFF_PTR_16:%.*]] = add i32 [[P_OFF]], 16
+; CHECK-NEXT: [[V_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF_PTR_16]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT: [[V_EXT_4:%.*]] = shufflevector <4 x i32> [[V_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[V:%.*]] = shufflevector <8 x i32> [[V_PARTS_0]], <8 x i32> [[V_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: ret <8 x i32> [[V]]
+;
+ %v = load <8 x i32>, ptr addrspace(7) %p, align 16, !nontemporal !0, !mem.cache_hint !1, !annotation !3, !pcsections !4
+ ret <8 x i32> %v
+}
+
+!0 = !{i32 1}
+!1 = !{i32 0, !2}
+!2 = !{!"test.cache", i32 1}
+!3 = !{!"memory access"}
+!4 = !{!"metadata_section"}
+;.
+; CHECK: [[META0]] = !{i32 1}
+; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"test.cache", i32 1}
+;.
>From f373158016fef5b98f1855b60ed9cdc85ef1c9da Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 01:10:25 +0000
Subject: [PATCH 2/2] [AMDGPU] Use shared memory metadata propagation
---
.../Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp | 5 +++--
.../lower-buffer-fat-pointers-memory-metadata.ll | 14 ++++++++------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 89f024ca8a5dc..1e1a01be30e94 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -225,6 +225,7 @@
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -586,7 +587,7 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitLoadInst(LoadInst &LI) {
LoadInst *NewLI = IRB.CreateAlignedLoad(
IntLeafTy, Ptr, commonAlignment(LI.getAlign(), Off), Name);
NewLI->setVolatile(LI.isVolatile());
- copyMetadataForLoad(*NewLI, LI);
+ copyMetadataForMemoryAccess(*NewLI, LI);
NewLI->setAAMetadata(AATags.adjustForAccess(Off, IntLeafTy, DL));
Value *V = NewLI;
if (LeafTy != IntLeafTy)
@@ -1192,7 +1193,7 @@ bool LegalizeBufferContentTypesVisitor::visitLoadImpl(
LoadInst *NewLI = IRB.CreateAlignedLoad(
LoadableType, NewPtr, commonAlignment(OrigLI.getAlign(), ByteOffset),
Name + ".off." + Twine(ByteOffset));
- copyMetadataForLoad(*NewLI, OrigLI);
+ copyMetadataForMemoryAccess(*NewLI, OrigLI);
NewLI->setAAMetadata(
AANodes.adjustForAccess(ByteOffset, LoadableType, DL));
NewLI->setAtomic(OrigLI.getOrdering(), OrigLI.getSyncScopeID());
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
index c4ee77dda50ed..0464d85b961d5 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-memory-metadata.ll
@@ -7,7 +7,7 @@ target triple = "amdgpu9.00--"
define {ptr addrspace(7), i32} @load_fat_pointer_aggregate(ptr %p) {
; CHECK-LABEL: define { { ptr addrspace(8), i32 }, i32 } @load_fat_pointer_aggregate(
; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT: [[V_0:%.*]] = load i160, ptr [[P]], align 32, !nontemporal [[META0:![0-9]+]], !mem.cache_hint [[META1:![0-9]+]]
+; CHECK-NEXT: [[V_0:%.*]] = load i160, ptr [[P]], align 32, !nontemporal [[META0:![0-9]+]], !annotation [[META1:![0-9]+]], !pcsections [[META2:![0-9]+]], !mem.cache_hint [[META3:![0-9]+]]
; CHECK-NEXT: [[TMP1:%.*]] = lshr i160 [[V_0]], 32
; CHECK-NEXT: [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128
; CHECK-NEXT: [[V_0_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8)
@@ -16,7 +16,7 @@ define {ptr addrspace(7), i32} @load_fat_pointer_aggregate(ptr %p) {
; CHECK-NEXT: [[V_0_PTR:%.*]] = insertvalue { ptr addrspace(8), i32 } [[TMP3]], i32 [[V_0_PTR_OFF]], 1
; CHECK-NEXT: [[V_0_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } poison, { ptr addrspace(8), i32 } [[V_0_PTR]], 0
; CHECK-NEXT: [[P_OFF_32:%.*]] = getelementptr nuw i8, ptr [[P]], i64 32
-; CHECK-NEXT: [[V_1:%.*]] = load i32, ptr [[P_OFF_32]], align 32, !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT: [[V_1:%.*]] = load i32, ptr [[P_OFF_32]], align 32, !nontemporal [[META0]], !annotation [[META1]], !pcsections [[META2]], !mem.cache_hint [[META3]]
; CHECK-NEXT: [[V_1_AGG:%.*]] = insertvalue { { ptr addrspace(8), i32 }, i32 } [[V_0_AGG]], i32 [[V_1]], 1
; CHECK-NEXT: ret { { ptr addrspace(8), i32 }, i32 } [[V_1_AGG]]
;
@@ -30,11 +30,11 @@ define <8 x i32> @load_wide_buffer(ptr addrspace(7) %p) {
; CHECK-SAME: { ptr addrspace(8), i32 } [[P:%.*]]) {
; CHECK-NEXT: [[P_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 0
; CHECK-NEXT: [[P_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[P]], 1
-; CHECK-NEXT: [[V_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT: [[V_OFF_0:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF]], i32 0, i32 0), !nontemporal [[META0]], !annotation [[META1]], !pcsections [[META2]], !mem.cache_hint [[META3]]
; CHECK-NEXT: [[V_EXT_0:%.*]] = shufflevector <4 x i32> [[V_OFF_0]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[V_PARTS_0:%.*]] = shufflevector <8 x i32> poison, <8 x i32> [[V_EXT_0]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[P_OFF_PTR_16:%.*]] = add i32 [[P_OFF]], 16
-; CHECK-NEXT: [[V_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF_PTR_16]], i32 0, i32 0), !nontemporal [[META0]], !mem.cache_hint [[META1]]
+; CHECK-NEXT: [[V_OFF_16:%.*]] = call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) align 16 [[P_RSRC]], i32 [[P_OFF_PTR_16]], i32 0, i32 0), !nontemporal [[META0]], !annotation [[META1]], !pcsections [[META2]], !mem.cache_hint [[META3]]
; CHECK-NEXT: [[V_EXT_4:%.*]] = shufflevector <4 x i32> [[V_OFF_16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[V:%.*]] = shufflevector <8 x i32> [[V_PARTS_0]], <8 x i32> [[V_EXT_4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: ret <8 x i32> [[V]]
@@ -50,6 +50,8 @@ define <8 x i32> @load_wide_buffer(ptr addrspace(7) %p) {
!4 = !{!"metadata_section"}
;.
; CHECK: [[META0]] = !{i32 1}
-; CHECK: [[META1]] = !{i32 0, [[META2:![0-9]+]]}
-; CHECK: [[META2]] = !{!"test.cache", i32 1}
+; CHECK: [[META1]] = !{!"memory access"}
+; CHECK: [[META2]] = !{!"metadata_section"}
+; CHECK: [[META3]] = !{i32 0, [[META4:![0-9]+]]}
+; CHECK: [[META4]] = !{!"test.cache", i32 1}
;.
More information about the llvm-branch-commits
mailing list