[llvm] [AMDGPU] Lower mem intrinsics on LDS pointers in AMDGPUSwLowerLDS (PR #202552)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 02:12:13 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
memcpy/memset/memmove on LDS pointers were left in the local address space, silently writing into the malloc'ed pointer LDS cell instead of the global buffer backing the lowered LDS
Collect and translate them to the global pointer like the other LDS memory operations under asan
---
Full diff: https://github.com/llvm/llvm-project/pull/202552.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp (+46)
- (added) llvm/test/CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-memintrinsic-asan.ll (+107)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
index ccf6e328b4cbf..f349323b8b86f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSwLowerLDS.cpp
@@ -656,6 +656,12 @@ void AMDGPUSwLowerLDS::getLDSMemoryInstructions(
if (ASC->getSrcAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
ASC->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS)
LDSInstructions.insert(&Inst);
+ } else if (AnyMemIntrinsic *MI = dyn_cast<AnyMemIntrinsic>(&Inst)) {
+ if (MI->getDestAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
+ LDSInstructions.insert(&Inst);
+ else if (auto *MTI = dyn_cast<AnyMemTransferInst>(MI))
+ if (MTI->getSourceAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
+ LDSInstructions.insert(&Inst);
} else
continue;
}
@@ -729,6 +735,46 @@ void AMDGPUSwLowerLDS::translateLDSMemoryOperationsToGlobalMemory(
AsanInfo.Instructions.insert(NewXCHG);
XCHG->replaceAllUsesWith(NewXCHG);
XCHG->eraseFromParent();
+ } else if (AnyMemIntrinsic *MI = dyn_cast<AnyMemIntrinsic>(Inst)) {
+ Value *NewDest = MI->getRawDest();
+ if (MI->getDestAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
+ NewDest = getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, NewDest);
+ CallInst *NewMI = nullptr;
+ if (AnyMemSetInst *MSI = dyn_cast<AnyMemSetInst>(MI)) {
+ if (MI->isAtomic())
+ NewMI = IRB.CreateElementUnorderedAtomicMemSet(
+ NewDest, MSI->getValue(), MSI->getLength(),
+ MSI->getDestAlign().valueOrOne(), MSI->getElementSizeInBytes());
+ else
+ NewMI = IRB.CreateMemSet(NewDest, MSI->getValue(), MSI->getLength(),
+ MSI->getDestAlign(),
+ cast<MemSetInst>(MI)->isVolatile());
+ } else if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
+ Value *NewSrc = MTI->getRawSource();
+ if (MTI->getSourceAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
+ NewSrc = getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, NewSrc);
+ if (MI->isAtomic()) {
+ if (MI->getIntrinsicID() ==
+ Intrinsic::memmove_element_unordered_atomic)
+ NewMI = IRB.CreateElementUnorderedAtomicMemMove(
+ NewDest, MTI->getDestAlign().valueOrOne(), NewSrc,
+ MTI->getSourceAlign().valueOrOne(), MTI->getLength(),
+ MTI->getElementSizeInBytes());
+ else
+ NewMI = IRB.CreateElementUnorderedAtomicMemCpy(
+ NewDest, MTI->getDestAlign().valueOrOne(), NewSrc,
+ MTI->getSourceAlign().valueOrOne(), MTI->getLength(),
+ MTI->getElementSizeInBytes());
+ } else
+ NewMI = IRB.CreateMemTransferInst(
+ MI->getIntrinsicID(), NewDest, MTI->getDestAlign(), NewSrc,
+ MTI->getSourceAlign(), MTI->getLength(),
+ cast<MemTransferInst>(MI)->isVolatile());
+ } else
+ report_fatal_error("Unimplemented LDS lowering memory intrinsic");
+ AsanInfo.Instructions.insert(NewMI);
+ MI->replaceAllUsesWith(NewMI);
+ MI->eraseFromParent();
} else if (AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(Inst)) {
Value *AIOperand = ASC->getPointerOperand();
Value *Replacement =
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-memintrinsic-asan.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-memintrinsic-asan.ll
new file mode 100644
index 0000000000000..652d46a842803
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-memintrinsic-asan.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=amdgpu-sw-lower-lds -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; Test to check that memset/memcpy/memmove on LDS pointers are lowered to the
+; global memory buffer that backs the lowered LDS, rather than left targeting
+; the LDS address space.
+
+ at lds_1 = internal addrspace(3) global [16 x i8] poison, align 4
+ at lds_2 = internal addrspace(3) global [16 x i8] poison, align 4
+
+define amdgpu_kernel void @memintrinsic_kernel(ptr addrspace(1) %src) sanitize_address {
+; CHECK-LABEL: define amdgpu_kernel void @memintrinsic_kernel(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[WID:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
+; CHECK-NEXT: [[TMP3:%.*]] = or i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[MALLOC:.*]], label %[[BB20:.*]]
+; CHECK: [[MALLOC]]:
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(1) getelementptr inbounds ([[LLVM_AMDGCN_SW_LDS_MEMINTRINSIC_KERNEL_MD_TYPE:%.*]], ptr addrspace(1) @llvm.amdgcn.sw.lds.memintrinsic_kernel.md, i32 0, i32 2, i32 0), align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(1) getelementptr inbounds ([[LLVM_AMDGCN_SW_LDS_MEMINTRINSIC_KERNEL_MD_TYPE]], ptr addrspace(1) @llvm.amdgcn.sw.lds.memintrinsic_kernel.md, i32 0, i32 2, i32 2), align 4
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = call ptr @llvm.returnaddress.p0(i32 0)
+; CHECK-NEXT: [[TMP11:%.*]] = ptrtoint ptr [[TMP10]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = call i64 @__asan_malloc_impl(i64 [[TMP9]], i64 [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = inttoptr i64 [[TMP12]] to ptr addrspace(1)
+; CHECK-NEXT: store ptr addrspace(1) [[TMP13]], ptr addrspace(3) @llvm.amdgcn.sw.lds.memintrinsic_kernel, align 8
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP13]], i64 8
+; CHECK-NEXT: [[TMP15:%.*]] = ptrtoint ptr addrspace(1) [[TMP14]] to i64
+; CHECK-NEXT: call void @__asan_poison_region(i64 [[TMP15]], i64 24)
+; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP13]], i64 48
+; CHECK-NEXT: [[TMP17:%.*]] = ptrtoint ptr addrspace(1) [[TMP16]] to i64
+; CHECK-NEXT: call void @__asan_poison_region(i64 [[TMP17]], i64 16)
+; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP13]], i64 80
+; CHECK-NEXT: [[TMP19:%.*]] = ptrtoint ptr addrspace(1) [[TMP18]] to i64
+; CHECK-NEXT: call void @__asan_poison_region(i64 [[TMP19]], i64 16)
+; CHECK-NEXT: br label %[[BB20]]
+; CHECK: [[BB20]]:
+; CHECK-NEXT: [[XYZCOND:%.*]] = phi i1 [ false, %[[WID]] ], [ true, %[[MALLOC]] ]
+; CHECK-NEXT: call void @llvm.amdgcn.s.barrier()
+; CHECK-NEXT: [[TMP21:%.*]] = load ptr addrspace(1), ptr addrspace(3) @llvm.amdgcn.sw.lds.memintrinsic_kernel, align 8
+; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(1) getelementptr inbounds ([[LLVM_AMDGCN_SW_LDS_MEMINTRINSIC_KERNEL_MD_TYPE]], ptr addrspace(1) @llvm.amdgcn.sw.lds.memintrinsic_kernel.md, i32 0, i32 1, i32 0), align 4
+; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @llvm.amdgcn.sw.lds.memintrinsic_kernel, i32 [[TMP22]]
+; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr addrspace(1) getelementptr inbounds ([[LLVM_AMDGCN_SW_LDS_MEMINTRINSIC_KERNEL_MD_TYPE]], ptr addrspace(1) @llvm.amdgcn.sw.lds.memintrinsic_kernel.md, i32 0, i32 2, i32 0), align 4
+; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr addrspace(3) @llvm.amdgcn.sw.lds.memintrinsic_kernel, i32 [[TMP24]]
+; CHECK-NEXT: [[A:%.*]] = getelementptr [16 x i8], ptr addrspace(3) [[TMP23]], i32 0, i32 0
+; CHECK-NEXT: [[B:%.*]] = getelementptr [16 x i8], ptr addrspace(3) [[TMP25]], i32 0, i32 0
+; CHECK-NEXT: [[TMP26:%.*]] = ptrtoint ptr addrspace(3) [[A]] to i32
+; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP26]]
+; CHECK-NEXT: call void @llvm.memset.p1.i32(ptr addrspace(1) [[TMP27]], i8 0, i32 16, i1 false)
+; CHECK-NEXT: [[TMP28:%.*]] = ptrtoint ptr addrspace(3) [[A]] to i32
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP28]]
+; CHECK-NEXT: call void @llvm.memcpy.p1.p1.i32(ptr addrspace(1) [[TMP29]], ptr addrspace(1) [[SRC]], i32 16, i1 false)
+; CHECK-NEXT: [[TMP30:%.*]] = ptrtoint ptr addrspace(3) [[A]] to i32
+; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP30]]
+; CHECK-NEXT: [[TMP32:%.*]] = ptrtoint ptr addrspace(3) [[B]] to i32
+; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP32]]
+; CHECK-NEXT: call void @llvm.memmove.p1.p1.i32(ptr addrspace(1) [[TMP31]], ptr addrspace(1) [[TMP33]], i32 16, i1 false)
+; CHECK-NEXT: [[TMP42:%.*]] = ptrtoint ptr addrspace(3) [[A]] to i32
+; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP42]]
+; CHECK-NEXT: call void @llvm.memset.element.unordered.atomic.p1.i32(ptr addrspace(1) align 4 [[TMP43]], i8 0, i32 16, i32 4)
+; CHECK-NEXT: [[TMP44:%.*]] = ptrtoint ptr addrspace(3) [[A]] to i32
+; CHECK-NEXT: [[TMP37:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP44]]
+; CHECK-NEXT: call void @llvm.memcpy.element.unordered.atomic.p1.p1.i32(ptr addrspace(1) align 4 [[TMP37]], ptr addrspace(1) align 4 [[SRC]], i32 16, i32 4)
+; CHECK-NEXT: [[TMP38:%.*]] = ptrtoint ptr addrspace(3) [[A]] to i32
+; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = ptrtoint ptr addrspace(3) [[B]] to i32
+; CHECK-NEXT: [[TMP41:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP21]], i32 [[TMP40]]
+; CHECK-NEXT: call void @llvm.memmove.element.unordered.atomic.p1.p1.i32(ptr addrspace(1) align 4 [[TMP39]], ptr addrspace(1) align 4 [[TMP41]], i32 16, i32 4)
+; CHECK-NEXT: br label %[[CONDFREE:.*]]
+; CHECK: [[CONDFREE]]:
+; CHECK-NEXT: call void @llvm.amdgcn.s.barrier()
+; CHECK-NEXT: br i1 [[XYZCOND]], label %[[FREE:.*]], label %[[END:.*]]
+; CHECK: [[FREE]]:
+; CHECK-NEXT: [[TMP34:%.*]] = call ptr @llvm.returnaddress.p0(i32 0)
+; CHECK-NEXT: [[TMP35:%.*]] = ptrtoint ptr [[TMP34]] to i64
+; CHECK-NEXT: [[TMP36:%.*]] = ptrtoint ptr addrspace(1) [[TMP21]] to i64
+; CHECK-NEXT: call void @__asan_free_impl(i64 [[TMP36]], i64 [[TMP35]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret void
+;
+ %a = getelementptr [16 x i8], ptr addrspace(3) @lds_1, i32 0, i32 0
+ %b = getelementptr [16 x i8], ptr addrspace(3) @lds_2, i32 0, i32 0
+ call void @llvm.memset.p3.i32(ptr addrspace(3) %a, i8 0, i32 16, i1 false)
+ call void @llvm.memcpy.p3.p1.i32(ptr addrspace(3) %a, ptr addrspace(1) %src, i32 16, i1 false)
+ call void @llvm.memmove.p3.p3.i32(ptr addrspace(3) %a, ptr addrspace(3) %b, i32 16, i1 false)
+ call void @llvm.memset.element.unordered.atomic.p3.i32(ptr addrspace(3) align 4 %a, i8 0, i32 16, i32 4)
+ call void @llvm.memcpy.element.unordered.atomic.p3.p1.i32(ptr addrspace(3) align 4 %a, ptr addrspace(1) align 4 %src, i32 16, i32 4)
+ call void @llvm.memmove.element.unordered.atomic.p3.p3.i32(ptr addrspace(3) align 4 %a, ptr addrspace(3) align 4 %b, i32 16, i32 4)
+ ret void
+}
+
+declare void @llvm.memset.p3.i32(ptr addrspace(3) nocapture writeonly, i8, i32, i1)
+declare void @llvm.memcpy.p3.p1.i32(ptr addrspace(3) nocapture writeonly, ptr addrspace(1) nocapture readonly, i32, i1)
+declare void @llvm.memmove.p3.p3.i32(ptr addrspace(3) nocapture writeonly, ptr addrspace(3) nocapture readonly, i32, i1)
+declare void @llvm.memset.element.unordered.atomic.p3.i32(ptr addrspace(3) nocapture writeonly, i8, i32, i32)
+declare void @llvm.memcpy.element.unordered.atomic.p3.p1.i32(ptr addrspace(3) nocapture writeonly, ptr addrspace(1) nocapture readonly, i32, i32)
+declare void @llvm.memmove.element.unordered.atomic.p3.p3.i32(ptr addrspace(3) nocapture writeonly, ptr addrspace(3) nocapture readonly, i32, i32)
+
+!llvm.module.flags = !{!0, !1}
+!0 = !{i32 4, !"nosanitize_address", i32 1}
+!1 = !{i32 1, !"amdhsa_code_object_version", i32 500}
``````````
</details>
https://github.com/llvm/llvm-project/pull/202552
More information about the llvm-commits
mailing list