[llvm] [AMDGPU] Add dereferenceable retAttr to a call to llvm.amdgcn.implicitarg.ptr in amdgpu-codegenprepare (PR #182206)
Yoonseo Choi via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 20 13:26:48 PST 2026
https://github.com/yoonseoch updated https://github.com/llvm/llvm-project/pull/182206
>From b14ae056dafa1c02e4fa6c22f43c68b092667b6a Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonchoi at amd.com>
Date: Wed, 18 Feb 2026 18:10:20 -0600
Subject: [PATCH 1/4] Mark dereferenceable calls in AMDGPU CodeGen Prepare
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 39 +++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 2cd1bf4a813c4..6f9ba9703df1c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -94,6 +94,12 @@ static cl::opt<bool> DisableFDivExpand(
cl::ReallyHidden,
cl::init(false));
+static cl::opt<bool> MarkDereferenceableCalls(
+ "amdgpu-codegenprepare-mark-dereferenceable-calls",
+ cl::desc("Mark implicitarg.ptr calls as dereferenceable to enable LICM "
+ "optimization"),
+ cl::ReallyHidden, cl::init(true));
+
class AMDGPUCodeGenPrepareImpl
: public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
public:
@@ -283,6 +289,36 @@ class AMDGPUCodeGenPrepare : public FunctionPass {
StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
};
+static bool markDereferenceableCalls(Function &F) {
+ if (F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
+ return false;
+
+ bool Changed = false;
+ unsigned CodeObjectVersion =
+ AMDGPU::getAMDHSACodeObjectVersion(*(F.getParent()));
+ uint64_t Bytes = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) ? 256 : 64;
+
+ for (BasicBlock &BB : F)
+ for (Instruction &I : BB)
+ if (auto *CI = dyn_cast<CallInst>(&I)) {
+ Function *Callee = CI->getCalledFunction();
+ if (!Callee)
+ continue;
+
+ // Check if this is a call to amdgcn_implicitarg_ptr intrinsic
+ Intrinsic::ID IID = Callee->getIntrinsicID();
+ if (IID != Intrinsic::amdgcn_implicitarg_ptr)
+ continue;
+
+ if (!CI->hasRetAttr(Attribute::Dereferenceable)) {
+ CI->addRetAttr(
+ Attribute::getWithDereferenceableBytes(CI->getContext(), Bytes));
+ Changed = true;
+ }
+ }
+
+ return Changed;
+}
} // end anonymous namespace
bool AMDGPUCodeGenPrepareImpl::run() {
@@ -304,6 +340,9 @@ bool AMDGPUCodeGenPrepareImpl::run() {
RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
}
+ if (MarkDereferenceableCalls)
+ MadeChange |= markDereferenceableCalls(F);
+
return MadeChange;
}
>From 5cdc79b33f8a5c521ce8733975c2ddc5c80a861e Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonchoi at amd.com>
Date: Wed, 18 Feb 2026 18:38:48 -0600
Subject: [PATCH 2/4] Adding new lit-tests
---
.../amdgpu-mark-dereferenceable-call.ll | 139 ++++++++++++++
.../licm-after-marking-dereferenceable.ll | 172 ++++++++++++++++++
2 files changed, 311 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
new file mode 100644
index 0000000000000..87dde8eceef5d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
@@ -0,0 +1,139 @@
+; RUN: opt -S -passes='amdgpu-codegenprepare' -mtriple=amdgcn-amd-amdhsa -disable-verify < %s | FileCheck %s --check-prefix=DEREF
+; RUN: opt -S -passes='loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa -disable-verify < %s | FileCheck %s --check-prefix=WITHOUT
+; RUN: opt -S -passes='amdgpu-codegenprepare,loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=WITH
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+
+define protected amdgpu_kernel void @foo(ptr addrspace(1) noundef readonly captures(none) %d_a.coerce, ptr addrspace(1) noundef readonly captures(none) %d_b.coerce, ptr addrspace(1) noundef writeonly captures(none) %d_c.coerce, i32 noundef %count) local_unnamed_addr #0 {
+; DEREF-LABEL: define protected amdgpu_kernel void @foo(
+; DEREF: tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+;
+; WITHOUT-LABEL: define protected amdgpu_kernel void @foo(
+; WITHOUT-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; WITHOUT-NEXT: [[ENTRY:.*:]]
+; WITHOUT-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; WITHOUT-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
+; WITHOUT-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; WITHOUT: [[FOR_BODY_LR_PH]]:
+; WITHOUT-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; WITHOUT-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; WITHOUT-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
+; WITHOUT-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
+; WITHOUT-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
+; WITHOUT-NEXT: [[TMP3:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; WITHOUT-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
+; WITHOUT-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
+; WITHOUT-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
+; WITHOUT-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
+; WITHOUT-NEXT: br label %[[FOR_BODY:.*]]
+;
+; WITH-LABEL: define protected amdgpu_kernel void @foo(
+; WITH-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; WITH-NEXT: [[ENTRY:.*:]]
+; WITH-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; WITH-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
+; WITH-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; WITH: [[FOR_BODY_LR_PH]]:
+; WITH-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; WITH-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; WITH-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
+; WITH-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
+; WITH-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
+; WITH-NEXT: [[TMP3:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; WITH-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
+; WITH-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
+; WITH-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
+; WITH-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
+; WITH-NEXT: [[DOTIN_I_I_I7:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 4
+; WITH-NEXT: [[CONV_I_I8:%.*]] = zext i16 [[DOTIN_I_I_I7]] to i32
+; WITH-NEXT: br label %[[FOR_BODY:.*]]
+;
+entry:
+ %0 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+ %cmp11 = icmp samesign ult i32 %0, 4
+ br i1 %cmp11, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph: ; preds = %entry
+ %1 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %2 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+ %cmp79 = icmp samesign ult i32 %2, 4
+ %conv = sext i32 %1 to i64
+ %mul = mul nsw i64 %conv, 192
+ %3 = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+ %4 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 12
+ %5 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 14
+ %.in.i.i.i = load i16, ptr addrspace(4) %5, align 2, !tbaa !12
+ %conv.i.i = zext i16 %.in.i.i.i to i32
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.cond.cleanup8, %entry
+ ret void
+
+for.body: ; preds = %for.cond.cleanup8, %for.body.lr.ph
+ %thread_y.012 = phi i32 [ %0, %for.body.lr.ph ], [ %add21, %for.cond.cleanup8 ]
+ br i1 %cmp79, label %for.body9.lr.ph, label %for.cond.cleanup8
+
+for.body9.lr.ph: ; preds = %for.body
+ %mul10 = shl nuw nsw i32 %thread_y.012, 2
+ %conv11 = zext nneg i32 %mul10 to i64
+ %add = add nuw nsw i64 %mul, %conv11
+ %.in.i.i.i7 = load i16, ptr addrspace(4) %4, align 4, !tbaa !12
+ %conv.i.i8 = zext i16 %.in.i.i.i7 to i32
+ br label %for.body9
+
+for.cond.cleanup8: ; preds = %for.body9, %for.body
+ %add21 = add nuw nsw i32 %thread_y.012, %conv.i.i
+ %cmp = icmp samesign ult i32 %add21, 4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !16
+
+for.body9: ; preds = %for.body9, %for.body9.lr.ph
+ %thread_x.010 = phi i32 [ %2, %for.body9.lr.ph ], [ %add18, %for.body9 ]
+ %conv12 = zext nneg i32 %thread_x.010 to i64
+ %add13 = add nuw nsw i64 %add, %conv12
+ %arrayidx = getelementptr inbounds double, ptr addrspace(1) %d_a.coerce, i64 %add13
+ %6 = load double, ptr addrspace(1) %arrayidx, align 8, !tbaa !18
+ %arrayidx14 = getelementptr inbounds double, ptr addrspace(1) %d_b.coerce, i64 %add13
+ %7 = load double, ptr addrspace(1) %arrayidx14, align 8, !tbaa !18
+ %add15 = fadd contract double %6, %7
+ %arrayidx16 = getelementptr inbounds double, ptr addrspace(1) %d_c.coerce, i64 %add13
+ store double %add15, ptr addrspace(1) %arrayidx16, align 8, !tbaa !18
+ %add18 = add nuw nsw i32 %thread_x.010, %conv.i.i8
+ %cmp7 = icmp samesign ult i32 %add18, 4
+ br i1 %cmp7, label %for.body9, label %for.cond.cleanup8, !llvm.loop !20
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #1
+
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,192" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx942" "uniform-work-group-size"="true" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!llvm.errno.tbaa = !{!7}
+!opencl.ocl.version = !{!11}
+
+!0 = !{i32 1, !"amdhsa_code_object_version", i32 600}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"int", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C++ TBAA"}
+!11 = !{i32 2, i32 0}
+!12 = !{!13, !13, i64 0}
+!13 = !{!"short", !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = distinct !{!16, !17}
+!17 = !{!"llvm.loop.mustprogress"}
+!18 = !{!19, !19, i64 0}
+!19 = !{!"double", !9, i64 0}
+!20 = distinct !{!20, !17}
diff --git a/llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll b/llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll
new file mode 100644
index 0000000000000..fb684fc84f81b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -O2 -mtriple=amdgcn--amdhsa -disable-verify < %s | FileCheck %s
+
+; There is a depth-2 loopnest.
+; Make sure there is no s_load instruction in inner loop's preheader as the load is hoisted to the outer loop's preheader.
+;
+
+; ModuleID = 'mark-dereferenceable-licm.ll'
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite)
+define protected amdgpu_kernel void @foo(ptr addrspace(1) noundef readonly captures(none) %d_a.coerce, ptr addrspace(1) noundef readonly captures(none) %d_b.coerce, ptr addrspace(1) noundef writeonly captures(none) %d_c.coerce, i32 noundef %count) local_unnamed_addr #0 {
+; CHECK-LABEL: foo:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: v_bfe_u32 v8, v0, 10, 10
+; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 4, v8
+; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT: s_cbranch_execz .LBB0_6
+; CHECK-NEXT: ; %bb.1: ; %for.body.lr.ph
+; CHECK-NEXT: s_load_dword s3, s[0:1], 0x2c
+; CHECK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
+; CHECK-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x10
+; CHECK-NEXT: v_lshlrev_b32_e32 v2, 2, v8
+; CHECK-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0xc0
+; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT: v_mad_i64_i32 v[4:5], s[0:1], s2, v1, v[2:3]
+; CHECK-NEXT: v_mov_b32_e32 v1, v3
+; CHECK-NEXT: v_lshl_add_u64 v[2:3], v[4:5], 0, v[0:1]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_lshr_b32 s12, s3, 16
+; CHECK-NEXT: s_and_b32 s13, s3, 0xffff
+; CHECK-NEXT: v_lshlrev_b64 v[6:7], 3, v[2:3]
+; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 4, v0
+; CHECK-NEXT: v_lshl_add_u64 v[2:3], s[8:9], 0, v[6:7]
+; CHECK-NEXT: s_lshl_b32 s2, s12, 5
+; CHECK-NEXT: s_mov_b32 s3, 0
+; CHECK-NEXT: s_lshl_b32 s14, s13, 3
+; CHECK-NEXT: v_lshl_add_u64 v[4:5], s[6:7], 0, v[6:7]
+; CHECK-NEXT: v_lshl_add_u64 v[6:7], s[4:5], 0, v[6:7]
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_branch .LBB0_3
+; CHECK-NEXT: .LBB0_2: ; %Flow18
+; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: v_add_u32_e32 v8, s12, v8
+; CHECK-NEXT: v_cmp_lt_u32_e64 s[0:1], 3, v8
+; CHECK-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, s[2:3]
+; CHECK-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[2:3]
+; CHECK-NEXT: s_or_b64 s[4:5], s[0:1], s[4:5]
+; CHECK-NEXT: v_lshl_add_u64 v[6:7], v[6:7], 0, s[2:3]
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_cbranch_execz .LBB0_6
+; CHECK-NEXT: .LBB0_3: ; %for.body
+; CHECK-NEXT: ; =>This Loop Header: Depth=1
+; CHECK-NEXT: ; Child Loop BB0_5 Depth 2
+; CHECK-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT: s_cbranch_execz .LBB0_2
+; CHECK-NEXT: ; %bb.4: ; %for.body9.lr.ph
+; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: s_mov_b64 s[8:9], 0
+; CHECK-NEXT: s_mov_b64 s[10:11], 0
+; CHECK-NEXT: v_mov_b32_e32 v1, v0
+; CHECK-NEXT: .LBB0_5: ; %for.body9
+; CHECK-NEXT: ; Parent Loop BB0_3 Depth=1
+; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
+; CHECK-NEXT: v_lshl_add_u64 v[10:11], v[6:7], 0, s[10:11]
+; CHECK-NEXT: v_lshl_add_u64 v[12:13], v[4:5], 0, s[10:11]
+; CHECK-NEXT: global_load_dwordx2 v[14:15], v[10:11], off
+; CHECK-NEXT: global_load_dwordx2 v[16:17], v[12:13], off
+; CHECK-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[10:11]
+; CHECK-NEXT: v_add_u32_e32 v1, s13, v1
+; CHECK-NEXT: s_add_u32 s10, s10, s14
+; CHECK-NEXT: s_addc_u32 s11, s11, 0
+; CHECK-NEXT: v_cmp_lt_u32_e64 s[0:1], 3, v1
+; CHECK-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_add_f64 v[12:13], v[14:15], v[16:17]
+; CHECK-NEXT: global_store_dwordx2 v[10:11], v[12:13], off
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; CHECK-NEXT: s_cbranch_execnz .LBB0_5
+; CHECK-NEXT: s_branch .LBB0_2
+; CHECK-NEXT: .LBB0_6: ; %Flow20
+; CHECK-NEXT: s_endpgm
+entry:
+ %0 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+ %cmp11 = icmp samesign ult i32 %0, 4
+ br i1 %cmp11, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph: ; preds = %entry
+ %1 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %2 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+ %cmp79 = icmp samesign ult i32 %2, 4
+ %conv = sext i32 %1 to i64
+ %mul = mul nsw i64 %conv, 192
+ %3 = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+ %4 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 12
+ %5 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 14
+ %.in.i.i.i = load i16, ptr addrspace(4) %5, align 2, !tbaa !12
+ %conv.i.i = zext i16 %.in.i.i.i to i32
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.cond.cleanup8, %entry
+ ret void
+
+for.body: ; preds = %for.cond.cleanup8, %for.body.lr.ph
+ %thread_y.012 = phi i32 [ %0, %for.body.lr.ph ], [ %add21, %for.cond.cleanup8 ]
+ br i1 %cmp79, label %for.body9.lr.ph, label %for.cond.cleanup8
+
+for.body9.lr.ph: ; preds = %for.body
+ %mul10 = shl nuw nsw i32 %thread_y.012, 2
+ %conv11 = zext nneg i32 %mul10 to i64
+ %add = add nuw nsw i64 %mul, %conv11
+ %.in.i.i.i7 = load i16, ptr addrspace(4) %4, align 4, !tbaa !12
+ %conv.i.i8 = zext i16 %.in.i.i.i7 to i32
+ br label %for.body9
+
+for.cond.cleanup8: ; preds = %for.body9, %for.body
+ %add21 = add nuw nsw i32 %thread_y.012, %conv.i.i
+ %cmp = icmp samesign ult i32 %add21, 4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !16
+
+for.body9: ; preds = %for.body9, %for.body9.lr.ph
+ %thread_x.010 = phi i32 [ %2, %for.body9.lr.ph ], [ %add18, %for.body9 ]
+ %conv12 = zext nneg i32 %thread_x.010 to i64
+ %add13 = add nuw nsw i64 %add, %conv12
+ %arrayidx = getelementptr inbounds double, ptr addrspace(1) %d_a.coerce, i64 %add13
+ %6 = load double, ptr addrspace(1) %arrayidx, align 8, !tbaa !18
+ %arrayidx14 = getelementptr inbounds double, ptr addrspace(1) %d_b.coerce, i64 %add13
+ %7 = load double, ptr addrspace(1) %arrayidx14, align 8, !tbaa !18
+ %add15 = fadd contract double %6, %7
+ %arrayidx16 = getelementptr inbounds double, ptr addrspace(1) %d_c.coerce, i64 %add13
+ store double %add15, ptr addrspace(1) %arrayidx16, align 8, !tbaa !18
+ %add18 = add nuw nsw i32 %thread_x.010, %conv.i.i8
+ %cmp7 = icmp samesign ult i32 %add18, 4
+ br i1 %cmp7, label %for.body9, label %for.cond.cleanup8, !llvm.loop !20
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,192" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx942" "uniform-work-group-size"="true" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!llvm.errno.tbaa = !{!7}
+!opencl.ocl.version = !{!11}
+
+!0 = !{i32 1, !"amdhsa_code_object_version", i32 600}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"int", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C++ TBAA"}
+!11 = !{i32 2, i32 0}
+!12 = !{!13, !13, i64 0}
+!13 = !{!"short", !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = distinct !{!16, !17}
+!17 = !{!"llvm.loop.mustprogress"}
+!18 = !{!19, !19, i64 0}
+!19 = !{!"double", !9, i64 0}
+!20 = distinct !{!20, !17}
>From b760cccb316043c0f2715a7af72ec592e2d0926f Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonchoi at amd.com>
Date: Wed, 18 Feb 2026 19:21:36 -0600
Subject: [PATCH 3/4] Update a lit-test
---
llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
index 87dde8eceef5d..ae73551c0230d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
@@ -1,5 +1,7 @@
; RUN: opt -S -passes='amdgpu-codegenprepare' -mtriple=amdgcn-amd-amdhsa -disable-verify < %s | FileCheck %s --check-prefix=DEREF
; RUN: opt -S -passes='loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa -disable-verify < %s | FileCheck %s --check-prefix=WITHOUT
+; RUN: opt -S -passes='amdgpu-codegenprepare,loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa -amdgpu-codegenprepare-mark-dereferenceable-calls=false < %s \
+; RUN: | FileCheck %s --check-prefix=WITHOUT
; RUN: opt -S -passes='amdgpu-codegenprepare,loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=WITH
target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
>From 5723dc5caa3b0144e185a2e35e31be29a66a4990 Mon Sep 17 00:00:00 2001
From: Yoonseo Choi <yoonchoi at amd.com>
Date: Fri, 20 Feb 2026 15:26:02 -0600
Subject: [PATCH 4/4] add dereferenceable in AMDGPUInstCombineIntrin
---
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 39 ---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 24 ++
llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 1 -
.../AMDGPU/amdgpu-instcombintrin-deref.ll | 293 ++++++++++++++++++
.../amdgpu-mark-dereferenceable-call.ll | 141 ---------
.../AMDGPU/implicit-arg-block-count.ll | 36 +--
.../CodeGen/AMDGPU/implicit-arg-v5-opt.ll | 12 +-
.../licm-after-marking-dereferenceable.ll | 172 ----------
8 files changed, 341 insertions(+), 377 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-instcombintrin-deref.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
delete mode 100644 llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 6f9ba9703df1c..2cd1bf4a813c4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -94,12 +94,6 @@ static cl::opt<bool> DisableFDivExpand(
cl::ReallyHidden,
cl::init(false));
-static cl::opt<bool> MarkDereferenceableCalls(
- "amdgpu-codegenprepare-mark-dereferenceable-calls",
- cl::desc("Mark implicitarg.ptr calls as dereferenceable to enable LICM "
- "optimization"),
- cl::ReallyHidden, cl::init(true));
-
class AMDGPUCodeGenPrepareImpl
: public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
public:
@@ -289,36 +283,6 @@ class AMDGPUCodeGenPrepare : public FunctionPass {
StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
};
-static bool markDereferenceableCalls(Function &F) {
- if (F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))
- return false;
-
- bool Changed = false;
- unsigned CodeObjectVersion =
- AMDGPU::getAMDHSACodeObjectVersion(*(F.getParent()));
- uint64_t Bytes = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) ? 256 : 64;
-
- for (BasicBlock &BB : F)
- for (Instruction &I : BB)
- if (auto *CI = dyn_cast<CallInst>(&I)) {
- Function *Callee = CI->getCalledFunction();
- if (!Callee)
- continue;
-
- // Check if this is a call to amdgcn_implicitarg_ptr intrinsic
- Intrinsic::ID IID = Callee->getIntrinsicID();
- if (IID != Intrinsic::amdgcn_implicitarg_ptr)
- continue;
-
- if (!CI->hasRetAttr(Attribute::Dereferenceable)) {
- CI->addRetAttr(
- Attribute::getWithDereferenceableBytes(CI->getContext(), Bytes));
- Changed = true;
- }
- }
-
- return Changed;
-}
} // end anonymous namespace
bool AMDGPUCodeGenPrepareImpl::run() {
@@ -340,9 +304,6 @@ bool AMDGPUCodeGenPrepareImpl::run() {
RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
}
- if (MarkDereferenceableCalls)
- MadeChange |= markDereferenceableCalls(F);
-
return MadeChange;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 0ebe69de56fa9..197a67c3bd42c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -722,6 +722,30 @@ std::optional<Instruction *>
GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Intrinsic::ID IID = II.getIntrinsicID();
switch (IID) {
+ case Intrinsic::amdgcn_implicitarg_ptr: {
+ bool Modified = false;
+ // Not checking calling function's "amdgpu-no-implicitarg-ptr" attribute
+ // under the assumption that a call to this intrinsic is not present with
+ // that attribute.
+ if (!II.hasRetAttr(Attribute::Dereferenceable)) {
+ // Add dereferenceable and alignment attributes to the result.
+ uint64_t Bytes = ST->getImplicitArgNumBytes(*II.getFunction());
+ II.addRetAttr(
+ Attribute::getWithDereferenceableBytes(II.getContext(), Bytes));
+ Modified = true;
+ }
+
+ if (!II.hasRetAttr(Attribute::Alignment)) {
+ Align Align = ST->getAlignmentForImplicitArgPtr();
+ II.addRetAttr(Attribute::getWithAlignment(II.getContext(), Align));
+ Modified = true;
+ }
+
+ if (Modified)
+ return &II;
+
+ return std::nullopt;
+ } break;
case Intrinsic::amdgcn_rcp: {
Value *Src = II.getArgOperand(0);
if (isa<PoisonValue>(Src))
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 300aca1a3d789..1cd693f46500d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -340,7 +340,6 @@ bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
}
unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
- assert(AMDGPU::isKernel(F));
// We don't allocate the segment if we know the implicit arguments weren't
// used, even if the ABI implies we need them.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-instcombintrin-deref.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-instcombintrin-deref.ll
new file mode 100644
index 0000000000000..36909ee5221d0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-instcombintrin-deref.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn -passes='instcombine' -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=DEREF
+; RUN: opt -S -mtriple=amdgcn -passes='loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=WITHOUT
+; RUN: opt -S -mtriple=amdgcn -passes='instcombine,loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=WITH
+
+; InstCombine added retattr dereferenceable(bytes) to a "call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()".
+; Subsequently, licm could hoist load from its returned pointer in depth-2 loop's preheader to depth-1 loop's preheader.
+; The load became safe to execute unconditionally as it load from a dereferenceable pointer.
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+
+define protected amdgpu_kernel void @foo(ptr addrspace(1) noundef readonly captures(none) %d_a.coerce, ptr addrspace(1) noundef readonly captures(none) %d_b.coerce, ptr addrspace(1) noundef writeonly captures(none) %d_c.coerce, i32 noundef %count) local_unnamed_addr #0 {
+; DEREF-LABEL: define protected amdgpu_kernel void @foo(
+; DEREF-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; DEREF-NEXT: [[ENTRY:.*:]]
+; DEREF-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; DEREF-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
+; DEREF-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; DEREF: [[FOR_BODY_LR_PH]]:
+; DEREF-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; DEREF-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; DEREF-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
+; DEREF-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
+; DEREF-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
+; DEREF-NEXT: [[TMP3:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; DEREF-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
+; DEREF-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
+; DEREF-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
+; DEREF-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
+; DEREF-NEXT: br label %[[FOR_BODY:.*]]
+; DEREF: [[FOR_COND_CLEANUP]]:
+; DEREF-NEXT: ret void
+; DEREF: [[FOR_BODY]]:
+; DEREF-NEXT: [[THREAD_Y_012:%.*]] = phi i32 [ [[TMP0]], %[[FOR_BODY_LR_PH]] ], [ [[ADD21:%.*]], %[[FOR_COND_CLEANUP8:.*]] ]
+; DEREF-NEXT: br i1 [[CMP79]], label %[[FOR_BODY9_LR_PH:.*]], label %[[FOR_COND_CLEANUP8]]
+; DEREF: [[FOR_BODY9_LR_PH]]:
+; DEREF-NEXT: [[MUL10:%.*]] = shl nuw nsw i32 [[THREAD_Y_012]], 2
+; DEREF-NEXT: [[CONV11:%.*]] = zext nneg i32 [[MUL10]] to i64
+; DEREF-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[MUL]], [[CONV11]]
+; DEREF-NEXT: [[DOTIN_I_I_I7:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 4, !tbaa [[SHORT_TBAA6]]
+; DEREF-NEXT: [[CONV_I_I8:%.*]] = zext i16 [[DOTIN_I_I_I7]] to i32
+; DEREF-NEXT: br label %[[FOR_BODY9:.*]]
+; DEREF: [[FOR_COND_CLEANUP8]]:
+; DEREF-NEXT: [[ADD21]] = add nuw nsw i32 [[THREAD_Y_012]], [[CONV_I_I]]
+; DEREF-NEXT: [[CMP:%.*]] = icmp samesign ult i32 [[ADD21]], 4
+; DEREF-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP10:![0-9]+]]
+; DEREF: [[FOR_BODY9]]:
+; DEREF-NEXT: [[THREAD_X_010:%.*]] = phi i32 [ [[TMP2]], %[[FOR_BODY9_LR_PH]] ], [ [[ADD18:%.*]], %[[FOR_BODY9]] ]
+; DEREF-NEXT: [[CONV12:%.*]] = zext nneg i32 [[THREAD_X_010]] to i64
+; DEREF-NEXT: [[ADD13:%.*]] = add nuw nsw i64 [[ADD]], [[CONV12]]
+; DEREF-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_A_COERCE]], i64 [[ADD13]]
+; DEREF-NEXT: [[TMP6:%.*]] = load double, ptr addrspace(1) [[ARRAYIDX]], align 8, !tbaa [[DOUBLE_TBAA12:![0-9]+]]
+; DEREF-NEXT: [[ARRAYIDX14:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_B_COERCE]], i64 [[ADD13]]
+; DEREF-NEXT: [[TMP7:%.*]] = load double, ptr addrspace(1) [[ARRAYIDX14]], align 8, !tbaa [[DOUBLE_TBAA12]]
+; DEREF-NEXT: [[ADD15:%.*]] = fadd contract double [[TMP6]], [[TMP7]]
+; DEREF-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_C_COERCE]], i64 [[ADD13]]
+; DEREF-NEXT: store double [[ADD15]], ptr addrspace(1) [[ARRAYIDX16]], align 8, !tbaa [[DOUBLE_TBAA12]]
+; DEREF-NEXT: [[ADD18]] = add nuw nsw i32 [[THREAD_X_010]], [[CONV_I_I8]]
+; DEREF-NEXT: [[CMP7:%.*]] = icmp samesign ult i32 [[ADD18]], 4
+; DEREF-NEXT: br i1 [[CMP7]], label %[[FOR_BODY9]], label %[[FOR_COND_CLEANUP8]], !llvm.loop [[LOOP14:![0-9]+]]
+;
+; WITHOUT-LABEL: define protected amdgpu_kernel void @foo(
+; WITHOUT-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; WITHOUT-NEXT: [[ENTRY:.*:]]
+; WITHOUT-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; WITHOUT-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
+; WITHOUT-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; WITHOUT: [[FOR_BODY_LR_PH]]:
+; WITHOUT-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; WITHOUT-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; WITHOUT-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
+; WITHOUT-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
+; WITHOUT-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
+; WITHOUT-NEXT: [[TMP3:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; WITHOUT-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
+; WITHOUT-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
+; WITHOUT-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
+; WITHOUT-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
+; WITHOUT-NEXT: br label %[[FOR_BODY:.*]]
+; WITHOUT: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; WITHOUT-NEXT: br label %[[FOR_COND_CLEANUP]]
+; WITHOUT: [[FOR_COND_CLEANUP]]:
+; WITHOUT-NEXT: ret void
+; WITHOUT: [[FOR_BODY]]:
+; WITHOUT-NEXT: [[THREAD_Y_012:%.*]] = phi i32 [ [[TMP0]], %[[FOR_BODY_LR_PH]] ], [ [[ADD21:%.*]], %[[FOR_COND_CLEANUP8:.*]] ]
+; WITHOUT-NEXT: br i1 [[CMP79]], label %[[FOR_BODY9_LR_PH:.*]], label %[[FOR_COND_CLEANUP8]]
+; WITHOUT: [[FOR_BODY9_LR_PH]]:
+; WITHOUT-NEXT: [[MUL10:%.*]] = shl nuw nsw i32 [[THREAD_Y_012]], 2
+; WITHOUT-NEXT: [[CONV11:%.*]] = zext nneg i32 [[MUL10]] to i64
+; WITHOUT-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[MUL]], [[CONV11]]
+; WITHOUT-NEXT: [[DOTIN_I_I_I7:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 4, !tbaa [[SHORT_TBAA6]]
+; WITHOUT-NEXT: [[CONV_I_I8:%.*]] = zext i16 [[DOTIN_I_I_I7]] to i32
+; WITHOUT-NEXT: br label %[[FOR_BODY9:.*]]
+; WITHOUT: [[FOR_COND_CLEANUP8_LOOPEXIT:.*]]:
+; WITHOUT-NEXT: br label %[[FOR_COND_CLEANUP8]]
+; WITHOUT: [[FOR_COND_CLEANUP8]]:
+; WITHOUT-NEXT: [[ADD21]] = add nuw nsw i32 [[THREAD_Y_012]], [[CONV_I_I]]
+; WITHOUT-NEXT: [[CMP:%.*]] = icmp samesign ult i32 [[ADD21]], 4
+; WITHOUT-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP10:![0-9]+]]
+; WITHOUT: [[FOR_BODY9]]:
+; WITHOUT-NEXT: [[THREAD_X_010:%.*]] = phi i32 [ [[TMP2]], %[[FOR_BODY9_LR_PH]] ], [ [[ADD18:%.*]], %[[FOR_BODY9]] ]
+; WITHOUT-NEXT: [[CONV12:%.*]] = zext nneg i32 [[THREAD_X_010]] to i64
+; WITHOUT-NEXT: [[ADD13:%.*]] = add nuw nsw i64 [[ADD]], [[CONV12]]
+; WITHOUT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_A_COERCE]], i64 [[ADD13]]
+; WITHOUT-NEXT: [[TMP6:%.*]] = load double, ptr addrspace(1) [[ARRAYIDX]], align 8, !tbaa [[DOUBLE_TBAA12:![0-9]+]]
+; WITHOUT-NEXT: [[ARRAYIDX14:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_B_COERCE]], i64 [[ADD13]]
+; WITHOUT-NEXT: [[TMP7:%.*]] = load double, ptr addrspace(1) [[ARRAYIDX14]], align 8, !tbaa [[DOUBLE_TBAA12]]
+; WITHOUT-NEXT: [[ADD15:%.*]] = fadd contract double [[TMP6]], [[TMP7]]
+; WITHOUT-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_C_COERCE]], i64 [[ADD13]]
+; WITHOUT-NEXT: store double [[ADD15]], ptr addrspace(1) [[ARRAYIDX16]], align 8, !tbaa [[DOUBLE_TBAA12]]
+; WITHOUT-NEXT: [[ADD18]] = add nuw nsw i32 [[THREAD_X_010]], [[CONV_I_I8]]
+; WITHOUT-NEXT: [[CMP7:%.*]] = icmp samesign ult i32 [[ADD18]], 4
+; WITHOUT-NEXT: br i1 [[CMP7]], label %[[FOR_BODY9]], label %[[FOR_COND_CLEANUP8_LOOPEXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+;
+; WITH-LABEL: define protected amdgpu_kernel void @foo(
+; WITH-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; WITH-NEXT: [[ENTRY:.*:]]
+; WITH-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+; WITH-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
+; WITH-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
+; WITH: [[FOR_BODY_LR_PH]]:
+; WITH-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
+; WITH-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+; WITH-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
+; WITH-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
+; WITH-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
+; WITH-NEXT: [[TMP3:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; WITH-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
+; WITH-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
+; WITH-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
+; WITH-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
+; WITH-NEXT: [[DOTIN_I_I_I7:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 4
+; WITH-NEXT: [[CONV_I_I8:%.*]] = zext i16 [[DOTIN_I_I_I7]] to i32
+; WITH-NEXT: br label %[[FOR_BODY:.*]]
+; WITH: [[FOR_COND_CLEANUP_LOOPEXIT:.*]]:
+; WITH-NEXT: br label %[[FOR_COND_CLEANUP]]
+; WITH: [[FOR_COND_CLEANUP]]:
+; WITH-NEXT: ret void
+; WITH: [[FOR_BODY]]:
+; WITH-NEXT: [[THREAD_Y_012:%.*]] = phi i32 [ [[TMP0]], %[[FOR_BODY_LR_PH]] ], [ [[ADD21:%.*]], %[[FOR_COND_CLEANUP8:.*]] ]
+; WITH-NEXT: br i1 [[CMP79]], label %[[FOR_BODY9_LR_PH:.*]], label %[[FOR_COND_CLEANUP8]]
+; WITH: [[FOR_BODY9_LR_PH]]:
+; WITH-NEXT: [[MUL10:%.*]] = shl nuw nsw i32 [[THREAD_Y_012]], 2
+; WITH-NEXT: [[CONV11:%.*]] = zext nneg i32 [[MUL10]] to i64
+; WITH-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[MUL]], [[CONV11]]
+; WITH-NEXT: br label %[[FOR_BODY9:.*]]
+; WITH: [[FOR_COND_CLEANUP8_LOOPEXIT:.*]]:
+; WITH-NEXT: br label %[[FOR_COND_CLEANUP8]]
+; WITH: [[FOR_COND_CLEANUP8]]:
+; WITH-NEXT: [[ADD21]] = add nuw nsw i32 [[THREAD_Y_012]], [[CONV_I_I]]
+; WITH-NEXT: [[CMP:%.*]] = icmp samesign ult i32 [[ADD21]], 4
+; WITH-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], !llvm.loop [[LOOP10:![0-9]+]]
+; WITH: [[FOR_BODY9]]:
+; WITH-NEXT: [[THREAD_X_010:%.*]] = phi i32 [ [[TMP2]], %[[FOR_BODY9_LR_PH]] ], [ [[ADD18:%.*]], %[[FOR_BODY9]] ]
+; WITH-NEXT: [[CONV12:%.*]] = zext nneg i32 [[THREAD_X_010]] to i64
+; WITH-NEXT: [[ADD13:%.*]] = add nuw nsw i64 [[ADD]], [[CONV12]]
+; WITH-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_A_COERCE]], i64 [[ADD13]]
+; WITH-NEXT: [[TMP6:%.*]] = load double, ptr addrspace(1) [[ARRAYIDX]], align 8, !tbaa [[DOUBLE_TBAA12:![0-9]+]]
+; WITH-NEXT: [[ARRAYIDX14:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_B_COERCE]], i64 [[ADD13]]
+; WITH-NEXT: [[TMP7:%.*]] = load double, ptr addrspace(1) [[ARRAYIDX14]], align 8, !tbaa [[DOUBLE_TBAA12]]
+; WITH-NEXT: [[ADD15:%.*]] = fadd contract double [[TMP6]], [[TMP7]]
+; WITH-NEXT: [[ARRAYIDX16:%.*]] = getelementptr inbounds double, ptr addrspace(1) [[D_C_COERCE]], i64 [[ADD13]]
+; WITH-NEXT: store double [[ADD15]], ptr addrspace(1) [[ARRAYIDX16]], align 8, !tbaa [[DOUBLE_TBAA12]]
+; WITH-NEXT: [[ADD18]] = add nuw nsw i32 [[THREAD_X_010]], [[CONV_I_I8]]
+; WITH-NEXT: [[CMP7:%.*]] = icmp samesign ult i32 [[ADD18]], 4
+; WITH-NEXT: br i1 [[CMP7]], label %[[FOR_BODY9]], label %[[FOR_COND_CLEANUP8_LOOPEXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+;
+entry:
+ %0 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
+ %cmp11 = icmp samesign ult i32 %0, 4
+ br i1 %cmp11, label %for.body.lr.ph, label %for.cond.cleanup
+
+for.body.lr.ph: ; preds = %entry
+ %1 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %2 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
+ %cmp79 = icmp samesign ult i32 %2, 4
+ %conv = sext i32 %1 to i64
+ %mul = mul nsw i64 %conv, 192
+ %3 = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+ %4 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 12
+ %5 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 14
+ %.in.i.i.i = load i16, ptr addrspace(4) %5, align 2, !tbaa !12
+ %conv.i.i = zext i16 %.in.i.i.i to i32
+ br label %for.body
+
+for.cond.cleanup: ; preds = %for.cond.cleanup8, %entry
+ ret void
+
+for.body: ; preds = %for.cond.cleanup8, %for.body.lr.ph
+ %thread_y.012 = phi i32 [ %0, %for.body.lr.ph ], [ %add21, %for.cond.cleanup8 ]
+ br i1 %cmp79, label %for.body9.lr.ph, label %for.cond.cleanup8
+
+for.body9.lr.ph: ; preds = %for.body
+ %mul10 = shl nuw nsw i32 %thread_y.012, 2
+ %conv11 = zext nneg i32 %mul10 to i64
+ %add = add nuw nsw i64 %mul, %conv11
+ %.in.i.i.i7 = load i16, ptr addrspace(4) %4, align 4, !tbaa !12
+ %conv.i.i8 = zext i16 %.in.i.i.i7 to i32
+ br label %for.body9
+
+for.cond.cleanup8: ; preds = %for.body9, %for.body
+ %add21 = add nuw nsw i32 %thread_y.012, %conv.i.i
+ %cmp = icmp samesign ult i32 %add21, 4
+ br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !16
+
+for.body9: ; preds = %for.body9, %for.body9.lr.ph
+ %thread_x.010 = phi i32 [ %2, %for.body9.lr.ph ], [ %add18, %for.body9 ]
+ %conv12 = zext nneg i32 %thread_x.010 to i64
+ %add13 = add nuw nsw i64 %add, %conv12
+ %arrayidx = getelementptr inbounds double, ptr addrspace(1) %d_a.coerce, i64 %add13
+ %6 = load double, ptr addrspace(1) %arrayidx, align 8, !tbaa !18
+ %arrayidx14 = getelementptr inbounds double, ptr addrspace(1) %d_b.coerce, i64 %add13
+ %7 = load double, ptr addrspace(1) %arrayidx14, align 8, !tbaa !18
+ %add15 = fadd contract double %6, %7
+ %arrayidx16 = getelementptr inbounds double, ptr addrspace(1) %d_c.coerce, i64 %add13
+ store double %add15, ptr addrspace(1) %arrayidx16, align 8, !tbaa !18
+ %add18 = add nuw nsw i32 %thread_x.010, %conv.i.i8
+ %cmp7 = icmp samesign ult i32 %add18, 4
+ br i1 %cmp7, label %for.body9, label %for.cond.cleanup8, !llvm.loop !20
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #1
+
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,192" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx942" "uniform-work-group-size"="true" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{!0}
+!llvm.errno.tbaa = !{!7}
+!opencl.ocl.version = !{!11}
+
+!0 = !{i32 1, !"amdhsa_code_object_version", i32 600}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"int", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C++ TBAA"}
+!11 = !{i32 2, i32 0}
+!12 = !{!13, !13, i64 0}
+!13 = !{!"short", !14, i64 0}
+!14 = !{!"omnipotent char", !15, i64 0}
+!15 = !{!"Simple C/C++ TBAA"}
+!16 = distinct !{!16, !17}
+!17 = !{!"llvm.loop.mustprogress"}
+!18 = !{!19, !19, i64 0}
+!19 = !{!"double", !9, i64 0}
+!20 = distinct !{!20, !17}
+;.
+; DEREF: [[META3:![0-9]+]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; DEREF: [[META4]] = !{!"Simple C++ TBAA"}
+; DEREF: [[SHORT_TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; DEREF: [[META7]] = !{!"short", [[META8:![0-9]+]], i64 0}
+; DEREF: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; DEREF: [[META9]] = !{!"Simple C/C++ TBAA"}
+; DEREF: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; DEREF: [[META11]] = !{!"llvm.loop.mustprogress"}
+; DEREF: [[DOUBLE_TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; DEREF: [[META13]] = !{!"double", [[META3]], i64 0}
+; DEREF: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]]}
+;.
+; WITHOUT: [[META3:![0-9]+]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; WITHOUT: [[META4]] = !{!"Simple C++ TBAA"}
+; WITHOUT: [[SHORT_TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; WITHOUT: [[META7]] = !{!"short", [[META8:![0-9]+]], i64 0}
+; WITHOUT: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; WITHOUT: [[META9]] = !{!"Simple C/C++ TBAA"}
+; WITHOUT: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; WITHOUT: [[META11]] = !{!"llvm.loop.mustprogress"}
+; WITHOUT: [[DOUBLE_TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; WITHOUT: [[META13]] = !{!"double", [[META3]], i64 0}
+; WITHOUT: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]]}
+;.
+; WITH: [[META3:![0-9]+]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; WITH: [[META4]] = !{!"Simple C++ TBAA"}
+; WITH: [[SHORT_TBAA6]] = !{[[META7:![0-9]+]], [[META7]], i64 0}
+; WITH: [[META7]] = !{!"short", [[META8:![0-9]+]], i64 0}
+; WITH: [[META8]] = !{!"omnipotent char", [[META9:![0-9]+]], i64 0}
+; WITH: [[META9]] = !{!"Simple C/C++ TBAA"}
+; WITH: [[LOOP10]] = distinct !{[[LOOP10]], [[META11:![0-9]+]]}
+; WITH: [[META11]] = !{!"llvm.loop.mustprogress"}
+; WITH: [[DOUBLE_TBAA12]] = !{[[META13:![0-9]+]], [[META13]], i64 0}
+; WITH: [[META13]] = !{!"double", [[META3]], i64 0}
+; WITH: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]]}
+;.
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
deleted file mode 100644
index ae73551c0230d..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-mark-dereferenceable-call.ll
+++ /dev/null
@@ -1,141 +0,0 @@
-; RUN: opt -S -passes='amdgpu-codegenprepare' -mtriple=amdgcn-amd-amdhsa -disable-verify < %s | FileCheck %s --check-prefix=DEREF
-; RUN: opt -S -passes='loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa -disable-verify < %s | FileCheck %s --check-prefix=WITHOUT
-; RUN: opt -S -passes='amdgpu-codegenprepare,loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa -amdgpu-codegenprepare-mark-dereferenceable-calls=false < %s \
-; RUN: | FileCheck %s --check-prefix=WITHOUT
-; RUN: opt -S -passes='amdgpu-codegenprepare,loop-mssa(licm)' -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=WITH
-
-target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
-
-define protected amdgpu_kernel void @foo(ptr addrspace(1) noundef readonly captures(none) %d_a.coerce, ptr addrspace(1) noundef readonly captures(none) %d_b.coerce, ptr addrspace(1) noundef writeonly captures(none) %d_c.coerce, i32 noundef %count) local_unnamed_addr #0 {
-; DEREF-LABEL: define protected amdgpu_kernel void @foo(
-; DEREF: tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-;
-; WITHOUT-LABEL: define protected amdgpu_kernel void @foo(
-; WITHOUT-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-; WITHOUT-NEXT: [[ENTRY:.*:]]
-; WITHOUT-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
-; WITHOUT-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
-; WITHOUT-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; WITHOUT: [[FOR_BODY_LR_PH]]:
-; WITHOUT-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
-; WITHOUT-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
-; WITHOUT-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
-; WITHOUT-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
-; WITHOUT-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
-; WITHOUT-NEXT: [[TMP3:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; WITHOUT-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
-; WITHOUT-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
-; WITHOUT-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
-; WITHOUT-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
-; WITHOUT-NEXT: br label %[[FOR_BODY:.*]]
-;
-; WITH-LABEL: define protected amdgpu_kernel void @foo(
-; WITH-SAME: ptr addrspace(1) noundef readonly captures(none) [[D_A_COERCE:%.*]], ptr addrspace(1) noundef readonly captures(none) [[D_B_COERCE:%.*]], ptr addrspace(1) noundef writeonly captures(none) [[D_C_COERCE:%.*]], i32 noundef [[COUNT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
-; WITH-NEXT: [[ENTRY:.*:]]
-; WITH-NEXT: [[TMP0:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
-; WITH-NEXT: [[CMP11:%.*]] = icmp samesign ult i32 [[TMP0]], 4
-; WITH-NEXT: br i1 [[CMP11]], label %[[FOR_BODY_LR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]
-; WITH: [[FOR_BODY_LR_PH]]:
-; WITH-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.amdgcn.workgroup.id.x()
-; WITH-NEXT: [[TMP2:%.*]] = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
-; WITH-NEXT: [[CMP79:%.*]] = icmp samesign ult i32 [[TMP2]], 4
-; WITH-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64
-; WITH-NEXT: [[MUL:%.*]] = mul nsw i64 [[CONV]], 192
-; WITH-NEXT: [[TMP3:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; WITH-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 12
-; WITH-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[TMP3]], i64 14
-; WITH-NEXT: [[DOTIN_I_I_I:%.*]] = load i16, ptr addrspace(4) [[TMP5]], align 2, !tbaa [[SHORT_TBAA6:![0-9]+]]
-; WITH-NEXT: [[CONV_I_I:%.*]] = zext i16 [[DOTIN_I_I_I]] to i32
-; WITH-NEXT: [[DOTIN_I_I_I7:%.*]] = load i16, ptr addrspace(4) [[TMP4]], align 4
-; WITH-NEXT: [[CONV_I_I8:%.*]] = zext i16 [[DOTIN_I_I_I7]] to i32
-; WITH-NEXT: br label %[[FOR_BODY:.*]]
-;
-entry:
- %0 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
- %cmp11 = icmp samesign ult i32 %0, 4
- br i1 %cmp11, label %for.body.lr.ph, label %for.cond.cleanup
-
-for.body.lr.ph: ; preds = %entry
- %1 = tail call i32 @llvm.amdgcn.workgroup.id.x()
- %2 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
- %cmp79 = icmp samesign ult i32 %2, 4
- %conv = sext i32 %1 to i64
- %mul = mul nsw i64 %conv, 192
- %3 = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
- %4 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 12
- %5 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 14
- %.in.i.i.i = load i16, ptr addrspace(4) %5, align 2, !tbaa !12
- %conv.i.i = zext i16 %.in.i.i.i to i32
- br label %for.body
-
-for.cond.cleanup: ; preds = %for.cond.cleanup8, %entry
- ret void
-
-for.body: ; preds = %for.cond.cleanup8, %for.body.lr.ph
- %thread_y.012 = phi i32 [ %0, %for.body.lr.ph ], [ %add21, %for.cond.cleanup8 ]
- br i1 %cmp79, label %for.body9.lr.ph, label %for.cond.cleanup8
-
-for.body9.lr.ph: ; preds = %for.body
- %mul10 = shl nuw nsw i32 %thread_y.012, 2
- %conv11 = zext nneg i32 %mul10 to i64
- %add = add nuw nsw i64 %mul, %conv11
- %.in.i.i.i7 = load i16, ptr addrspace(4) %4, align 4, !tbaa !12
- %conv.i.i8 = zext i16 %.in.i.i.i7 to i32
- br label %for.body9
-
-for.cond.cleanup8: ; preds = %for.body9, %for.body
- %add21 = add nuw nsw i32 %thread_y.012, %conv.i.i
- %cmp = icmp samesign ult i32 %add21, 4
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !16
-
-for.body9: ; preds = %for.body9, %for.body9.lr.ph
- %thread_x.010 = phi i32 [ %2, %for.body9.lr.ph ], [ %add18, %for.body9 ]
- %conv12 = zext nneg i32 %thread_x.010 to i64
- %add13 = add nuw nsw i64 %add, %conv12
- %arrayidx = getelementptr inbounds double, ptr addrspace(1) %d_a.coerce, i64 %add13
- %6 = load double, ptr addrspace(1) %arrayidx, align 8, !tbaa !18
- %arrayidx14 = getelementptr inbounds double, ptr addrspace(1) %d_b.coerce, i64 %add13
- %7 = load double, ptr addrspace(1) %arrayidx14, align 8, !tbaa !18
- %add15 = fadd contract double %6, %7
- %arrayidx16 = getelementptr inbounds double, ptr addrspace(1) %d_c.coerce, i64 %add13
- store double %add15, ptr addrspace(1) %arrayidx16, align 8, !tbaa !18
- %add18 = add nuw nsw i32 %thread_x.010, %conv.i.i8
- %cmp7 = icmp samesign ult i32 %add18, 4
- br i1 %cmp7, label %for.body9, label %for.cond.cleanup8, !llvm.loop !20
-}
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y() #1
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #1
-
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,192" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx942" "uniform-work-group-size"="true" }
-attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-
-!llvm.module.flags = !{!0}
-!llvm.errno.tbaa = !{!7}
-!opencl.ocl.version = !{!11}
-
-!0 = !{i32 1, !"amdhsa_code_object_version", i32 600}
-!7 = !{!8, !8, i64 0}
-!8 = !{!"int", !9, i64 0}
-!9 = !{!"omnipotent char", !10, i64 0}
-!10 = !{!"Simple C++ TBAA"}
-!11 = !{i32 2, i32 0}
-!12 = !{!13, !13, i64 0}
-!13 = !{!"short", !14, i64 0}
-!14 = !{!"omnipotent char", !15, i64 0}
-!15 = !{!"Simple C/C++ TBAA"}
-!16 = distinct !{!16, !17}
-!17 = !{!"llvm.loop.mustprogress"}
-!18 = !{!19, !19, i64 0}
-!19 = !{!"double", !9, i64 0}
-!20 = distinct !{!20, !17}
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-arg-block-count.ll b/llvm/test/CodeGen/AMDGPU/implicit-arg-block-count.ll
index 25e43a0f332c6..d698fb33c502d 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-arg-block-count.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-arg-block-count.ll
@@ -4,7 +4,7 @@
define i32 @num_blocks_x() {
; CHECK-LABEL: define i32 @num_blocks_x() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG]], align 4, !invariant.load [[META0:![0-9]+]], !noundef [[META0]]
; CHECK-NEXT: ret i32 [[TMP0]]
;
@@ -23,7 +23,7 @@ entry:
define i32 @num_blocks_y() {
; CHECK-LABEL: define i32 @num_blocks_y() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 4
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(4) [[TMP0]], align 4, !invariant.load [[META0]], !noundef [[META0]]
; CHECK-NEXT: ret i32 [[TMP1]]
@@ -43,7 +43,7 @@ entry:
define i32 @num_blocks_z() {
; CHECK-LABEL: define i32 @num_blocks_z() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 8
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(4) [[TMP0]], align 4, !invariant.load [[META0]], !noundef [[META0]]
; CHECK-NEXT: ret i32 [[TMP1]]
@@ -64,7 +64,7 @@ define i32 @num_blocks(i32 %dim) {
; CHECK-LABEL: define i32 @num_blocks(
; CHECK-SAME: i32 [[DIM:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP1:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[TMP1:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; CHECK-NEXT: switch i32 [[DIM]], label %[[DEFAULT:.*]] [
; CHECK-NEXT: i32 0, label %[[DIM_X:.*]]
; CHECK-NEXT: i32 1, label %[[DIM_Y:.*]]
@@ -133,7 +133,7 @@ exit:
define i64 @larger() {
; CHECK-LABEL: define i64 @larger() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; CHECK-NEXT: [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG]], align 4, !invariant.load [[META0]], !noundef [[META0]]
; CHECK-NEXT: [[CONV_GRID_X:%.*]] = zext i32 [[GRID_SIZE_X]] to i64
; CHECK-NEXT: ret i64 [[CONV_GRID_X]]
@@ -157,8 +157,8 @@ define i32 @bad_offset() {
; CHECK-NEXT: [[DISPATCH:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
; CHECK-NEXT: [[D_GEP_Y:%.*]] = getelementptr i8, ptr addrspace(4) [[DISPATCH]], i64 16
; CHECK-NEXT: [[GRID_SIZE_Y:%.*]] = load i32, ptr addrspace(4) [[D_GEP_Y]], align 4
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
; CHECK-NEXT: [[WG_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[I_GEP_X]], align 2
; CHECK-NEXT: [[CONV_X:%.*]] = zext i16 [[WG_SIZE_X]] to i32
; CHECK-NEXT: [[COUNT_X:%.*]] = udiv i32 [[GRID_SIZE_Y]], [[CONV_X]]
@@ -201,8 +201,8 @@ define i32 @wrong_cast() {
; CHECK-NEXT: [[DISPATCH:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
; CHECK-NEXT: [[D_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[DISPATCH]], i64 12
; CHECK-NEXT: [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[D_GEP_X]], align 4
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
; CHECK-NEXT: [[WG_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[I_GEP_X]], align 2
; CHECK-NEXT: [[CONV_X:%.*]] = sext i16 [[WG_SIZE_X]] to i32
; CHECK-NEXT: [[COUNT_X:%.*]] = udiv i32 [[GRID_SIZE_X]], [[CONV_X]]
@@ -226,8 +226,8 @@ define i32 @wrong_size() {
; CHECK-NEXT: [[DISPATCH:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
; CHECK-NEXT: [[D_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[DISPATCH]], i64 12
; CHECK-NEXT: [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[D_GEP_X]], align 4
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
; CHECK-NEXT: [[WG_SIZE_X:%.*]] = load i8, ptr addrspace(4) [[I_GEP_X]], align 2
; CHECK-NEXT: [[CONV_X:%.*]] = zext i8 [[WG_SIZE_X]] to i32
; CHECK-NEXT: [[COUNT_X:%.*]] = udiv i32 [[GRID_SIZE_X]], [[CONV_X]]
@@ -248,11 +248,11 @@ entry:
define i32 @wrong_intrinsic() {
; CHECK-LABEL: define i32 @wrong_intrinsic() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[DISPATCH:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; CHECK-NEXT: [[D_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[DISPATCH]], i64 16
+; CHECK-NEXT: [[DISPATCH:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[D_GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[DISPATCH]], i64 16
; CHECK-NEXT: [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[D_GEP_X]], align 4
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
; CHECK-NEXT: [[WG_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[I_GEP_X]], align 2
; CHECK-NEXT: [[CONV_X:%.*]] = zext i16 [[WG_SIZE_X]] to i32
; CHECK-NEXT: [[COUNT_X:%.*]] = udiv i32 [[GRID_SIZE_X]], [[CONV_X]]
@@ -277,8 +277,8 @@ define i16 @empty_use() {
; CHECK-NEXT: [[D_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[DISPATCH]], i64 12
; CHECK-NEXT: [[GRID_SIZE_X:%.*]] = load i32, ptr addrspace(4) [[D_GEP_X]], align 4
; CHECK-NEXT: [[TRUNC_X:%.*]] = trunc i32 [[GRID_SIZE_X]] to i16
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[I_GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG]], i64 12
; CHECK-NEXT: [[WG_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[I_GEP_X]], align 2
; CHECK-NEXT: [[COUNT_X:%.*]] = udiv i16 [[TRUNC_X]], [[WG_SIZE_X]]
; CHECK-NEXT: ret i16 [[COUNT_X]]
@@ -298,7 +298,7 @@ entry:
define i32 @multiple_use() {
; CHECK-LABEL: define i32 @multiple_use() {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[IMPLICITARG:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; CHECK-NEXT: [[IMPLICITARG:%.*]] = call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr addrspace(4) [[IMPLICITARG]], align 4, !invariant.load [[META0]], !noundef [[META0]]
; CHECK-NEXT: [[SUM:%.*]] = shl i32 [[TMP0]], 1
; CHECK-NEXT: ret i32 [[SUM]]
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll b/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll
index 3563e737f5520..d9bcd7e443b40 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-arg-v5-opt.ll
@@ -4,7 +4,7 @@
; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
define amdgpu_kernel void @get_local_size_x(ptr addrspace(1) %out) #0 {
; GCN-LABEL: @get_local_size_x(
-; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; GCN-NEXT: [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
; GCN-NEXT: [[LOCAL_SIZE:%.*]] = load i16, ptr addrspace(4) [[GEP_LOCAL_SIZE]], align 4
; GCN-NEXT: store i16 [[LOCAL_SIZE]], ptr addrspace(1) [[OUT:%.*]], align 2
@@ -24,7 +24,7 @@ define amdgpu_kernel void @get_local_size_x(ptr addrspace(1) %out) #0 {
; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
define amdgpu_kernel void @get_local_size_y(ptr addrspace(1) %out) #0 {
; GCN-LABEL: @get_local_size_y(
-; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; GCN-NEXT: [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 14
; GCN-NEXT: [[LOCAL_SIZE:%.*]] = load i16, ptr addrspace(4) [[GEP_LOCAL_SIZE]], align 2
; GCN-NEXT: store i16 [[LOCAL_SIZE]], ptr addrspace(1) [[OUT:%.*]], align 2
@@ -45,7 +45,7 @@ define amdgpu_kernel void @get_local_size_y(ptr addrspace(1) %out) #0 {
; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
define amdgpu_kernel void @get_local_size_z(ptr addrspace(1) %out) #0 {
; GCN-LABEL: @get_local_size_z(
-; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; GCN-NEXT: [[GEP_LOCAL_SIZE:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 16
; GCN-NEXT: [[LOCAL_SIZE:%.*]] = load i16, ptr addrspace(4) [[GEP_LOCAL_SIZE]], align 4
; GCN-NEXT: store i16 [[LOCAL_SIZE]], ptr addrspace(1) [[OUT:%.*]], align 2
@@ -105,7 +105,7 @@ define amdgpu_kernel void @get_remainder_z(ptr addrspace(1) %out) #0 {
; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
define amdgpu_kernel void @get_work_group_size_x(ptr addrspace(1) %out) #0 {
; GCN-LABEL: @get_work_group_size_x(
-; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; GCN-NEXT: [[GEP_X:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
; GCN-NEXT: [[GROUP_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[GEP_X]], align 4
; GCN-NEXT: store i16 [[GROUP_SIZE_X]], ptr addrspace(1) [[OUT:%.*]], align 2
@@ -121,7 +121,7 @@ define amdgpu_kernel void @get_work_group_size_x(ptr addrspace(1) %out) #0 {
; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
define amdgpu_kernel void @get_work_group_size_y(ptr addrspace(1) %out) #0 {
; GCN-LABEL: @get_work_group_size_y(
-; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; GCN-NEXT: [[GEP_Y:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 14
; GCN-NEXT: [[GROUP_SIZE_Y:%.*]] = load i16, ptr addrspace(4) [[GEP_Y]], align 2
; GCN-NEXT: store i16 [[GROUP_SIZE_Y]], ptr addrspace(1) [[OUT:%.*]], align 2
@@ -137,7 +137,7 @@ define amdgpu_kernel void @get_work_group_size_y(ptr addrspace(1) %out) #0 {
; Function Attrs: mustprogress nofree norecurse nosync nounwind readnone willreturn
define amdgpu_kernel void @get_work_group_size_z(ptr addrspace(1) %out) #0 {
; GCN-LABEL: @get_work_group_size_z(
-; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; GCN-NEXT: [[IMPLICITARG_PTR:%.*]] = tail call dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
; GCN-NEXT: [[GEP_Z:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 16
; GCN-NEXT: [[GROUP_SIZE_Z:%.*]] = load i16, ptr addrspace(4) [[GEP_Z]], align 4
; GCN-NEXT: store i16 [[GROUP_SIZE_Z]], ptr addrspace(1) [[OUT:%.*]], align 2
diff --git a/llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll b/llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll
deleted file mode 100644
index fb684fc84f81b..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/licm-after-marking-dereferenceable.ll
+++ /dev/null
@@ -1,172 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -O2 -mtriple=amdgcn--amdhsa -disable-verify < %s | FileCheck %s
-
-; There is a depth-2 loopnest.
-; Make sure there is no s_load instruction in inner loop's preheader as the load is hoisted to the outer loop's preheader.
-;
-
-; ModuleID = 'mark-dereferenceable-licm.ll'
-target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
-
-; Function Attrs: mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite)
-define protected amdgpu_kernel void @foo(ptr addrspace(1) noundef readonly captures(none) %d_a.coerce, ptr addrspace(1) noundef readonly captures(none) %d_b.coerce, ptr addrspace(1) noundef writeonly captures(none) %d_c.coerce, i32 noundef %count) local_unnamed_addr #0 {
-; CHECK-LABEL: foo:
-; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: v_bfe_u32 v8, v0, 10, 10
-; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 4, v8
-; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; CHECK-NEXT: s_cbranch_execz .LBB0_6
-; CHECK-NEXT: ; %bb.1: ; %for.body.lr.ph
-; CHECK-NEXT: s_load_dword s3, s[0:1], 0x2c
-; CHECK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
-; CHECK-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x10
-; CHECK-NEXT: v_lshlrev_b32_e32 v2, 2, v8
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_mov_b32_e32 v1, 0xc0
-; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT: v_mad_i64_i32 v[4:5], s[0:1], s2, v1, v[2:3]
-; CHECK-NEXT: v_mov_b32_e32 v1, v3
-; CHECK-NEXT: v_lshl_add_u64 v[2:3], v[4:5], 0, v[0:1]
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_lshr_b32 s12, s3, 16
-; CHECK-NEXT: s_and_b32 s13, s3, 0xffff
-; CHECK-NEXT: v_lshlrev_b64 v[6:7], 3, v[2:3]
-; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 4, v0
-; CHECK-NEXT: v_lshl_add_u64 v[2:3], s[8:9], 0, v[6:7]
-; CHECK-NEXT: s_lshl_b32 s2, s12, 5
-; CHECK-NEXT: s_mov_b32 s3, 0
-; CHECK-NEXT: s_lshl_b32 s14, s13, 3
-; CHECK-NEXT: v_lshl_add_u64 v[4:5], s[6:7], 0, v[6:7]
-; CHECK-NEXT: v_lshl_add_u64 v[6:7], s[4:5], 0, v[6:7]
-; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: s_branch .LBB0_3
-; CHECK-NEXT: .LBB0_2: ; %Flow18
-; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
-; CHECK-NEXT: v_add_u32_e32 v8, s12, v8
-; CHECK-NEXT: v_cmp_lt_u32_e64 s[0:1], 3, v8
-; CHECK-NEXT: v_lshl_add_u64 v[2:3], v[2:3], 0, s[2:3]
-; CHECK-NEXT: v_lshl_add_u64 v[4:5], v[4:5], 0, s[2:3]
-; CHECK-NEXT: s_or_b64 s[4:5], s[0:1], s[4:5]
-; CHECK-NEXT: v_lshl_add_u64 v[6:7], v[6:7], 0, s[2:3]
-; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; CHECK-NEXT: s_cbranch_execz .LBB0_6
-; CHECK-NEXT: .LBB0_3: ; %for.body
-; CHECK-NEXT: ; =>This Loop Header: Depth=1
-; CHECK-NEXT: ; Child Loop BB0_5 Depth 2
-; CHECK-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; CHECK-NEXT: s_cbranch_execz .LBB0_2
-; CHECK-NEXT: ; %bb.4: ; %for.body9.lr.ph
-; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
-; CHECK-NEXT: s_mov_b64 s[10:11], 0
-; CHECK-NEXT: v_mov_b32_e32 v1, v0
-; CHECK-NEXT: .LBB0_5: ; %for.body9
-; CHECK-NEXT: ; Parent Loop BB0_3 Depth=1
-; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_lshl_add_u64 v[10:11], v[6:7], 0, s[10:11]
-; CHECK-NEXT: v_lshl_add_u64 v[12:13], v[4:5], 0, s[10:11]
-; CHECK-NEXT: global_load_dwordx2 v[14:15], v[10:11], off
-; CHECK-NEXT: global_load_dwordx2 v[16:17], v[12:13], off
-; CHECK-NEXT: v_lshl_add_u64 v[10:11], v[2:3], 0, s[10:11]
-; CHECK-NEXT: v_add_u32_e32 v1, s13, v1
-; CHECK-NEXT: s_add_u32 s10, s10, s14
-; CHECK-NEXT: s_addc_u32 s11, s11, 0
-; CHECK-NEXT: v_cmp_lt_u32_e64 s[0:1], 3, v1
-; CHECK-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_add_f64 v[12:13], v[14:15], v[16:17]
-; CHECK-NEXT: global_store_dwordx2 v[10:11], v[12:13], off
-; CHECK-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; CHECK-NEXT: s_cbranch_execnz .LBB0_5
-; CHECK-NEXT: s_branch .LBB0_2
-; CHECK-NEXT: .LBB0_6: ; %Flow20
-; CHECK-NEXT: s_endpgm
-entry:
- %0 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y()
- %cmp11 = icmp samesign ult i32 %0, 4
- br i1 %cmp11, label %for.body.lr.ph, label %for.cond.cleanup
-
-for.body.lr.ph: ; preds = %entry
- %1 = tail call i32 @llvm.amdgcn.workgroup.id.x()
- %2 = tail call noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x()
- %cmp79 = icmp samesign ult i32 %2, 4
- %conv = sext i32 %1 to i64
- %mul = mul nsw i64 %conv, 192
- %3 = tail call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
- %4 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 12
- %5 = getelementptr inbounds nuw i8, ptr addrspace(4) %3, i64 14
- %.in.i.i.i = load i16, ptr addrspace(4) %5, align 2, !tbaa !12
- %conv.i.i = zext i16 %.in.i.i.i to i32
- br label %for.body
-
-for.cond.cleanup: ; preds = %for.cond.cleanup8, %entry
- ret void
-
-for.body: ; preds = %for.cond.cleanup8, %for.body.lr.ph
- %thread_y.012 = phi i32 [ %0, %for.body.lr.ph ], [ %add21, %for.cond.cleanup8 ]
- br i1 %cmp79, label %for.body9.lr.ph, label %for.cond.cleanup8
-
-for.body9.lr.ph: ; preds = %for.body
- %mul10 = shl nuw nsw i32 %thread_y.012, 2
- %conv11 = zext nneg i32 %mul10 to i64
- %add = add nuw nsw i64 %mul, %conv11
- %.in.i.i.i7 = load i16, ptr addrspace(4) %4, align 4, !tbaa !12
- %conv.i.i8 = zext i16 %.in.i.i.i7 to i32
- br label %for.body9
-
-for.cond.cleanup8: ; preds = %for.body9, %for.body
- %add21 = add nuw nsw i32 %thread_y.012, %conv.i.i
- %cmp = icmp samesign ult i32 %add21, 4
- br i1 %cmp, label %for.body, label %for.cond.cleanup, !llvm.loop !16
-
-for.body9: ; preds = %for.body9, %for.body9.lr.ph
- %thread_x.010 = phi i32 [ %2, %for.body9.lr.ph ], [ %add18, %for.body9 ]
- %conv12 = zext nneg i32 %thread_x.010 to i64
- %add13 = add nuw nsw i64 %add, %conv12
- %arrayidx = getelementptr inbounds double, ptr addrspace(1) %d_a.coerce, i64 %add13
- %6 = load double, ptr addrspace(1) %arrayidx, align 8, !tbaa !18
- %arrayidx14 = getelementptr inbounds double, ptr addrspace(1) %d_b.coerce, i64 %add13
- %7 = load double, ptr addrspace(1) %arrayidx14, align 8, !tbaa !18
- %add15 = fadd contract double %6, %7
- %arrayidx16 = getelementptr inbounds double, ptr addrspace(1) %d_c.coerce, i64 %add13
- store double %add15, ptr addrspace(1) %arrayidx16, align 8, !tbaa !18
- %add18 = add nuw nsw i32 %thread_x.010, %conv.i.i8
- %cmp7 = icmp samesign ult i32 %add18, 4
- br i1 %cmp7, label %for.body9, label %for.cond.cleanup8, !llvm.loop !20
-}
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.y() #1
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef align 4 ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr() #1
-
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
-
-attributes #0 = { mustprogress nofree norecurse nosync nounwind memory(argmem: readwrite) "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,192" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx942" "uniform-work-group-size"="true" }
-attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-
-!llvm.module.flags = !{!0}
-!llvm.errno.tbaa = !{!7}
-!opencl.ocl.version = !{!11}
-
-!0 = !{i32 1, !"amdhsa_code_object_version", i32 600}
-!7 = !{!8, !8, i64 0}
-!8 = !{!"int", !9, i64 0}
-!9 = !{!"omnipotent char", !10, i64 0}
-!10 = !{!"Simple C++ TBAA"}
-!11 = !{i32 2, i32 0}
-!12 = !{!13, !13, i64 0}
-!13 = !{!"short", !14, i64 0}
-!14 = !{!"omnipotent char", !15, i64 0}
-!15 = !{!"Simple C/C++ TBAA"}
-!16 = distinct !{!16, !17}
-!17 = !{!"llvm.loop.mustprogress"}
-!18 = !{!19, !19, i64 0}
-!19 = !{!"double", !9, i64 0}
-!20 = distinct !{!20, !17}
More information about the llvm-commits
mailing list