[llvm] [LICM] Improve hoisting of calls marked with invariant_load metadata (PR #193678)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 00:10:52 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Carl Ritson (perlfu)
<details>
<summary>Changes</summary>
- Consider calls marked with invariant.load speculatable
- Allow LICM to hoist call instructions marked invariant.load
- Preserve invariant.load metadata when hoisting in LICM
This improves hoisting of AMDGPU s.buffer.load memory intrinsics.
---
Full diff: https://github.com/llvm/llvm-project/pull/193678.diff
3 Files Affected:
- (modified) llvm/lib/Analysis/ValueTracking.cpp (+3)
- (modified) llvm/lib/Transforms/Scalar/LICM.cpp (+6-1)
- (added) llvm/test/CodeGen/AMDGPU/licm-sbuffer-load.ll (+79)
``````````diff
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index d304c036594c9..7ad0ba2dc7504 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -7305,6 +7305,9 @@ bool llvm::isSafeToSpeculativelyExecuteWithOpcode(
return false;
const Function *Callee = CI->getCalledFunction();
+ // Assume we can always speculate invariant loads.
+ if (CI->hasMetadata(LLVMContext::MD_invariant_load))
+ return true;
// The called function could have undefined behavior or side-effects, even
// if marked readnone nounwind.
if (!Callee || !Callee->isSpeculatable())
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 03edf9e3b0c1e..028d9719f3112 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -1232,6 +1232,10 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
// Assumes don't actually alias anything or throw
return true;
+ if (CI->hasMetadata(LLVMContext::MD_invariant_load))
+ // Call is always invariant
+ return true;
+
// Handle simple cases by querying alias analysis.
MemoryEffects Behavior = AA->getMemoryEffects(CI);
@@ -1702,7 +1706,8 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
// time in isGuaranteedToExecute if we don't actually have anything to
// drop. It is a compile time optimization, not required for correctness.
!SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop)) {
- I.dropUBImplyingAttrsAndMetadata();
+ // Preserve invariant.load as moving load does not change its invariance
+ I.dropUBImplyingAttrsAndMetadata({LLVMContext::MD_invariant_load});
}
if (isa<PHINode>(I))
diff --git a/llvm/test/CodeGen/AMDGPU/licm-sbuffer-load.ll b/llvm/test/CodeGen/AMDGPU/licm-sbuffer-load.ll
new file mode 100644
index 0000000000000..629336adea33f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/licm-sbuffer-load.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -passes=licm | FileCheck %s
+
+define amdgpu_ps void @licm_s_buffer_load(ptr addrspace(1) nocapture %out, i32 inreg noundef %userdata) {
+; CHECK-LABEL: define amdgpu_ps void @licm_s_buffer_load(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[OUT:%.*]], i32 inreg noundef [[USERDATA:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[I:%.*]] = call i64 @llvm.amdgcn.s.getpc()
+; CHECK-NEXT: [[I1:%.*]] = and i64 [[I]], -4294967296
+; CHECK-NEXT: [[I2:%.*]] = zext i32 [[USERDATA]] to i64
+; CHECK-NEXT: [[I3:%.*]] = or disjoint i64 [[I1]], [[I2]]
+; CHECK-NEXT: [[I4:%.*]] = inttoptr i64 [[I3]] to ptr addrspace(4)
+; CHECK-NEXT: br label %[[PREHEADER:.*]]
+; CHECK: [[PREHEADER]]:
+; CHECK-NEXT: [[DESC:%.*]] = load <4 x i32>, ptr addrspace(4) [[I4]], align 16
+; CHECK-NEXT: [[VAL_0:%.*]] = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> [[DESC]], i32 16, i32 0), !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT: [[COND_0:%.*]] = icmp eq i32 [[VAL_0]], 0
+; CHECK-NEXT: [[VAL_1:%.*]] = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> [[DESC]], i32 20, i32 0), !invariant.load [[META0]]
+; CHECK-NEXT: [[COND_1:%.*]] = icmp ne i32 [[VAL_1]], 0
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IDX:%.*]] = phi i32 [ 0, %[[PREHEADER]] ], [ [[NEXT_I:%.*]], %[[LOOP_END:.*]] ]
+; CHECK-NEXT: br i1 [[COND_0]], label %[[BRANCH_A:.*]], label %[[LOOP_END]]
+; CHECK: [[BRANCH_A]]:
+; CHECK-NEXT: br i1 [[COND_1]], label %[[BRANCH_B:.*]], label %[[LOOP_END]]
+; CHECK: [[BRANCH_B]]:
+; CHECK-NEXT: [[IDX_I64:%.*]] = sext i32 [[IDX]] to i64
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[IDX_I64]]
+; CHECK-NEXT: store i32 [[VAL_1]], ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT: br label %[[LOOP_END]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[NEXT_I]] = add nuw nsw i32 [[IDX]], 1
+; CHECK-NEXT: [[COND_BREAK:%.*]] = icmp eq i32 [[NEXT_I]], 1024
+; CHECK-NEXT: br i1 [[COND_BREAK]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %i = call i64 @llvm.amdgcn.s.getpc()
+ %i1 = and i64 %i, -4294967296
+ %i2 = zext i32 %userdata to i64
+ %i3 = or disjoint i64 %i1, %i2
+ %i4 = inttoptr i64 %i3 to ptr addrspace(4)
+ br label %preheader
+
+preheader:
+ %desc = load <4 x i32>, ptr addrspace(4) %i4, align 16
+ br label %loop
+
+loop:
+ %idx = phi i32 [ 0, %preheader ], [ %next.i, %loop.end ]
+ %val.0 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 16, i32 0), !invariant.load !0
+ %cond.0 = icmp eq i32 %val.0, 0
+ br i1 %cond.0, label %branch.a, label %loop.end
+
+branch.a:
+ %val.1 = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %desc, i32 20, i32 0), !invariant.load !0
+ %cond.1 = icmp ne i32 %val.1, 0
+ br i1 %cond.1, label %branch.b, label %loop.end
+
+branch.b:
+ %idx.i64 = sext i32 %idx to i64
+ %ptr = getelementptr inbounds i32, ptr addrspace(1) %out, i64 %idx.i64
+ store i32 %val.1, ptr addrspace(1) %ptr, align 4
+ br label %loop.end
+
+loop.end:
+ %next.i = add nuw nsw i32 %idx, 1
+ %cond.break = icmp eq i32 %next.i, 1024
+ br i1 %cond.break, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+!0 = !{}
+;.
+; CHECK: [[META0]] = !{}
+;.
``````````
</details>
https://github.com/llvm/llvm-project/pull/193678
More information about the llvm-commits
mailing list