[llvm] [AMDGPU] Gate runtime unroll of LDS loops instead of overriding it (PR #222853)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 15 00:15:18 PDT 2026
https://github.com/xgxanq updated https://github.com/llvm/llvm-project/pull/222853
>From 0a23edc931d1b1606cfa63ab6fc32e53b8e2efa8 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Fri, 11 Sep 2026 06:34:29 +0000
Subject: [PATCH 1/2] [AMDGPU] Gate runtime unroll of LDS loops instead of
overriding it
In AMDGPUTTIImpl::getUnrollingPreferences, change UP.Runtime = UnrollRuntimeLocal
to UP.Runtime &= UnrollRuntimeLocal for loops touching local (LDS, addrspace(3))
memory. The old assignment could force runtime unrolling *on* even when other
preferences had disabled it; with the and, -amdgpu-unroll-runtime-local can only
narrow runtime unrolling of LDS loops, never re-enable it. This gives the knob a
well-defined, per-loop effect (see llvm/llvm-project#147700).
Add lit tests:
- runtime-unroll-local.ll: per-loop gating -- with the knob off, only the LDS
loop's runtime-unroll epilogue is suppressed; a neighboring global-memory
loop stays unrolled.
- unroll-runtime-local-mfma.ll: a convergent MFMA K-loop reading LDS with a
runtime trip count is not runtime-unrolled with the knob off.
Assisted-by: Claude Code
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 4 +-
.../LoopUnroll/AMDGPU/runtime-unroll-local.ll | 57 +++++++++++++++++++
.../AMDGPU/unroll-runtime-local-mfma.ll | 34 +++++++++++
3 files changed, 93 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a7556278b7e0de..28f70515d7b97c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -225,9 +225,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(
(!isa<GlobalVariable>(GEP->getPointerOperand()) &&
!isa<Argument>(GEP->getPointerOperand())))
continue;
- LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
+ LLVM_DEBUG(dbgs() << "Gating unroll runtime by local knob for loop:\n"
<< *L << " due to LDS use.\n");
- UP.Runtime = UnrollRuntimeLocal;
+ UP.Runtime &= UnrollRuntimeLocal;
}
// Check if GEP depends on a value defined by this loop itself.
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
new file mode 100644
index 00000000000000..40c8fcc517506d
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
@@ -0,0 +1,57 @@
+; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -S %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefixes=CHECK,NOLOCAL
+
+; -amdgpu-unroll-runtime-local gates runtime unrolling per loop, based on
+; whether that loop touches local (LDS, addrspace(3)) memory. With the knob
+; off, only the LDS loop is suppressed; the global-memory loop is unaffected.
+;
+; knob=true (default) knob=false
+; %lds.loop unrolled NOT unrolled
+; %global.loop unrolled unrolled (knob has no effect)
+
+ at lds = internal unnamed_addr addrspace(3) global [256 x i32] poison, align 4
+
+; CHECK-LABEL: @two_loops(
+define void @two_loops(ptr addrspace(1) %out, i32 %n, i32 %m) {
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %lds.loop, label %global.preheader
+
+; The LDS loop: gated by the knob. Its runtime-unroll epilogue block
+; (lds.loop.epil) appears only when the knob is on.
+;
+; DEFAULT: lds.loop.epil:
+;
+; NOLOCAL-NOT: lds.loop.epil
+lds.loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %lds.loop ]
+ %idx = zext i32 %iv to i64
+ %ptr = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 %idx
+ store i32 %iv, ptr addrspace(3) %ptr, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %global.preheader, label %lds.loop
+
+global.preheader:
+ %cmp2 = icmp sgt i32 %m, 0
+ br i1 %cmp2, label %global.loop, label %exit
+
+; The global-memory loop: never touches LDS, so the knob must not affect it.
+; Its runtime-unroll epilogue block (global.loop.epil) appears under both
+; knob settings.
+;
+; DEFAULT: global.loop.epil:
+;
+; NOLOCAL: global.loop.epil:
+global.loop:
+ %jv = phi i32 [ 0, %global.preheader ], [ %jv.next, %global.loop ]
+ %jdx = zext i32 %jv to i64
+ %gptr = getelementptr inbounds i32, ptr addrspace(1) %out, i64 %jdx
+ store i32 %jv, ptr addrspace(1) %gptr, align 4
+ %jv.next = add nuw nsw i32 %jv, 1
+ %exitcond2 = icmp eq i32 %jv.next, %m
+ br i1 %exitcond2, label %exit, label %global.loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
new file mode 100644
index 00000000000000..5a2d4b8c558b4c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
@@ -0,0 +1,34 @@
+; RUN: opt -mtriple=amdgpu9.50-amd-amdhsa -passes=loop-unroll -S %s \
+; RUN: | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: opt -mtriple=amdgpu9.50-amd-amdhsa -passes=loop-unroll \
+; RUN: -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefixes=CHECK,NOLOCAL
+
+; A convergent MFMA K-loop reading LDS (addrspace(3)) with a runtime trip
+; count, modeled on a triton bf16 GEMM inner loop. By default it is runtime
+; unrolled; with -amdgpu-unroll-runtime-local off the LDS loop is not.
+
+ at global_smem = external addrspace(3) global [0 x i8], align 16
+
+; CHECK-LABEL: @gemm_k_loop(
+; DEFAULT: loop.epil:
+; NOLOCAL-NOT: loop.epil
+define amdgpu_kernel void @gemm_k_loop(<8 x bfloat> %a, i32 %n) {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi <4 x float> [ zeroinitializer, %entry ], [ %mfma, %loop ]
+ %off = shl i32 %iv, 4
+ %ptr = getelementptr inbounds i8, ptr addrspace(3) @global_smem, i32 %off
+ %b = load <8 x bfloat>, ptr addrspace(3) %ptr, align 16
+ %mfma = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %a, <8 x bfloat> %b, <4 x float> %acc, i32 0, i32 0, i32 0)
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exit = icmp eq i32 %iv.next, %n
+ br i1 %exit, label %end, label %loop
+
+end:
+ ret void
+}
+
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat>, <8 x bfloat>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
>From 29512044e5208942f5fb1df2a132bbed0158d1b9 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Tue, 15 Sep 2026 07:10:44 +0000
Subject: [PATCH 2/2] [AMDGPU][NFC] Address review: accurate debug print and
positive unroll checks
Only emit the "Gating unroll runtime" debug message when the local knob
actually turns runtime unrolling off (UP.Runtime was set and the knob is
false), matching what the &= does. This only affects LLVM_DEBUG output.
Replace the FileCheck -NOT assertions in the two LDS runtime-unroll tests
with positive checks of what is really emitted: with the knob off the LDS
loop stays intact (single body, latch branch back to itself), and the
global/MFMA loop is still unrolled with its epilogue.
Assisted-by: Claude Code
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 5 +-
.../LoopUnroll/AMDGPU/runtime-unroll-local.ll | 264 +++++++++++++++++-
.../AMDGPU/unroll-runtime-local-mfma.ll | 17 +-
3 files changed, 273 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 28f70515d7b97c..4da7bf8004997b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -225,8 +225,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(
(!isa<GlobalVariable>(GEP->getPointerOperand()) &&
!isa<Argument>(GEP->getPointerOperand())))
continue;
- LLVM_DEBUG(dbgs() << "Gating unroll runtime by local knob for loop:\n"
- << *L << " due to LDS use.\n");
+ if (UP.Runtime && !UnrollRuntimeLocal)
+ LLVM_DEBUG(dbgs() << "Gating unroll runtime by local knob for loop:\n"
+ << *L << " due to LDS use.\n");
UP.Runtime &= UnrollRuntimeLocal;
}
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
index 40c8fcc517506d..04ff4a5f478d32 100644
--- a/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -S %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefixes=CHECK,NOLOCAL
@@ -11,18 +12,257 @@
@lds = internal unnamed_addr addrspace(3) global [256 x i32] poison, align 4
-; CHECK-LABEL: @two_loops(
define void @two_loops(ptr addrspace(1) %out, i32 %n, i32 %m) {
+; DEFAULT-LABEL: define void @two_loops(
+; DEFAULT-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) {
+; DEFAULT-NEXT: [[ENTRY:.*:]]
+; DEFAULT-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; DEFAULT-NEXT: br i1 [[CMP]], label %[[LDS_LOOP_PREHEADER:.*]], label %[[GLOBAL_PREHEADER:.*]]
+; DEFAULT: [[LDS_LOOP_PREHEADER]]:
+; DEFAULT-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; DEFAULT-NEXT: [[XTRAITER:%.*]] = and i32 [[N]], 7
+; DEFAULT-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 7
+; DEFAULT-NEXT: br i1 [[TMP1]], label %[[LDS_LOOP_EPIL_PREHEADER:.*]], label %[[LDS_LOOP_PREHEADER_NEW:.*]]
+; DEFAULT: [[LDS_LOOP_PREHEADER_NEW]]:
+; DEFAULT-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[N]], [[XTRAITER]]
+; DEFAULT-NEXT: br label %[[LDS_LOOP:.*]]
+; DEFAULT: [[LDS_LOOP]]:
+; DEFAULT-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[LDS_LOOP_PREHEADER_NEW]] ], [ [[IV_NEXT_7:%.*]], %[[LDS_LOOP]] ]
+; DEFAULT-NEXT: [[NITER:%.*]] = phi i32 [ 0, %[[LDS_LOOP_PREHEADER_NEW]] ], [ [[NITER_NEXT_7:%.*]], %[[LDS_LOOP]] ]
+; DEFAULT-NEXT: [[IDX:%.*]] = zext i32 [[IV]] to i64
+; DEFAULT-NEXT: [[PTR:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX]]
+; DEFAULT-NEXT: store i32 [[IV]], ptr addrspace(3) [[PTR]], align 4
+; DEFAULT-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i32 [[IV]], 1
+; DEFAULT-NEXT: [[IDX_1:%.*]] = zext i32 [[IV_NEXT]] to i64
+; DEFAULT-NEXT: [[PTR_1:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_1]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT]], ptr addrspace(3) [[PTR_1]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_1:%.*]] = add nuw nsw i32 [[IV]], 2
+; DEFAULT-NEXT: [[IDX_2:%.*]] = zext i32 [[IV_NEXT_1]] to i64
+; DEFAULT-NEXT: [[PTR_2:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_2]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT_1]], ptr addrspace(3) [[PTR_2]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_2:%.*]] = add nuw nsw i32 [[IV]], 3
+; DEFAULT-NEXT: [[IDX_3:%.*]] = zext i32 [[IV_NEXT_2]] to i64
+; DEFAULT-NEXT: [[PTR_3:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_3]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT_2]], ptr addrspace(3) [[PTR_3]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_3:%.*]] = add nuw nsw i32 [[IV]], 4
+; DEFAULT-NEXT: [[IDX_4:%.*]] = zext i32 [[IV_NEXT_3]] to i64
+; DEFAULT-NEXT: [[PTR_4:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_4]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT_3]], ptr addrspace(3) [[PTR_4]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_4:%.*]] = add nuw nsw i32 [[IV]], 5
+; DEFAULT-NEXT: [[IDX_5:%.*]] = zext i32 [[IV_NEXT_4]] to i64
+; DEFAULT-NEXT: [[PTR_5:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_5]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT_4]], ptr addrspace(3) [[PTR_5]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_5:%.*]] = add nuw nsw i32 [[IV]], 6
+; DEFAULT-NEXT: [[IDX_6:%.*]] = zext i32 [[IV_NEXT_5]] to i64
+; DEFAULT-NEXT: [[PTR_6:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_6]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT_5]], ptr addrspace(3) [[PTR_6]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_6:%.*]] = add nuw nsw i32 [[IV]], 7
+; DEFAULT-NEXT: [[IDX_7:%.*]] = zext i32 [[IV_NEXT_6]] to i64
+; DEFAULT-NEXT: [[PTR_7:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_7]]
+; DEFAULT-NEXT: store i32 [[IV_NEXT_6]], ptr addrspace(3) [[PTR_7]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_7]] = add nuw nsw i32 [[IV]], 8
+; DEFAULT-NEXT: [[NITER_NEXT_7]] = add nuw nsw i32 [[NITER]], 8
+; DEFAULT-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i32 [[NITER_NEXT_7]], [[UNROLL_ITER]]
+; DEFAULT-NEXT: br i1 [[NITER_NCMP_7]], label %[[GLOBAL_PREHEADER_LOOPEXIT_UNR_LCSSA:.*]], label %[[LDS_LOOP]]
+; DEFAULT: [[GLOBAL_PREHEADER_LOOPEXIT_UNR_LCSSA]]:
+; DEFAULT-NEXT: [[IV_UNR:%.*]] = phi i32 [ [[IV_NEXT_7]], %[[LDS_LOOP]] ]
+; DEFAULT-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; DEFAULT-NEXT: br i1 [[LCMP_MOD]], label %[[LDS_LOOP_EPIL_PREHEADER]], label %[[GLOBAL_PREHEADER_LOOPEXIT:.*]]
+; DEFAULT: [[LDS_LOOP_EPIL_PREHEADER]]:
+; DEFAULT-NEXT: [[IV_EPIL_INIT:%.*]] = phi i32 [ 0, %[[LDS_LOOP_PREHEADER]] ], [ [[IV_UNR]], %[[GLOBAL_PREHEADER_LOOPEXIT_UNR_LCSSA]] ]
+; DEFAULT-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; DEFAULT-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]])
+; DEFAULT-NEXT: br label %[[LDS_LOOP_EPIL:.*]]
+; DEFAULT: [[LDS_LOOP_EPIL]]:
+; DEFAULT-NEXT: [[IV_EPIL:%.*]] = phi i32 [ [[IV_NEXT_EPIL:%.*]], %[[LDS_LOOP_EPIL]] ], [ [[IV_EPIL_INIT]], %[[LDS_LOOP_EPIL_PREHEADER]] ]
+; DEFAULT-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, %[[LDS_LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LDS_LOOP_EPIL]] ]
+; DEFAULT-NEXT: [[IDX_EPIL:%.*]] = zext i32 [[IV_EPIL]] to i64
+; DEFAULT-NEXT: [[PTR_EPIL:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX_EPIL]]
+; DEFAULT-NEXT: store i32 [[IV_EPIL]], ptr addrspace(3) [[PTR_EPIL]], align 4
+; DEFAULT-NEXT: [[IV_NEXT_EPIL]] = add nuw nsw i32 [[IV_EPIL]], 1
+; DEFAULT-NEXT: [[EXITCOND_EPIL:%.*]] = icmp eq i32 [[IV_NEXT_EPIL]], [[N]]
+; DEFAULT-NEXT: [[EPIL_ITER_NEXT]] = add i32 [[EPIL_ITER]], 1
+; DEFAULT-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i32 [[EPIL_ITER_NEXT]], [[XTRAITER]]
+; DEFAULT-NEXT: br i1 [[EPIL_ITER_CMP]], label %[[LDS_LOOP_EPIL]], label %[[GLOBAL_PREHEADER_LOOPEXIT_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; DEFAULT: [[GLOBAL_PREHEADER_LOOPEXIT_EPILOG_LCSSA]]:
+; DEFAULT-NEXT: br label %[[GLOBAL_PREHEADER_LOOPEXIT]]
+; DEFAULT: [[GLOBAL_PREHEADER_LOOPEXIT]]:
+; DEFAULT-NEXT: br label %[[GLOBAL_PREHEADER]]
+; DEFAULT: [[GLOBAL_PREHEADER]]:
+; DEFAULT-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[M]], 0
+; DEFAULT-NEXT: br i1 [[CMP2]], label %[[GLOBAL_LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; DEFAULT: [[GLOBAL_LOOP_PREHEADER]]:
+; DEFAULT-NEXT: [[TMP2:%.*]] = add i32 [[M]], -1
+; DEFAULT-NEXT: [[XTRAITER2:%.*]] = and i32 [[M]], 7
+; DEFAULT-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 7
+; DEFAULT-NEXT: br i1 [[TMP3]], label %[[GLOBAL_LOOP_EPIL_PREHEADER:.*]], label %[[GLOBAL_LOOP_PREHEADER_NEW:.*]]
+; DEFAULT: [[GLOBAL_LOOP_PREHEADER_NEW]]:
+; DEFAULT-NEXT: [[UNROLL_ITER6:%.*]] = sub i32 [[M]], [[XTRAITER2]]
+; DEFAULT-NEXT: br label %[[GLOBAL_LOOP:.*]]
+; DEFAULT: [[GLOBAL_LOOP]]:
+; DEFAULT-NEXT: [[JV:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_PREHEADER_NEW]] ], [ [[JV_NEXT_7:%.*]], %[[GLOBAL_LOOP]] ]
+; DEFAULT-NEXT: [[NITER7:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_PREHEADER_NEW]] ], [ [[NITER7_NEXT_7:%.*]], %[[GLOBAL_LOOP]] ]
+; DEFAULT-NEXT: [[JDX:%.*]] = zext i32 [[JV]] to i64
+; DEFAULT-NEXT: [[GPTR:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX]]
+; DEFAULT-NEXT: store i32 [[JV]], ptr addrspace(1) [[GPTR]], align 4
+; DEFAULT-NEXT: [[JV_NEXT:%.*]] = add nuw nsw i32 [[JV]], 1
+; DEFAULT-NEXT: [[JDX_1:%.*]] = zext i32 [[JV_NEXT]] to i64
+; DEFAULT-NEXT: [[GPTR_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_1]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT]], ptr addrspace(1) [[GPTR_1]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_1:%.*]] = add nuw nsw i32 [[JV]], 2
+; DEFAULT-NEXT: [[JDX_2:%.*]] = zext i32 [[JV_NEXT_1]] to i64
+; DEFAULT-NEXT: [[GPTR_2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_2]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT_1]], ptr addrspace(1) [[GPTR_2]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_2:%.*]] = add nuw nsw i32 [[JV]], 3
+; DEFAULT-NEXT: [[JDX_3:%.*]] = zext i32 [[JV_NEXT_2]] to i64
+; DEFAULT-NEXT: [[GPTR_3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_3]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT_2]], ptr addrspace(1) [[GPTR_3]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_3:%.*]] = add nuw nsw i32 [[JV]], 4
+; DEFAULT-NEXT: [[JDX_4:%.*]] = zext i32 [[JV_NEXT_3]] to i64
+; DEFAULT-NEXT: [[GPTR_4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_4]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT_3]], ptr addrspace(1) [[GPTR_4]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_4:%.*]] = add nuw nsw i32 [[JV]], 5
+; DEFAULT-NEXT: [[JDX_5:%.*]] = zext i32 [[JV_NEXT_4]] to i64
+; DEFAULT-NEXT: [[GPTR_5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_5]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT_4]], ptr addrspace(1) [[GPTR_5]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_5:%.*]] = add nuw nsw i32 [[JV]], 6
+; DEFAULT-NEXT: [[JDX_6:%.*]] = zext i32 [[JV_NEXT_5]] to i64
+; DEFAULT-NEXT: [[GPTR_6:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_6]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT_5]], ptr addrspace(1) [[GPTR_6]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_6:%.*]] = add nuw nsw i32 [[JV]], 7
+; DEFAULT-NEXT: [[JDX_7:%.*]] = zext i32 [[JV_NEXT_6]] to i64
+; DEFAULT-NEXT: [[GPTR_7:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_7]]
+; DEFAULT-NEXT: store i32 [[JV_NEXT_6]], ptr addrspace(1) [[GPTR_7]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_7]] = add nuw nsw i32 [[JV]], 8
+; DEFAULT-NEXT: [[NITER7_NEXT_7]] = add nuw nsw i32 [[NITER7]], 8
+; DEFAULT-NEXT: [[NITER7_NCMP_7:%.*]] = icmp eq i32 [[NITER7_NEXT_7]], [[UNROLL_ITER6]]
+; DEFAULT-NEXT: br i1 [[NITER7_NCMP_7]], label %[[EXIT_LOOPEXIT_UNR_LCSSA:.*]], label %[[GLOBAL_LOOP]]
+; DEFAULT: [[EXIT_LOOPEXIT_UNR_LCSSA]]:
+; DEFAULT-NEXT: [[JV_UNR:%.*]] = phi i32 [ [[JV_NEXT_7]], %[[GLOBAL_LOOP]] ]
+; DEFAULT-NEXT: [[LCMP_MOD4:%.*]] = icmp ne i32 [[XTRAITER2]], 0
+; DEFAULT-NEXT: br i1 [[LCMP_MOD4]], label %[[GLOBAL_LOOP_EPIL_PREHEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; DEFAULT: [[GLOBAL_LOOP_EPIL_PREHEADER]]:
+; DEFAULT-NEXT: [[JV_EPIL_INIT:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_PREHEADER]] ], [ [[JV_UNR]], %[[EXIT_LOOPEXIT_UNR_LCSSA]] ]
+; DEFAULT-NEXT: [[LCMP_MOD5:%.*]] = icmp ne i32 [[XTRAITER2]], 0
+; DEFAULT-NEXT: call void @llvm.assume(i1 [[LCMP_MOD5]])
+; DEFAULT-NEXT: br label %[[GLOBAL_LOOP_EPIL:.*]]
+; DEFAULT: [[GLOBAL_LOOP_EPIL]]:
+; DEFAULT-NEXT: [[JV_EPIL:%.*]] = phi i32 [ [[JV_NEXT_EPIL:%.*]], %[[GLOBAL_LOOP_EPIL]] ], [ [[JV_EPIL_INIT]], %[[GLOBAL_LOOP_EPIL_PREHEADER]] ]
+; DEFAULT-NEXT: [[EPIL_ITER3:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER3_NEXT:%.*]], %[[GLOBAL_LOOP_EPIL]] ]
+; DEFAULT-NEXT: [[JDX_EPIL:%.*]] = zext i32 [[JV_EPIL]] to i64
+; DEFAULT-NEXT: [[GPTR_EPIL:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_EPIL]]
+; DEFAULT-NEXT: store i32 [[JV_EPIL]], ptr addrspace(1) [[GPTR_EPIL]], align 4
+; DEFAULT-NEXT: [[JV_NEXT_EPIL]] = add nuw nsw i32 [[JV_EPIL]], 1
+; DEFAULT-NEXT: [[EXITCOND2_EPIL:%.*]] = icmp eq i32 [[JV_NEXT_EPIL]], [[M]]
+; DEFAULT-NEXT: [[EPIL_ITER3_NEXT]] = add i32 [[EPIL_ITER3]], 1
+; DEFAULT-NEXT: [[EPIL_ITER3_CMP:%.*]] = icmp ne i32 [[EPIL_ITER3_NEXT]], [[XTRAITER2]]
+; DEFAULT-NEXT: br i1 [[EPIL_ITER3_CMP]], label %[[GLOBAL_LOOP_EPIL]], label %[[EXIT_LOOPEXIT_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; DEFAULT: [[EXIT_LOOPEXIT_EPILOG_LCSSA]]:
+; DEFAULT-NEXT: br label %[[EXIT_LOOPEXIT]]
+; DEFAULT: [[EXIT_LOOPEXIT]]:
+; DEFAULT-NEXT: br label %[[EXIT]]
+; DEFAULT: [[EXIT]]:
+; DEFAULT-NEXT: ret void
+;
+; NOLOCAL-LABEL: define void @two_loops(
+; NOLOCAL-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) {
+; NOLOCAL-NEXT: [[ENTRY:.*:]]
+; NOLOCAL-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N]], 0
+; NOLOCAL-NEXT: br i1 [[CMP]], label %[[LDS_LOOP_PREHEADER:.*]], label %[[GLOBAL_PREHEADER:.*]]
+; NOLOCAL: [[LDS_LOOP_PREHEADER]]:
+; NOLOCAL-NEXT: br label %[[LDS_LOOP:.*]]
+; NOLOCAL: [[LDS_LOOP]]:
+; NOLOCAL-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LDS_LOOP]] ], [ 0, %[[LDS_LOOP_PREHEADER]] ]
+; NOLOCAL-NEXT: [[IDX:%.*]] = zext i32 [[IV]] to i64
+; NOLOCAL-NEXT: [[PTR:%.*]] = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 [[IDX]]
+; NOLOCAL-NEXT: store i32 [[IV]], ptr addrspace(3) [[PTR]], align 4
+; NOLOCAL-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; NOLOCAL-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; NOLOCAL-NEXT: br i1 [[EXITCOND]], label %[[GLOBAL_PREHEADER_LOOPEXIT:.*]], label %[[LDS_LOOP]]
+; NOLOCAL: [[GLOBAL_PREHEADER_LOOPEXIT]]:
+; NOLOCAL-NEXT: br label %[[GLOBAL_PREHEADER]]
+; NOLOCAL: [[GLOBAL_PREHEADER]]:
+; NOLOCAL-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[M]], 0
+; NOLOCAL-NEXT: br i1 [[CMP2]], label %[[GLOBAL_LOOP_PREHEADER:.*]], label %[[EXIT:.*]]
+; NOLOCAL: [[GLOBAL_LOOP_PREHEADER]]:
+; NOLOCAL-NEXT: [[TMP0:%.*]] = add i32 [[M]], -1
+; NOLOCAL-NEXT: [[XTRAITER:%.*]] = and i32 [[M]], 7
+; NOLOCAL-NEXT: [[TMP1:%.*]] = icmp ult i32 [[TMP0]], 7
+; NOLOCAL-NEXT: br i1 [[TMP1]], label %[[GLOBAL_LOOP_EPIL_PREHEADER:.*]], label %[[GLOBAL_LOOP_PREHEADER_NEW:.*]]
+; NOLOCAL: [[GLOBAL_LOOP_PREHEADER_NEW]]:
+; NOLOCAL-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[M]], [[XTRAITER]]
+; NOLOCAL-NEXT: br label %[[GLOBAL_LOOP:.*]]
+; NOLOCAL: [[GLOBAL_LOOP]]:
+; NOLOCAL-NEXT: [[JV:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_PREHEADER_NEW]] ], [ [[JV_NEXT_7:%.*]], %[[GLOBAL_LOOP]] ]
+; NOLOCAL-NEXT: [[NITER:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_PREHEADER_NEW]] ], [ [[NITER_NEXT_7:%.*]], %[[GLOBAL_LOOP]] ]
+; NOLOCAL-NEXT: [[JDX:%.*]] = zext i32 [[JV]] to i64
+; NOLOCAL-NEXT: [[GPTR:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX]]
+; NOLOCAL-NEXT: store i32 [[JV]], ptr addrspace(1) [[GPTR]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT:%.*]] = add nuw nsw i32 [[JV]], 1
+; NOLOCAL-NEXT: [[JDX_1:%.*]] = zext i32 [[JV_NEXT]] to i64
+; NOLOCAL-NEXT: [[GPTR_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_1]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT]], ptr addrspace(1) [[GPTR_1]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_1:%.*]] = add nuw nsw i32 [[JV]], 2
+; NOLOCAL-NEXT: [[JDX_2:%.*]] = zext i32 [[JV_NEXT_1]] to i64
+; NOLOCAL-NEXT: [[GPTR_2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_2]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT_1]], ptr addrspace(1) [[GPTR_2]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_2:%.*]] = add nuw nsw i32 [[JV]], 3
+; NOLOCAL-NEXT: [[JDX_3:%.*]] = zext i32 [[JV_NEXT_2]] to i64
+; NOLOCAL-NEXT: [[GPTR_3:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_3]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT_2]], ptr addrspace(1) [[GPTR_3]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_3:%.*]] = add nuw nsw i32 [[JV]], 4
+; NOLOCAL-NEXT: [[JDX_4:%.*]] = zext i32 [[JV_NEXT_3]] to i64
+; NOLOCAL-NEXT: [[GPTR_4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_4]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT_3]], ptr addrspace(1) [[GPTR_4]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_4:%.*]] = add nuw nsw i32 [[JV]], 5
+; NOLOCAL-NEXT: [[JDX_5:%.*]] = zext i32 [[JV_NEXT_4]] to i64
+; NOLOCAL-NEXT: [[GPTR_5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_5]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT_4]], ptr addrspace(1) [[GPTR_5]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_5:%.*]] = add nuw nsw i32 [[JV]], 6
+; NOLOCAL-NEXT: [[JDX_6:%.*]] = zext i32 [[JV_NEXT_5]] to i64
+; NOLOCAL-NEXT: [[GPTR_6:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_6]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT_5]], ptr addrspace(1) [[GPTR_6]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_6:%.*]] = add nuw nsw i32 [[JV]], 7
+; NOLOCAL-NEXT: [[JDX_7:%.*]] = zext i32 [[JV_NEXT_6]] to i64
+; NOLOCAL-NEXT: [[GPTR_7:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_7]]
+; NOLOCAL-NEXT: store i32 [[JV_NEXT_6]], ptr addrspace(1) [[GPTR_7]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_7]] = add nuw nsw i32 [[JV]], 8
+; NOLOCAL-NEXT: [[NITER_NEXT_7]] = add nuw nsw i32 [[NITER]], 8
+; NOLOCAL-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i32 [[NITER_NEXT_7]], [[UNROLL_ITER]]
+; NOLOCAL-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_LOOPEXIT_UNR_LCSSA:.*]], label %[[GLOBAL_LOOP]]
+; NOLOCAL: [[EXIT_LOOPEXIT_UNR_LCSSA]]:
+; NOLOCAL-NEXT: [[JV_UNR:%.*]] = phi i32 [ [[JV_NEXT_7]], %[[GLOBAL_LOOP]] ]
+; NOLOCAL-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; NOLOCAL-NEXT: br i1 [[LCMP_MOD]], label %[[GLOBAL_LOOP_EPIL_PREHEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; NOLOCAL: [[GLOBAL_LOOP_EPIL_PREHEADER]]:
+; NOLOCAL-NEXT: [[JV_EPIL_INIT:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_PREHEADER]] ], [ [[JV_UNR]], %[[EXIT_LOOPEXIT_UNR_LCSSA]] ]
+; NOLOCAL-NEXT: [[LCMP_MOD1:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; NOLOCAL-NEXT: call void @llvm.assume(i1 [[LCMP_MOD1]])
+; NOLOCAL-NEXT: br label %[[GLOBAL_LOOP_EPIL:.*]]
+; NOLOCAL: [[GLOBAL_LOOP_EPIL]]:
+; NOLOCAL-NEXT: [[JV_EPIL:%.*]] = phi i32 [ [[JV_NEXT_EPIL:%.*]], %[[GLOBAL_LOOP_EPIL]] ], [ [[JV_EPIL_INIT]], %[[GLOBAL_LOOP_EPIL_PREHEADER]] ]
+; NOLOCAL-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, %[[GLOBAL_LOOP_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[GLOBAL_LOOP_EPIL]] ]
+; NOLOCAL-NEXT: [[JDX_EPIL:%.*]] = zext i32 [[JV_EPIL]] to i64
+; NOLOCAL-NEXT: [[GPTR_EPIL:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 [[JDX_EPIL]]
+; NOLOCAL-NEXT: store i32 [[JV_EPIL]], ptr addrspace(1) [[GPTR_EPIL]], align 4
+; NOLOCAL-NEXT: [[JV_NEXT_EPIL]] = add nuw nsw i32 [[JV_EPIL]], 1
+; NOLOCAL-NEXT: [[EXITCOND2_EPIL:%.*]] = icmp eq i32 [[JV_NEXT_EPIL]], [[M]]
+; NOLOCAL-NEXT: [[EPIL_ITER_NEXT]] = add i32 [[EPIL_ITER]], 1
+; NOLOCAL-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i32 [[EPIL_ITER_NEXT]], [[XTRAITER]]
+; NOLOCAL-NEXT: br i1 [[EPIL_ITER_CMP]], label %[[GLOBAL_LOOP_EPIL]], label %[[EXIT_LOOPEXIT_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; NOLOCAL: [[EXIT_LOOPEXIT_EPILOG_LCSSA]]:
+; NOLOCAL-NEXT: br label %[[EXIT_LOOPEXIT]]
+; NOLOCAL: [[EXIT_LOOPEXIT]]:
+; NOLOCAL-NEXT: br label %[[EXIT]]
+; NOLOCAL: [[EXIT]]:
+; NOLOCAL-NEXT: ret void
+;
entry:
%cmp = icmp sgt i32 %n, 0
br i1 %cmp, label %lds.loop, label %global.preheader
; The LDS loop: gated by the knob. Its runtime-unroll epilogue block
; (lds.loop.epil) appears only when the knob is on.
-;
-; DEFAULT: lds.loop.epil:
-;
-; NOLOCAL-NOT: lds.loop.epil
lds.loop:
%iv = phi i32 [ 0, %entry ], [ %iv.next, %lds.loop ]
%idx = zext i32 %iv to i64
@@ -39,10 +279,6 @@ global.preheader:
; The global-memory loop: never touches LDS, so the knob must not affect it.
; Its runtime-unroll epilogue block (global.loop.epil) appears under both
; knob settings.
-;
-; DEFAULT: global.loop.epil:
-;
-; NOLOCAL: global.loop.epil:
global.loop:
%jv = phi i32 [ 0, %global.preheader ], [ %jv.next, %global.loop ]
%jdx = zext i32 %jv to i64
@@ -55,3 +291,13 @@ global.loop:
exit:
ret void
}
+;.
+; DEFAULT: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; DEFAULT: [[META1]] = !{!"llvm.loop.unroll.disable"}
+; DEFAULT: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+;.
+; NOLOCAL: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; NOLOCAL: [[META1]] = !{!"llvm.loop.unroll.disable"}
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
index 5a2d4b8c558b4c..edffa534e68aad 100644
--- a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
@@ -10,8 +10,21 @@
@global_smem = external addrspace(3) global [0 x i8], align 16
; CHECK-LABEL: @gemm_k_loop(
-; DEFAULT: loop.epil:
-; NOLOCAL-NOT: loop.epil
+;
+; With the knob on (default), the LDS loop is runtime unrolled: the body is
+; replicated (mfma.1 .. mfma.7) and a remainder epilogue is emitted.
+; DEFAULT: loop:
+; DEFAULT: %mfma = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(
+; DEFAULT: %mfma.7 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(
+; DEFAULT: loop.epil:
+;
+; With the knob off, the LDS loop is left intact: one mfma per iteration and a
+; single latch branch back to itself, no unrolled copies and no epilogue.
+; NOLOCAL: loop:
+; NOLOCAL: %mfma = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(
+; NOLOCAL-NEXT: %iv.next = add nuw nsw i32 %iv, 1
+; NOLOCAL-NEXT: %exit = icmp eq i32 %iv.next, %n
+; NOLOCAL-NEXT: br i1 %exit, label %end, label %loop
define amdgpu_kernel void @gemm_k_loop(<8 x bfloat> %a, i32 %n) {
entry:
br label %loop
More information about the llvm-commits
mailing list