[llvm] [AMDGPU] Gate runtime unroll of LDS loops instead of overriding it (PR #222853)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 23:52:54 PDT 2026
https://github.com/xgxanq updated https://github.com/llvm/llvm-project/pull/222853
>From f16c831acd4f7b9c450187ed3d534151f59b73db Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Fri, 11 Sep 2026 06:34:29 +0000
Subject: [PATCH] [AMDGPU] Gate runtime unroll of LDS loops instead of
overriding it
In AMDGPUTTIImpl::getUnrollingPreferences, change UP.Runtime = UnrollRuntimeLocal
to UP.Runtime &= UnrollRuntimeLocal for loops touching local (LDS, addrspace(3))
memory. The old assignment could force runtime unrolling *on* even when other
preferences had disabled it; with the and, -amdgpu-unroll-runtime-local and
-amdgpu-unroll-threshold-local can only narrow runtime unrolling of LDS loops,
never re-enable it. This gives those knobs a well-defined effect for tuning
LDS-loop unrolling (see llvm/llvm-project#147700).
Add lit tests: runtime-unroll-local.ll (per-loop gating),
unroll-threshold-local.ll (threshold knob), unroll-runtime-local-mfma.ll
(reduced triton bf16 GEMM K-loop on gfx950 not unrolled with the knob off).
Assisted-by: Claude Code
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 11 +-
.../LoopUnroll/AMDGPU/runtime-unroll-local.ll | 63 +++
.../AMDGPU/unroll-runtime-local-mfma.ll | 415 ++++++++++++++++++
.../AMDGPU/unroll-threshold-local.ll | 38 ++
4 files changed, 522 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-threshold-local.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a7556278b7e0d..b2e6d43faee43 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -51,9 +51,10 @@ static cl::opt<unsigned> UnrollThresholdIf(
cl::init(200), cl::Hidden);
static cl::opt<bool> UnrollRuntimeLocal(
- "amdgpu-unroll-runtime-local",
- cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),
- cl::init(true), cl::Hidden);
+ "amdgpu-unroll-runtime-local",
+ cl::desc("Gate runtime unroll for AMDGPU by this knob if local memory used "
+ "in a loop"),
+ cl::init(true), cl::Hidden);
static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
"amdgpu-unroll-max-block-to-analyze",
@@ -225,9 +226,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(
(!isa<GlobalVariable>(GEP->getPointerOperand()) &&
!isa<Argument>(GEP->getPointerOperand())))
continue;
- LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
+ LLVM_DEBUG(dbgs() << "Gating unroll runtime by local knob for loop:\n"
<< *L << " due to LDS use.\n");
- UP.Runtime = UnrollRuntimeLocal;
+ UP.Runtime &= UnrollRuntimeLocal;
}
// Check if GEP depends on a value defined by this loop itself.
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
new file mode 100644
index 0000000000000..080609f8f4b85
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
@@ -0,0 +1,63 @@
+; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -S %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefixes=CHECK,NOLOCAL
+
+; -amdgpu-unroll-runtime-local gates runtime unrolling per loop, based on
+; whether that loop touches local (LDS, addrspace(3)) memory. With the knob
+; off, only the LDS loop is suppressed; the global-memory loop is unaffected.
+;
+; knob=true (default) knob=false
+; %lds.loop unrolled NOT unrolled
+; %global.loop unrolled unrolled (knob has no effect)
+
+ at lds = internal unnamed_addr addrspace(3) global [256 x i32] poison, align 4
+
+; CHECK-LABEL: @two_loops(
+define void @two_loops(ptr addrspace(1) %out, i32 %n, i32 %m) {
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %lds.loop, label %global.preheader
+
+; The LDS loop: gated by the knob.
+;
+; DEFAULT: lds.loop:
+; DEFAULT: %lds.xtraiter = and i32 %n,
+; DEFAULT: lds.loop.epil:
+;
+; NOLOCAL: lds.loop:
+; NOLOCAL-NOT: lds.xtraiter
+; NOLOCAL-NOT: lds.loop.epil
+lds.loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %lds.loop ]
+ %idx = zext i32 %iv to i64
+ %ptr = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 %idx
+ store i32 %iv, ptr addrspace(3) %ptr, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %global.preheader, label %lds.loop
+
+global.preheader:
+ %cmp2 = icmp sgt i32 %m, 0
+ br i1 %cmp2, label %global.loop, label %exit
+
+; The global-memory loop: never touches LDS, so the knob must not affect it.
+; It stays runtime-unrolled under both knob settings.
+;
+; DEFAULT: global.loop:
+; DEFAULT: %global.xtraiter = and i32 %m,
+; DEFAULT: global.loop.epil:
+;
+; NOLOCAL: global.loop:
+; NOLOCAL: %global.xtraiter = and i32 %m,
+; NOLOCAL: global.loop.epil:
+global.loop:
+ %jv = phi i32 [ 0, %global.preheader ], [ %jv.next, %global.loop ]
+ %jdx = zext i32 %jv to i64
+ %gptr = getelementptr inbounds i32, ptr addrspace(1) %out, i64 %jdx
+ store i32 %jv, ptr addrspace(1) %gptr, align 4
+ %jv.next = add nuw nsw i32 %jv, 1
+ %exitcond2 = icmp eq i32 %jv.next, %m
+ br i1 %exitcond2, label %exit, label %global.loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
new file mode 100644
index 0000000000000..9b6059b5eacad
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
@@ -0,0 +1,415 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes=loop-unroll \
+; RUN: -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefix=NOLOCAL
+;
+; Reduced triton bf16 GEMM K-loop (gfx950): convergent MFMA body reading LDS
+; (addrspace(3) @global_smem). Runtime unrolling here adds a by-2 epilogue that
+; doubles the MFMA body and regresses occupancy. With
+; -amdgpu-unroll-runtime-local off, this LDS K-loop must not be runtime-unrolled.
+;
+; NOLOCAL-LABEL: @_gemm_a16_w16_kernel
+; NOLOCAL-NOT: xtraiter
+; NOLOCAL-NOT: .epil
+; NOLOCAL-NOT: unr-lcssa
+
+ at global_smem = external addrspace(3) global [0 x i8], align 16
+
+define amdgpu_kernel void @_gemm_a16_w16_kernel_BLOCK_SIZE_M_16_BLOCK_SIZE_N_16_BLOCK_SIZE_K_256_GROUP_SIZE_M_1_NUM_KSPLIT_1_SPLITK_BLOCK_SIZE_4096_EVEN_K_1_EVEN_MN_0_cache_modifier_CG_activation_NONE_use_activation_0_ADD_BIAS_0_SKIP_REDUCE_0(ptr addrspace(1) inreg nofree readonly captures(none) %0, ptr addrspace(1) inreg nofree readonly captures(none) %1, ptr addrspace(1) inreg nofree writeonly captures(none) %2, i32 inreg %3, i32 inreg %4, i32 inreg %5, i32 inreg %6, i32 inreg %7, i32 inreg %8, i32 inreg %9, ptr addrspace(1) inreg nofree readnone captures(none) %10, ptr addrspace(1) inreg nofree readnone captures(none) %11) {
+ %13 = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %14 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %13)
+ %15 = lshr i32 %14, 6
+ %16 = and i32 %15, 7
+ %17 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %18 = add i32 %3, 15
+ %19 = sdiv i32 %18, 16
+ %20 = add i32 %4, 15
+ %21 = sdiv i32 %20, 16
+ %22 = mul i32 %21, %19
+ %23 = add i32 %22, 7
+ %24 = sdiv i32 %23, 8
+ %25 = srem i32 %22, 8
+ %26 = icmp eq i32 %25, 0
+ %27 = select i1 %26, i32 8, i32 %25
+ %28 = srem i32 %17, 8
+ %29 = sdiv i32 %17, 8
+ %30 = icmp slt i32 %28, %27
+ br i1 %30, label %31, label %33
+
+31: ; preds = %12
+ %32 = mul nsw i32 %24, %28
+ br label %39
+
+33: ; preds = %12
+ %34 = mul nsw i32 %27, %24
+ %35 = sub nsw i32 %28, %27
+ %36 = add nsw i32 %24, -1
+ %37 = mul i32 %35, %36
+ %38 = add i32 %37, %34
+ br label %39
+
+39: ; preds = %31, %33
+ %.pn = phi i32 [ %38, %33 ], [ %32, %31 ]
+ %40 = add i32 %.pn, %29
+ %41 = sdiv i32 %40, %21
+ %42 = srem i32 %40, %21
+ %43 = icmp sgt i32 %5, 0
+ br i1 %43, label %44, label %333
+
+44: ; preds = %39
+ %45 = shl i32 %41, 4
+ %46 = lshr i32 %13, 2
+ %47 = and i32 %46, 8
+ %48 = or disjoint i32 %16, %47
+ %49 = and i32 %13, 15
+ %50 = and i32 %13, 48
+ %51 = lshr exact i32 %50, 2
+ %52 = or disjoint i32 %51, 1
+ %53 = or disjoint i32 %51, 2
+ %54 = or disjoint i32 %51, 3
+ %55 = or disjoint i32 %45, %48
+ %56 = srem i32 %55, %3
+ %57 = shl nsw i32 %42, 4
+ %58 = or disjoint i32 %57, %48
+ %59 = srem i32 %58, %4
+ %60 = shl nuw nsw i32 %13, 3
+ %61 = and i32 %60, 248
+ %62 = mul i32 %56, %6
+ %63 = add i32 %62, %61
+ %64 = mul i32 %59, %7
+ %65 = add i32 %64, %61
+ %66 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %0, i16 0, i64 2147483646, i32 159744)
+ %67 = mul nuw nsw i32 %16, 1056
+ %68 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %67
+ %69 = shl i32 %63, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %66, ptr addrspace(3) %68, i32 16, i32 %69, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %70 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %1, i16 0, i64 2147483646, i32 159744)
+ %71 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), i32 %67
+ %72 = shl i32 %65, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %70, ptr addrspace(3) %71, i32 16, i32 %72, i32 0, i32 0, i32 3)
+ tail call void @llvm.amdgcn.asyncmark()
+ %73 = icmp samesign ugt i32 %5, 256
+ %74 = add i32 %63, 256
+ %75 = add i32 %65, 256
+ %76 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 8448), i32 %67
+ %77 = shl i32 %74, 1
+ %78 = select i1 %73, i32 %77, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %66, ptr addrspace(3) %76, i32 16, i32 %78, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %79 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 33760), i32 %67
+ %80 = shl i32 %75, 1
+ %81 = select i1 %73, i32 %80, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %70, ptr addrspace(3) %79, i32 16, i32 %81, i32 0, i32 0, i32 3)
+ tail call void @llvm.amdgcn.asyncmark()
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ fence syncscope("workgroup") release
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire
+ %82 = icmp samesign ugt i32 %5, 512
+ %83 = shl nuw nsw i32 %13, 10
+ %84 = and i32 %83, 7168
+ %85 = shl nuw nsw i32 %13, 6
+ %86 = and i32 %85, 512
+ %87 = or disjoint i32 %86, %50
+ %88 = or disjoint i32 %87, %84
+ %89 = lshr exact i32 %84, 5
+ %90 = add nuw nsw i32 %88, %89
+ %91 = add nuw nsw i32 %90, 64
+ %92 = add nuw nsw i32 %90, 128
+ %93 = add nuw nsw i32 %90, 192
+ %94 = add nuw nsw i32 %90, 256
+ %95 = add nuw nsw i32 %90, 320
+ %96 = add nuw nsw i32 %90, 384
+ %97 = add nuw nsw i32 %90, 448
+ br i1 %82, label %.lr.ph, label %._crit_edge
+
+.lr.ph: ; preds = %44
+ %98 = tail call i32 @llvm.umin.i32(i32 %5, i32 4096)
+ %99 = add nuw nsw i32 %98, 255
+ %100 = lshr i32 %99, 8
+ %101 = add nsw i32 %100, -3
+ br label %102
+
+102: ; preds = %.lr.ph, %102
+ %.pn3555 = phi i32 [ %75, %.lr.ph ], [ %114, %102 ]
+ %.pn1954 = phi i32 [ %74, %.lr.ph ], [ %113, %102 ]
+ %103 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 33760), %.lr.ph ], [ %140, %102 ]
+ %104 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), %.lr.ph ], [ %103, %102 ]
+ %105 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 8448), %.lr.ph ], [ %121, %102 ]
+ %106 = phi ptr addrspace(3) [ @global_smem, %.lr.ph ], [ %105, %102 ]
+ %107 = phi i32 [ 1, %.lr.ph ], [ %117, %102 ]
+ %108 = phi float [ 0.000000e+00, %.lr.ph ], [ %171, %102 ]
+ %109 = phi float [ 0.000000e+00, %.lr.ph ], [ %172, %102 ]
+ %110 = phi float [ 0.000000e+00, %.lr.ph ], [ %173, %102 ]
+ %111 = phi float [ 0.000000e+00, %.lr.ph ], [ %174, %102 ]
+ %112 = phi i32 [ 0, %.lr.ph ], [ %175, %102 ]
+ %113 = add i32 %.pn1954, 256
+ %114 = add i32 %.pn3555, 256
+ %115 = add i32 %107, 1
+ %116 = icmp slt i32 %115, 3
+ %117 = select i1 %116, i32 %115, i32 0
+ %118 = shl i32 %117, 12
+ %119 = lshr exact i32 %118, 5
+ %120 = add i32 %119, %118
+ %121 = getelementptr [2 x i8], ptr addrspace(3) @global_smem, i32 %120
+ %122 = getelementptr inbounds nuw i8, ptr addrspace(3) %121, i32 %67
+ %123 = shl i32 %113, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %66, ptr addrspace(3) %122, i32 16, i32 %123, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %124 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %90
+ %125 = load <8 x bfloat>, ptr addrspace(3) %124, align 16
+ %126 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %91
+ %127 = load <8 x bfloat>, ptr addrspace(3) %126, align 16
+ %128 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %92
+ %129 = load <8 x bfloat>, ptr addrspace(3) %128, align 16
+ %130 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %93
+ %131 = load <8 x bfloat>, ptr addrspace(3) %130, align 16
+ %132 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %94
+ %133 = load <8 x bfloat>, ptr addrspace(3) %132, align 16
+ %134 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %95
+ %135 = load <8 x bfloat>, ptr addrspace(3) %134, align 16
+ %136 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %96
+ %137 = load <8 x bfloat>, ptr addrspace(3) %136, align 16
+ %138 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %97
+ %139 = load <8 x bfloat>, ptr addrspace(3) %138, align 16
+ %140 = getelementptr [2 x i8], ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), i32 %120
+ %141 = getelementptr inbounds nuw i8, ptr addrspace(3) %140, i32 %67
+ %142 = shl i32 %114, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %70, ptr addrspace(3) %141, i32 16, i32 %142, i32 0, i32 0, i32 3)
+ tail call void @llvm.amdgcn.asyncmark()
+ %143 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %90
+ %144 = load <8 x bfloat>, ptr addrspace(3) %143, align 16
+ %145 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %91
+ %146 = load <8 x bfloat>, ptr addrspace(3) %145, align 16
+ %147 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %92
+ %148 = load <8 x bfloat>, ptr addrspace(3) %147, align 16
+ %149 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %93
+ %150 = load <8 x bfloat>, ptr addrspace(3) %149, align 16
+ %151 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %94
+ %152 = load <8 x bfloat>, ptr addrspace(3) %151, align 16
+ %153 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %95
+ %154 = load <8 x bfloat>, ptr addrspace(3) %153, align 16
+ %155 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %96
+ %156 = load <8 x bfloat>, ptr addrspace(3) %155, align 16
+ %157 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %97
+ %158 = load <8 x bfloat>, ptr addrspace(3) %157, align 16
+ %159 = insertelement <4 x float> poison, float %108, i64 0
+ %160 = insertelement <4 x float> %159, float %109, i64 1
+ %161 = insertelement <4 x float> %160, float %110, i64 2
+ %162 = insertelement <4 x float> %161, float %111, i64 3
+ %163 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %144, <8 x bfloat> %125, <4 x float> %162, i32 0, i32 0, i32 0)
+ %164 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %146, <8 x bfloat> %127, <4 x float> %163, i32 0, i32 0, i32 0)
+ %165 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %148, <8 x bfloat> %129, <4 x float> %164, i32 0, i32 0, i32 0)
+ %166 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %150, <8 x bfloat> %131, <4 x float> %165, i32 0, i32 0, i32 0)
+ %167 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %152, <8 x bfloat> %133, <4 x float> %166, i32 0, i32 0, i32 0)
+ %168 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %154, <8 x bfloat> %135, <4 x float> %167, i32 0, i32 0, i32 0)
+ %169 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %156, <8 x bfloat> %137, <4 x float> %168, i32 0, i32 0, i32 0)
+ %170 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %158, <8 x bfloat> %139, <4 x float> %169, i32 0, i32 0, i32 0)
+ %171 = extractelement <4 x float> %170, i64 0
+ %172 = extractelement <4 x float> %170, i64 1
+ %173 = extractelement <4 x float> %170, i64 2
+ %174 = extractelement <4 x float> %170, i64 3
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ fence syncscope("workgroup") release
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire
+ %175 = add nuw nsw i32 %112, 1
+ %exitcond.not = icmp eq i32 %112, %101
+ br i1 %exitcond.not, label %._crit_edge, label %102
+
+._crit_edge: ; preds = %102, %44
+ %176 = phi float [ 0.000000e+00, %44 ], [ %171, %102 ]
+ %177 = phi float [ 0.000000e+00, %44 ], [ %172, %102 ]
+ %178 = phi float [ 0.000000e+00, %44 ], [ %173, %102 ]
+ %179 = phi float [ 0.000000e+00, %44 ], [ %174, %102 ]
+ %.lcssa52 = phi ptr addrspace(3) [ @global_smem, %44 ], [ %105, %102 ]
+ %.lcssa51 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 8448), %44 ], [ %121, %102 ]
+ %180 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), %44 ], [ %103, %102 ]
+ %.lcssa = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 33760), %44 ], [ %140, %102 ]
+ %181 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %97
+ %182 = load <8 x bfloat>, ptr addrspace(3) %181, align 16
+ %183 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %96
+ %184 = load <8 x bfloat>, ptr addrspace(3) %183, align 16
+ %185 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %95
+ %186 = load <8 x bfloat>, ptr addrspace(3) %185, align 16
+ %187 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %94
+ %188 = load <8 x bfloat>, ptr addrspace(3) %187, align 16
+ %189 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %93
+ %190 = load <8 x bfloat>, ptr addrspace(3) %189, align 16
+ %191 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %92
+ %192 = load <8 x bfloat>, ptr addrspace(3) %191, align 16
+ %193 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %91
+ %194 = load <8 x bfloat>, ptr addrspace(3) %193, align 16
+ %195 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %90
+ %196 = load <8 x bfloat>, ptr addrspace(3) %195, align 16
+ %197 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %97
+ %198 = load <8 x bfloat>, ptr addrspace(3) %197, align 16
+ %199 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %96
+ %200 = load <8 x bfloat>, ptr addrspace(3) %199, align 16
+ %201 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %95
+ %202 = load <8 x bfloat>, ptr addrspace(3) %201, align 16
+ %203 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %94
+ %204 = load <8 x bfloat>, ptr addrspace(3) %203, align 16
+ %205 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %93
+ %206 = load <8 x bfloat>, ptr addrspace(3) %205, align 16
+ %207 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %92
+ %208 = load <8 x bfloat>, ptr addrspace(3) %207, align 16
+ %209 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %91
+ %210 = load <8 x bfloat>, ptr addrspace(3) %209, align 16
+ %211 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %90
+ %212 = load <8 x bfloat>, ptr addrspace(3) %211, align 16
+ %213 = insertelement <4 x float> poison, float %176, i64 0
+ %214 = insertelement <4 x float> %213, float %177, i64 1
+ %215 = insertelement <4 x float> %214, float %178, i64 2
+ %216 = insertelement <4 x float> %215, float %179, i64 3
+ %217 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %196, <8 x bfloat> %212, <4 x float> %216, i32 0, i32 0, i32 0)
+ %218 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %194, <8 x bfloat> %210, <4 x float> %217, i32 0, i32 0, i32 0)
+ %219 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %192, <8 x bfloat> %208, <4 x float> %218, i32 0, i32 0, i32 0)
+ %220 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %190, <8 x bfloat> %206, <4 x float> %219, i32 0, i32 0, i32 0)
+ %221 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %188, <8 x bfloat> %204, <4 x float> %220, i32 0, i32 0, i32 0)
+ %222 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %186, <8 x bfloat> %202, <4 x float> %221, i32 0, i32 0, i32 0)
+ %223 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %184, <8 x bfloat> %200, <4 x float> %222, i32 0, i32 0, i32 0)
+ %224 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %182, <8 x bfloat> %198, <4 x float> %223, i32 0, i32 0, i32 0)
+ %225 = extractelement <4 x float> %224, i64 0
+ %226 = extractelement <4 x float> %224, i64 1
+ %227 = extractelement <4 x float> %224, i64 2
+ %228 = extractelement <4 x float> %224, i64 3
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 0)
+ fence syncscope("workgroup") release
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire
+ br i1 %73, label %229, label %274
+
+229: ; preds = %._crit_edge
+ %230 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %97
+ %231 = load <8 x bfloat>, ptr addrspace(3) %230, align 16
+ %232 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %96
+ %233 = load <8 x bfloat>, ptr addrspace(3) %232, align 16
+ %234 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %95
+ %235 = load <8 x bfloat>, ptr addrspace(3) %234, align 16
+ %236 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %94
+ %237 = load <8 x bfloat>, ptr addrspace(3) %236, align 16
+ %238 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %93
+ %239 = load <8 x bfloat>, ptr addrspace(3) %238, align 16
+ %240 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %92
+ %241 = load <8 x bfloat>, ptr addrspace(3) %240, align 16
+ %242 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %91
+ %243 = load <8 x bfloat>, ptr addrspace(3) %242, align 16
+ %244 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa, i32 %90
+ %245 = load <8 x bfloat>, ptr addrspace(3) %244, align 16
+ %246 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %97
+ %247 = load <8 x bfloat>, ptr addrspace(3) %246, align 16
+ %248 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %96
+ %249 = load <8 x bfloat>, ptr addrspace(3) %248, align 16
+ %250 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %95
+ %251 = load <8 x bfloat>, ptr addrspace(3) %250, align 16
+ %252 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %94
+ %253 = load <8 x bfloat>, ptr addrspace(3) %252, align 16
+ %254 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %93
+ %255 = load <8 x bfloat>, ptr addrspace(3) %254, align 16
+ %256 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %92
+ %257 = load <8 x bfloat>, ptr addrspace(3) %256, align 16
+ %258 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %91
+ %259 = load <8 x bfloat>, ptr addrspace(3) %258, align 16
+ %260 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa51, i32 %90
+ %261 = load <8 x bfloat>, ptr addrspace(3) %260, align 16
+ %262 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %245, <8 x bfloat> %261, <4 x float> %224, i32 0, i32 0, i32 0)
+ %263 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %243, <8 x bfloat> %259, <4 x float> %262, i32 0, i32 0, i32 0)
+ %264 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %241, <8 x bfloat> %257, <4 x float> %263, i32 0, i32 0, i32 0)
+ %265 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %239, <8 x bfloat> %255, <4 x float> %264, i32 0, i32 0, i32 0)
+ %266 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %237, <8 x bfloat> %253, <4 x float> %265, i32 0, i32 0, i32 0)
+ %267 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %235, <8 x bfloat> %251, <4 x float> %266, i32 0, i32 0, i32 0)
+ %268 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %233, <8 x bfloat> %249, <4 x float> %267, i32 0, i32 0, i32 0)
+ %269 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %231, <8 x bfloat> %247, <4 x float> %268, i32 0, i32 0, i32 0)
+ %270 = extractelement <4 x float> %269, i64 0
+ %271 = extractelement <4 x float> %269, i64 1
+ %272 = extractelement <4 x float> %269, i64 2
+ %273 = extractelement <4 x float> %269, i64 3
+ br label %274
+
+274: ; preds = %229, %._crit_edge
+ %275 = phi float [ %273, %229 ], [ %228, %._crit_edge ]
+ %276 = phi float [ %272, %229 ], [ %227, %._crit_edge ]
+ %277 = phi float [ %271, %229 ], [ %226, %._crit_edge ]
+ %278 = phi float [ %270, %229 ], [ %225, %._crit_edge ]
+ %279 = insertelement <2 x float> poison, float %278, i64 0
+ %280 = insertelement <2 x float> %279, float %277, i64 1
+ %281 = fptrunc <2 x float> %280 to <2 x bfloat>
+ %282 = insertelement <2 x float> poison, float %276, i64 0
+ %283 = insertelement <2 x float> %282, float %275, i64 1
+ %284 = fptrunc <2 x float> %283 to <2 x bfloat>
+ %285 = sext i32 %41 to i64
+ %286 = shl nsw i64 %285, 4
+ %287 = zext nneg i32 %49 to i64
+ %288 = zext nneg i32 %51 to i64
+ %289 = zext nneg i32 %52 to i64
+ %290 = zext nneg i32 %53 to i64
+ %291 = zext nneg i32 %54 to i64
+ %292 = or disjoint i64 %286, %287
+ %293 = sext i32 %42 to i64
+ %294 = shl nsw i64 %293, 4
+ %295 = or disjoint i64 %294, %288
+ %296 = or disjoint i64 %294, %289
+ %297 = or disjoint i64 %294, %290
+ %298 = or disjoint i64 %294, %291
+ %299 = trunc i64 %286 to i32
+ %300 = or disjoint i32 %49, %299
+ %301 = mul i32 %300, %9
+ %302 = trunc nsw i64 %294 to i32
+ %303 = add i32 %301, %302
+ %304 = add i32 %303, %51
+ %305 = add i32 %303, %52
+ %306 = add i32 %303, %53
+ %307 = add i32 %303, %54
+ %308 = sext i32 %3 to i64
+ %309 = icmp slt i64 %292, %308
+ %310 = sext i32 %4 to i64
+ %311 = icmp slt i64 %295, %310
+ %312 = icmp slt i64 %296, %310
+ %313 = icmp slt i64 %297, %310
+ %314 = icmp slt i64 %298, %310
+ %315 = and i1 %309, %311
+ %316 = and i1 %309, %312
+ %317 = and i1 %309, %313
+ %318 = and i1 %309, %314
+ %319 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %2, i16 0, i64 2147483646, i32 159744)
+ %320 = icmp eq i32 %16, 0
+ %321 = and i1 %320, %315
+ %.bc = bitcast <2 x bfloat> %281 to <2 x i16>
+ %.extract = extractelement <2 x i16> %.bc, i64 0
+ %322 = shl i32 %304, 1
+ %323 = select i1 %321, i32 %322, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 %.extract, ptr addrspace(8) %319, i32 %323, i32 0, i32 0)
+ %324 = and i1 %320, %316
+ %.extract45 = extractelement <2 x i16> %.bc, i64 1
+ %325 = shl i32 %305, 1
+ %326 = select i1 %324, i32 %325, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 %.extract45, ptr addrspace(8) %319, i32 %326, i32 0, i32 0)
+ %327 = and i1 %320, %317
+ %.bc46 = bitcast <2 x bfloat> %284 to <2 x i16>
+ %.extract47 = extractelement <2 x i16> %.bc46, i64 0
+ %328 = shl i32 %306, 1
+ %329 = select i1 %327, i32 %328, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 %.extract47, ptr addrspace(8) %319, i32 %329, i32 0, i32 0)
+ %330 = and i1 %320, %318
+ %.extract49 = extractelement <2 x i16> %.bc46, i64 1
+ %331 = shl i32 %307, 1
+ %332 = select i1 %330, i32 %331, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16 %.extract49, ptr addrspace(8) %319, i32 %332, i32 0, i32 0)
+ br label %333
+
+333: ; preds = %274, %39
+ ret void
+}
+
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.workgroup.id.x()
+declare i32 @llvm.amdgcn.readfirstlane.i32(i32)
+declare void @llvm.amdgcn.s.barrier()
+declare i32 @llvm.umin.i32(i32, i32)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat>, <8 x bfloat>, <4 x float>, i32 immarg, i32 immarg, i32 immarg)
+declare ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) readnone, i16, i64, i32)
+declare void @llvm.amdgcn.asyncmark()
+declare void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) readonly captures(none), ptr addrspace(3) writeonly captures(none), i32 immarg, i32, i32, i32 immarg, i32 immarg)
+declare void @llvm.amdgcn.raw.ptr.buffer.store.i16(i16, ptr addrspace(8) writeonly captures(none), i32, i32, i32 immarg)
+declare void @llvm.amdgcn.wait.asyncmark(i16 immarg)
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-threshold-local.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-threshold-local.ll
new file mode 100644
index 0000000000000..9e96dd9f1e68c
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-threshold-local.ll
@@ -0,0 +1,38 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=loop-unroll -amdgpu-unroll-threshold-local=1000 -S %s | FileCheck %s --check-prefix=HITHRESH
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=loop-unroll -amdgpu-unroll-threshold-local=600 -S %s | FileCheck %s --check-prefix=LOTHRESH
+
+; Lock the behavior of -amdgpu-unroll-threshold-local, the knob AMD/ROCm builds
+; use to work around excessive runtime unrolling of LDS loops
+; (llvm/llvm-project#147700, which regressed llama.cpp prompt processing on
+; ROCm 7.x). The value bounds the unroll budget granted to loops that address
+; local (LDS, addrspace(3)) memory: a larger value unrolls more aggressively, a
+; smaller value keeps unrolling conservative.
+;
+; With the default-sized threshold (1000) this LDS loop is runtime-unrolled;
+; lowering the threshold to the workaround value (600) suppresses it.
+
+ at lds = internal unnamed_addr addrspace(3) global [1024 x i32] poison, align 4
+
+; HITHRESH-LABEL: @lds_threshold_gated(
+; HITHRESH: %xtraiter = and i32
+; HITHRESH: for.body.epil:
+;
+; LOTHRESH-LABEL: @lds_threshold_gated(
+; LOTHRESH-NOT: xtraiter
+; LOTHRESH-NOT: epil
+define amdgpu_kernel void @lds_threshold_gated(i32 %n) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+ %idx = zext i32 %iv to i64
+ %ptr = getelementptr inbounds [1024 x i32], ptr addrspace(3) @lds, i64 0, i64 %idx
+ store i32 %iv, ptr addrspace(3) %ptr, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %for.body
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list