[llvm] Amdgpu gate lds runtime unroll (PR #222853)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 23:33:05 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: xgxanq
<details>
<summary>Changes</summary>
---
Patch is 29.80 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222853.diff
4 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp (+4-3)
- (added) llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll (+63)
- (added) llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll (+415)
- (added) llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-threshold-local.ll (+38)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a7556278b7e0d..d3bd602550a5e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -52,7 +52,8 @@ static cl::opt<unsigned> UnrollThresholdIf(
static cl::opt<bool> UnrollRuntimeLocal(
"amdgpu-unroll-runtime-local",
- cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),
+ cl::desc("Gate runtime unroll for AMDGPU by this knob if local memory used "
+ "in a loop"),
cl::init(true), cl::Hidden);
static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
@@ -225,9 +226,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(
(!isa<GlobalVariable>(GEP->getPointerOperand()) &&
!isa<Argument>(GEP->getPointerOperand())))
continue;
- LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
+ LLVM_DEBUG(dbgs() << "Gating unroll runtime by local knob for loop:\n"
<< *L << " due to LDS use.\n");
- UP.Runtime = UnrollRuntimeLocal;
+ UP.Runtime &= UnrollRuntimeLocal;
}
// Check if GEP depends on a value defined by this loop itself.
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
new file mode 100644
index 0000000000000..080609f8f4b85
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/runtime-unroll-local.ll
@@ -0,0 +1,63 @@
+; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -S %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: opt -mtriple=amdgpu-- -passes=loop-unroll -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefixes=CHECK,NOLOCAL
+
+; -amdgpu-unroll-runtime-local gates runtime unrolling per loop, based on
+; whether that loop touches local (LDS, addrspace(3)) memory. With the knob
+; off, only the LDS loop is suppressed; the global-memory loop is unaffected.
+;
+; knob=true (default) knob=false
+; %lds.loop unrolled NOT unrolled
+; %global.loop unrolled unrolled (knob has no effect)
+
+ at lds = internal unnamed_addr addrspace(3) global [256 x i32] poison, align 4
+
+; CHECK-LABEL: @two_loops(
+define void @two_loops(ptr addrspace(1) %out, i32 %n, i32 %m) {
+entry:
+ %cmp = icmp sgt i32 %n, 0
+ br i1 %cmp, label %lds.loop, label %global.preheader
+
+; The LDS loop: gated by the knob.
+;
+; DEFAULT: lds.loop:
+; DEFAULT: %lds.xtraiter = and i32 %n,
+; DEFAULT: lds.loop.epil:
+;
+; NOLOCAL: lds.loop:
+; NOLOCAL-NOT: lds.xtraiter
+; NOLOCAL-NOT: lds.loop.epil
+lds.loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %lds.loop ]
+ %idx = zext i32 %iv to i64
+ %ptr = getelementptr inbounds [256 x i32], ptr addrspace(3) @lds, i64 0, i64 %idx
+ store i32 %iv, ptr addrspace(3) %ptr, align 4
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %global.preheader, label %lds.loop
+
+global.preheader:
+ %cmp2 = icmp sgt i32 %m, 0
+ br i1 %cmp2, label %global.loop, label %exit
+
+; The global-memory loop: never touches LDS, so the knob must not affect it.
+; It stays runtime-unrolled under both knob settings.
+;
+; DEFAULT: global.loop:
+; DEFAULT: %global.xtraiter = and i32 %m,
+; DEFAULT: global.loop.epil:
+;
+; NOLOCAL: global.loop:
+; NOLOCAL: %global.xtraiter = and i32 %m,
+; NOLOCAL: global.loop.epil:
+global.loop:
+ %jv = phi i32 [ 0, %global.preheader ], [ %jv.next, %global.loop ]
+ %jdx = zext i32 %jv to i64
+ %gptr = getelementptr inbounds i32, ptr addrspace(1) %out, i64 %jdx
+ store i32 %jv, ptr addrspace(1) %gptr, align 4
+ %jv.next = add nuw nsw i32 %jv, 1
+ %exitcond2 = icmp eq i32 %jv.next, %m
+ br i1 %exitcond2, label %exit, label %global.loop
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
new file mode 100644
index 0000000000000..9b6059b5eacad
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-local-mfma.ll
@@ -0,0 +1,415 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes=loop-unroll \
+; RUN: -amdgpu-unroll-runtime-local=false -S %s | FileCheck %s --check-prefix=NOLOCAL
+;
+; Reduced triton bf16 GEMM K-loop (gfx950): convergent MFMA body reading LDS
+; (addrspace(3) @global_smem). Runtime unrolling here adds a by-2 epilogue that
+; doubles the MFMA body and regresses occupancy. With
+; -amdgpu-unroll-runtime-local off, this LDS K-loop must not be runtime-unrolled.
+;
+; NOLOCAL-LABEL: @_gemm_a16_w16_kernel
+; NOLOCAL-NOT: xtraiter
+; NOLOCAL-NOT: .epil
+; NOLOCAL-NOT: unr-lcssa
+
+ at global_smem = external addrspace(3) global [0 x i8], align 16
+
+define amdgpu_kernel void @_gemm_a16_w16_kernel_BLOCK_SIZE_M_16_BLOCK_SIZE_N_16_BLOCK_SIZE_K_256_GROUP_SIZE_M_1_NUM_KSPLIT_1_SPLITK_BLOCK_SIZE_4096_EVEN_K_1_EVEN_MN_0_cache_modifier_CG_activation_NONE_use_activation_0_ADD_BIAS_0_SKIP_REDUCE_0(ptr addrspace(1) inreg nofree readonly captures(none) %0, ptr addrspace(1) inreg nofree readonly captures(none) %1, ptr addrspace(1) inreg nofree writeonly captures(none) %2, i32 inreg %3, i32 inreg %4, i32 inreg %5, i32 inreg %6, i32 inreg %7, i32 inreg %8, i32 inreg %9, ptr addrspace(1) inreg nofree readnone captures(none) %10, ptr addrspace(1) inreg nofree readnone captures(none) %11) {
+ %13 = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %14 = tail call i32 @llvm.amdgcn.readfirstlane.i32(i32 %13)
+ %15 = lshr i32 %14, 6
+ %16 = and i32 %15, 7
+ %17 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %18 = add i32 %3, 15
+ %19 = sdiv i32 %18, 16
+ %20 = add i32 %4, 15
+ %21 = sdiv i32 %20, 16
+ %22 = mul i32 %21, %19
+ %23 = add i32 %22, 7
+ %24 = sdiv i32 %23, 8
+ %25 = srem i32 %22, 8
+ %26 = icmp eq i32 %25, 0
+ %27 = select i1 %26, i32 8, i32 %25
+ %28 = srem i32 %17, 8
+ %29 = sdiv i32 %17, 8
+ %30 = icmp slt i32 %28, %27
+ br i1 %30, label %31, label %33
+
+31: ; preds = %12
+ %32 = mul nsw i32 %24, %28
+ br label %39
+
+33: ; preds = %12
+ %34 = mul nsw i32 %27, %24
+ %35 = sub nsw i32 %28, %27
+ %36 = add nsw i32 %24, -1
+ %37 = mul i32 %35, %36
+ %38 = add i32 %37, %34
+ br label %39
+
+39: ; preds = %31, %33
+ %.pn = phi i32 [ %38, %33 ], [ %32, %31 ]
+ %40 = add i32 %.pn, %29
+ %41 = sdiv i32 %40, %21
+ %42 = srem i32 %40, %21
+ %43 = icmp sgt i32 %5, 0
+ br i1 %43, label %44, label %333
+
+44: ; preds = %39
+ %45 = shl i32 %41, 4
+ %46 = lshr i32 %13, 2
+ %47 = and i32 %46, 8
+ %48 = or disjoint i32 %16, %47
+ %49 = and i32 %13, 15
+ %50 = and i32 %13, 48
+ %51 = lshr exact i32 %50, 2
+ %52 = or disjoint i32 %51, 1
+ %53 = or disjoint i32 %51, 2
+ %54 = or disjoint i32 %51, 3
+ %55 = or disjoint i32 %45, %48
+ %56 = srem i32 %55, %3
+ %57 = shl nsw i32 %42, 4
+ %58 = or disjoint i32 %57, %48
+ %59 = srem i32 %58, %4
+ %60 = shl nuw nsw i32 %13, 3
+ %61 = and i32 %60, 248
+ %62 = mul i32 %56, %6
+ %63 = add i32 %62, %61
+ %64 = mul i32 %59, %7
+ %65 = add i32 %64, %61
+ %66 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %0, i16 0, i64 2147483646, i32 159744)
+ %67 = mul nuw nsw i32 %16, 1056
+ %68 = getelementptr inbounds nuw i8, ptr addrspace(3) @global_smem, i32 %67
+ %69 = shl i32 %63, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %66, ptr addrspace(3) %68, i32 16, i32 %69, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %70 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %1, i16 0, i64 2147483646, i32 159744)
+ %71 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), i32 %67
+ %72 = shl i32 %65, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %70, ptr addrspace(3) %71, i32 16, i32 %72, i32 0, i32 0, i32 3)
+ tail call void @llvm.amdgcn.asyncmark()
+ %73 = icmp samesign ugt i32 %5, 256
+ %74 = add i32 %63, 256
+ %75 = add i32 %65, 256
+ %76 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 8448), i32 %67
+ %77 = shl i32 %74, 1
+ %78 = select i1 %73, i32 %77, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %66, ptr addrspace(3) %76, i32 16, i32 %78, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %79 = getelementptr inbounds nuw i8, ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 33760), i32 %67
+ %80 = shl i32 %75, 1
+ %81 = select i1 %73, i32 %80, i32 -2147483648
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %70, ptr addrspace(3) %79, i32 16, i32 %81, i32 0, i32 0, i32 3)
+ tail call void @llvm.amdgcn.asyncmark()
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ fence syncscope("workgroup") release
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire
+ %82 = icmp samesign ugt i32 %5, 512
+ %83 = shl nuw nsw i32 %13, 10
+ %84 = and i32 %83, 7168
+ %85 = shl nuw nsw i32 %13, 6
+ %86 = and i32 %85, 512
+ %87 = or disjoint i32 %86, %50
+ %88 = or disjoint i32 %87, %84
+ %89 = lshr exact i32 %84, 5
+ %90 = add nuw nsw i32 %88, %89
+ %91 = add nuw nsw i32 %90, 64
+ %92 = add nuw nsw i32 %90, 128
+ %93 = add nuw nsw i32 %90, 192
+ %94 = add nuw nsw i32 %90, 256
+ %95 = add nuw nsw i32 %90, 320
+ %96 = add nuw nsw i32 %90, 384
+ %97 = add nuw nsw i32 %90, 448
+ br i1 %82, label %.lr.ph, label %._crit_edge
+
+.lr.ph: ; preds = %44
+ %98 = tail call i32 @llvm.umin.i32(i32 %5, i32 4096)
+ %99 = add nuw nsw i32 %98, 255
+ %100 = lshr i32 %99, 8
+ %101 = add nsw i32 %100, -3
+ br label %102
+
+102: ; preds = %.lr.ph, %102
+ %.pn3555 = phi i32 [ %75, %.lr.ph ], [ %114, %102 ]
+ %.pn1954 = phi i32 [ %74, %.lr.ph ], [ %113, %102 ]
+ %103 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 33760), %.lr.ph ], [ %140, %102 ]
+ %104 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), %.lr.ph ], [ %103, %102 ]
+ %105 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 8448), %.lr.ph ], [ %121, %102 ]
+ %106 = phi ptr addrspace(3) [ @global_smem, %.lr.ph ], [ %105, %102 ]
+ %107 = phi i32 [ 1, %.lr.ph ], [ %117, %102 ]
+ %108 = phi float [ 0.000000e+00, %.lr.ph ], [ %171, %102 ]
+ %109 = phi float [ 0.000000e+00, %.lr.ph ], [ %172, %102 ]
+ %110 = phi float [ 0.000000e+00, %.lr.ph ], [ %173, %102 ]
+ %111 = phi float [ 0.000000e+00, %.lr.ph ], [ %174, %102 ]
+ %112 = phi i32 [ 0, %.lr.ph ], [ %175, %102 ]
+ %113 = add i32 %.pn1954, 256
+ %114 = add i32 %.pn3555, 256
+ %115 = add i32 %107, 1
+ %116 = icmp slt i32 %115, 3
+ %117 = select i1 %116, i32 %115, i32 0
+ %118 = shl i32 %117, 12
+ %119 = lshr exact i32 %118, 5
+ %120 = add i32 %119, %118
+ %121 = getelementptr [2 x i8], ptr addrspace(3) @global_smem, i32 %120
+ %122 = getelementptr inbounds nuw i8, ptr addrspace(3) %121, i32 %67
+ %123 = shl i32 %113, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %66, ptr addrspace(3) %122, i32 16, i32 %123, i32 0, i32 0, i32 0)
+ tail call void @llvm.amdgcn.asyncmark()
+ %124 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %90
+ %125 = load <8 x bfloat>, ptr addrspace(3) %124, align 16
+ %126 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %91
+ %127 = load <8 x bfloat>, ptr addrspace(3) %126, align 16
+ %128 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %92
+ %129 = load <8 x bfloat>, ptr addrspace(3) %128, align 16
+ %130 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %93
+ %131 = load <8 x bfloat>, ptr addrspace(3) %130, align 16
+ %132 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %94
+ %133 = load <8 x bfloat>, ptr addrspace(3) %132, align 16
+ %134 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %95
+ %135 = load <8 x bfloat>, ptr addrspace(3) %134, align 16
+ %136 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %96
+ %137 = load <8 x bfloat>, ptr addrspace(3) %136, align 16
+ %138 = getelementptr inbounds nuw i8, ptr addrspace(3) %106, i32 %97
+ %139 = load <8 x bfloat>, ptr addrspace(3) %138, align 16
+ %140 = getelementptr [2 x i8], ptr addrspace(3) getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), i32 %120
+ %141 = getelementptr inbounds nuw i8, ptr addrspace(3) %140, i32 %67
+ %142 = shl i32 %114, 1
+ tail call void @llvm.amdgcn.raw.ptr.buffer.load.async.lds(ptr addrspace(8) %70, ptr addrspace(3) %141, i32 16, i32 %142, i32 0, i32 0, i32 3)
+ tail call void @llvm.amdgcn.asyncmark()
+ %143 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %90
+ %144 = load <8 x bfloat>, ptr addrspace(3) %143, align 16
+ %145 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %91
+ %146 = load <8 x bfloat>, ptr addrspace(3) %145, align 16
+ %147 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %92
+ %148 = load <8 x bfloat>, ptr addrspace(3) %147, align 16
+ %149 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %93
+ %150 = load <8 x bfloat>, ptr addrspace(3) %149, align 16
+ %151 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %94
+ %152 = load <8 x bfloat>, ptr addrspace(3) %151, align 16
+ %153 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %95
+ %154 = load <8 x bfloat>, ptr addrspace(3) %153, align 16
+ %155 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %96
+ %156 = load <8 x bfloat>, ptr addrspace(3) %155, align 16
+ %157 = getelementptr inbounds nuw i8, ptr addrspace(3) %104, i32 %97
+ %158 = load <8 x bfloat>, ptr addrspace(3) %157, align 16
+ %159 = insertelement <4 x float> poison, float %108, i64 0
+ %160 = insertelement <4 x float> %159, float %109, i64 1
+ %161 = insertelement <4 x float> %160, float %110, i64 2
+ %162 = insertelement <4 x float> %161, float %111, i64 3
+ %163 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %144, <8 x bfloat> %125, <4 x float> %162, i32 0, i32 0, i32 0)
+ %164 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %146, <8 x bfloat> %127, <4 x float> %163, i32 0, i32 0, i32 0)
+ %165 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %148, <8 x bfloat> %129, <4 x float> %164, i32 0, i32 0, i32 0)
+ %166 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %150, <8 x bfloat> %131, <4 x float> %165, i32 0, i32 0, i32 0)
+ %167 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %152, <8 x bfloat> %133, <4 x float> %166, i32 0, i32 0, i32 0)
+ %168 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %154, <8 x bfloat> %135, <4 x float> %167, i32 0, i32 0, i32 0)
+ %169 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %156, <8 x bfloat> %137, <4 x float> %168, i32 0, i32 0, i32 0)
+ %170 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %158, <8 x bfloat> %139, <4 x float> %169, i32 0, i32 0, i32 0)
+ %171 = extractelement <4 x float> %170, i64 0
+ %172 = extractelement <4 x float> %170, i64 1
+ %173 = extractelement <4 x float> %170, i64 2
+ %174 = extractelement <4 x float> %170, i64 3
+ tail call void @llvm.amdgcn.wait.asyncmark(i16 2)
+ fence syncscope("workgroup") release
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire
+ %175 = add nuw nsw i32 %112, 1
+ %exitcond.not = icmp eq i32 %112, %101
+ br i1 %exitcond.not, label %._crit_edge, label %102
+
+._crit_edge: ; preds = %102, %44
+ %176 = phi float [ 0.000000e+00, %44 ], [ %171, %102 ]
+ %177 = phi float [ 0.000000e+00, %44 ], [ %172, %102 ]
+ %178 = phi float [ 0.000000e+00, %44 ], [ %173, %102 ]
+ %179 = phi float [ 0.000000e+00, %44 ], [ %174, %102 ]
+ %.lcssa52 = phi ptr addrspace(3) [ @global_smem, %44 ], [ %105, %102 ]
+ %.lcssa51 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 8448), %44 ], [ %121, %102 ]
+ %180 = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 25312), %44 ], [ %103, %102 ]
+ %.lcssa = phi ptr addrspace(3) [ getelementptr (i8, ptr addrspace(3) @global_smem, i32 33760), %44 ], [ %140, %102 ]
+ %181 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %97
+ %182 = load <8 x bfloat>, ptr addrspace(3) %181, align 16
+ %183 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %96
+ %184 = load <8 x bfloat>, ptr addrspace(3) %183, align 16
+ %185 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %95
+ %186 = load <8 x bfloat>, ptr addrspace(3) %185, align 16
+ %187 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %94
+ %188 = load <8 x bfloat>, ptr addrspace(3) %187, align 16
+ %189 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %93
+ %190 = load <8 x bfloat>, ptr addrspace(3) %189, align 16
+ %191 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %92
+ %192 = load <8 x bfloat>, ptr addrspace(3) %191, align 16
+ %193 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %91
+ %194 = load <8 x bfloat>, ptr addrspace(3) %193, align 16
+ %195 = getelementptr inbounds nuw i8, ptr addrspace(3) %180, i32 %90
+ %196 = load <8 x bfloat>, ptr addrspace(3) %195, align 16
+ %197 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %97
+ %198 = load <8 x bfloat>, ptr addrspace(3) %197, align 16
+ %199 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %96
+ %200 = load <8 x bfloat>, ptr addrspace(3) %199, align 16
+ %201 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %95
+ %202 = load <8 x bfloat>, ptr addrspace(3) %201, align 16
+ %203 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %94
+ %204 = load <8 x bfloat>, ptr addrspace(3) %203, align 16
+ %205 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %93
+ %206 = load <8 x bfloat>, ptr addrspace(3) %205, align 16
+ %207 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %92
+ %208 = load <8 x bfloat>, ptr addrspace(3) %207, align 16
+ %209 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %91
+ %210 = load <8 x bfloat>, ptr addrspace(3) %209, align 16
+ %211 = getelementptr inbounds nuw i8, ptr addrspace(3) %.lcssa52, i32 %90
+ %212 = load <8 x bfloat>, ptr addrspace(3) %211, align 16
+ %213 = insertelement <4 x float> poison, float %176, i64 0
+ %214 = insertelement <4 x float> %213, float %177, i64 1
+ %215 = insertelement <4 x float> %214, float %178, i64 2
+ %216 = insertelement <4 x float> %215, float %179, i64 3
+ %217 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %196, <8 x bfloat> %212, <4 x float> %216, i32 0, i32 0, i32 0)
+ %218 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %194, <8 x bfloat> %210, <4 x float> %217, i32 0, i32 0, i32 0)
+ %219 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %192, <8 x bfloat> %208, <4 x float> %218, i32 0, i32 0, i32 0)
+ %220 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %190, <8 x bfloat> %206, <4 x float> %219, i32 0, i32 0, i32 0)
+ %221 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %188, <8 x bfloat> %204, <4 x float> %220, i32 0, i32 0, i32 0)
+ %222 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %186, <8 x bfloat> %202, <4 x float> %221, i32 0, i32 0, i32 0)
+ %223 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %184, <8 x bfloat> %200, <4 x float> %222, i32 0, i32 0, i32 0)
+ %224 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %182, <8 x bfloat> %198, <4 x float> %223, i32 0, i32 0, i32 0)
+ %225 = extractelem...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/222853
More information about the llvm-commits
mailing list