[llvm] Reenable sched mfma rewrite (PR #180751)
Tony Linthicum via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 11:52:00 PST 2026
https://github.com/tlinthic updated https://github.com/llvm/llvm-project/pull/180751
>From 9affa75ce7ddcecb9ff0f6b86c6cafbec64f8e0f Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Mon, 26 Jan 2026 09:35:17 -0600
Subject: [PATCH 01/17] Turn mfma rewrite in scheduler back on by default
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b0441955e70b9..3dbace757add2 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(true));
+ cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
const unsigned ScheduleMetrics::ScaleFactor = 100;
>From 8b882a10d8a5fda232e91febbf20a11ec778772c Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Mon, 9 Feb 2026 08:55:48 -0600
Subject: [PATCH 02/17] Revert "Turn mfma rewrite in scheduler back on by
default"
This reverts commit fe5cc69dfdc58dc941ab3d2e9cd6bcf6a358cb90.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 3dbace757add2..b0441955e70b9 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
+ cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(true));
const unsigned ScheduleMetrics::ScaleFactor = 100;
>From b81fd46488e022f5c41b4d153067eecda5304585 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:12:31 -0600
Subject: [PATCH 03/17] Fix for issue #177696
Fixes problem with setting/resetting register types in the MFMA rewrite
sched stage.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 40 +-
.../AMDGPU/sched_mfma_rewrite_reject.ll | 1867 +++++++++++++++++
2 files changed, 1894 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b0441955e70b9..12526862927ed 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,7 +2184,6 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
-
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
@@ -2233,11 +2232,17 @@ bool RewriteMFMAFormStage::initHeuristics(
}
// Do the rewrite to allow for updated RP calculation.
- const TargetRegisterClass *VGPRRC = DAG.MRI.getRegClass(Dst.getReg());
- const TargetRegisterClass *AGPRRC = SRI->getEquivalentAGPRClass(VGPRRC);
- DAG.MRI.setRegClass(Dst.getReg(), AGPRRC);
- if (Src2->isReg())
- DAG.MRI.setRegClass(Src2->getReg(), AGPRRC);
+ const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
+ const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
+ DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
+ if (Src2->isReg()) {
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *AUseRC = SRI->getEquivalentVGPRClass(VUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
+ }
Changed = true;
}
}
@@ -2298,8 +2303,11 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
SpillCost *= (int64_t)RelativeFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0)
+ if (SpillCost > 0) {
+ LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite rejected.\n");
return SpillCost;
+ }
+ LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite successful.\n");
if (SpillCost < BestSpillCost)
BestSpillCost = SpillCost;
@@ -2342,17 +2350,23 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
// rewrite then these need to be restored anyway.
for (auto &[MI, OriginalOpcode] : RewriteCands) {
assert(TII->isMAI(*MI));
- const TargetRegisterClass *AGPRRC =
+ const TargetRegisterClass *ADefRC =
DAG.MRI.getRegClass(MI->getOperand(0).getReg());
- const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(AGPRRC);
+ const TargetRegisterClass *VDefRC = SRI->getEquivalentVGPRClass(ADefRC);
+ DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VDefRC);
+ MI->setDesc(TII->get(OriginalOpcode));
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
+ if (!Src2->isReg())
+ continue;
- if (Src2->isReg())
- DAG.MRI.setRegClass(Src2->getReg(), VGPRRC);
- DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VGPRRC);
- MI->setDesc(TII->get(OriginalOpcode));
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *AUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *VUseRC = SRI->getEquivalentVGPRClass(AUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), VUseRC);
}
return Cost + CopyCost;
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
new file mode 100644
index 0000000000000..c8f444c175be8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -0,0 +1,1867 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+
+ at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] undef, align 16
+ at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] undef, align 16
+
+; Function Attrs: alwaysinline nofree norecurse nounwind
+define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %0, ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %1, ptr addrspace(1) inreg noalias noundef nonnull align 16 %2) local_unnamed_addr #0 !reqd_work_group_size !2 {
+; This should be rejected for MFMA rewrite by the scheduler so we should see
+; no AGPR operands.
+; CHECK-NOT: v_mfma_f32_16x16x16_bf16 a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}]
+ %4 = tail call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+ call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %0, i64 64) ]
+ %5 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %0, i16 0, i64 5564160, i32 159744)
+ call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %1, i64 64) ]
+ %6 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %1, i16 0, i64 5564160, i32 159744)
+ call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %2, i64 64) ]
+ %7 = lshr i32 %4, 6
+ %8 = and i32 %7, 1
+ %9 = and i32 %4, 15
+ %10 = lshr i32 %4, 2
+ %11 = and i32 %10, 12
+ %12 = lshr i32 %4, 3
+ %13 = or disjoint i32 %4, 256
+ %14 = lshr i32 %13, 3
+ %15 = or disjoint i32 %4, 512
+ %16 = lshr i32 %15, 3
+ %17 = or disjoint i32 %4, 768
+ %18 = lshr i32 %17, 3
+ %19 = lshr i32 %4, 4
+ %20 = lshr i32 %13, 4
+ %21 = lshr i32 %15, 4
+ %22 = lshr i32 %17, 4
+ %23 = or disjoint i32 %19, 64
+ %24 = or disjoint i32 %19, 80
+ %25 = or disjoint i32 %19, 96
+ %26 = or disjoint i32 %19, 112
+ %27 = shl nuw nsw i32 %4, 3
+ %28 = and i32 %27, 56
+ %29 = and i32 %27, 120
+ %30 = tail call range(i32 0, 512) i32 @llvm.amdgcn.workgroup.id.x()
+ %31 = shl nuw nsw i32 %30, 6
+ %32 = and i32 %31, 1984
+ %33 = shl nuw nsw i32 %30, 2
+ %34 = and i32 %33, 1920
+ %35 = sub nuw nsw i32 2016, %34
+ %36 = tail call i32 @llvm.umin.i32(i32 %35, i32 128)
+ %37 = sub nuw nsw i32 2016, %32
+ %38 = tail call i32 @llvm.umin.i32(i32 %37, i32 64)
+ %39 = tail call i32 @llvm.umin.i32(i32 %38, i32 %28)
+ %40 = sub nsw i32 %38, %39
+ %41 = icmp sgt i32 %40, 7
+ %42 = tail call i32 @llvm.umin.i32(i32 %36, i32 %29)
+ %43 = sub nsw i32 %36, %42
+ %44 = icmp sgt i32 %43, 7
+ %45 = or disjoint i32 %39, %32
+ %.idx = mul nuw nsw i32 %12, 4032
+ %46 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx
+ %47 = getelementptr bfloat, ptr addrspace(7) %46, i32 %45
+ %48 = load <8 x bfloat>, ptr addrspace(7) %47, align 2
+ %.idx4 = mul nuw nsw i32 %14, 4032
+ %49 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx4
+ %50 = getelementptr bfloat, ptr addrspace(7) %49, i32 %45
+ %51 = load <8 x bfloat>, ptr addrspace(7) %50, align 2
+ %.idx6 = mul nuw nsw i32 %16, 4032
+ %52 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx6
+ %53 = getelementptr bfloat, ptr addrspace(7) %52, i32 %45
+ %54 = load <8 x bfloat>, ptr addrspace(7) %53, align 2
+ %.idx8 = mul nuw nsw i32 %18, 4032
+ %55 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx8
+ %56 = getelementptr bfloat, ptr addrspace(7) %55, i32 %45
+ %57 = load <8 x bfloat>, ptr addrspace(7) %56, align 2
+ %58 = or disjoint i32 %42, %34
+ %.idx10 = mul nuw nsw i32 %19, 4032
+ %59 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx10
+ %60 = getelementptr bfloat, ptr addrspace(7) %59, i32 %58
+ %61 = load <8 x bfloat>, ptr addrspace(7) %60, align 2
+ %.idx12 = mul nuw nsw i32 %20, 4032
+ %62 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx12
+ %63 = getelementptr bfloat, ptr addrspace(7) %62, i32 %58
+ %64 = load <8 x bfloat>, ptr addrspace(7) %63, align 2
+ %.idx14 = mul nuw nsw i32 %21, 4032
+ %65 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx14
+ %66 = getelementptr bfloat, ptr addrspace(7) %65, i32 %58
+ %67 = load <8 x bfloat>, ptr addrspace(7) %66, align 2
+ %.idx16 = mul nuw nsw i32 %22, 4032
+ %68 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx16
+ %69 = getelementptr bfloat, ptr addrspace(7) %68, i32 %58
+ %70 = load <8 x bfloat>, ptr addrspace(7) %69, align 2
+ %.idx18 = mul nuw nsw i32 %23, 4032
+ %71 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx18
+ %72 = getelementptr bfloat, ptr addrspace(7) %71, i32 %58
+ %73 = load <8 x bfloat>, ptr addrspace(7) %72, align 2
+ %.idx20 = mul nuw nsw i32 %24, 4032
+ %74 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx20
+ %75 = getelementptr bfloat, ptr addrspace(7) %74, i32 %58
+ %76 = load <8 x bfloat>, ptr addrspace(7) %75, align 2
+ %.idx22 = mul nuw nsw i32 %25, 4032
+ %77 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx22
+ %78 = getelementptr bfloat, ptr addrspace(7) %77, i32 %58
+ %79 = load <8 x bfloat>, ptr addrspace(7) %78, align 2
+ %.idx24 = mul nuw nsw i32 %26, 4032
+ %80 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx24
+ %81 = getelementptr bfloat, ptr addrspace(7) %80, i32 %58
+ %82 = load <8 x bfloat>, ptr addrspace(7) %81, align 2
+ %83 = select i1 %41, <8 x bfloat> %48, <8 x bfloat> zeroinitializer
+ %.idx26 = mul nuw nsw i32 %12, 136
+ %84 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx26
+ %85 = getelementptr bfloat, ptr addrspace(3) %84, i32 %28
+ store <8 x bfloat> %83, ptr addrspace(3) %85, align 8
+ %86 = select i1 %41, <8 x bfloat> %51, <8 x bfloat> zeroinitializer
+ %.idx28 = mul nuw nsw i32 %14, 136
+ %87 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx28
+ %88 = getelementptr bfloat, ptr addrspace(3) %87, i32 %28
+ store <8 x bfloat> %86, ptr addrspace(3) %88, align 8
+ %89 = select i1 %41, <8 x bfloat> %54, <8 x bfloat> zeroinitializer
+ %.idx30 = mul nuw nsw i32 %16, 136
+ %90 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx30
+ %91 = getelementptr bfloat, ptr addrspace(3) %90, i32 %28
+ store <8 x bfloat> %89, ptr addrspace(3) %91, align 8
+ %92 = select i1 %41, <8 x bfloat> %57, <8 x bfloat> zeroinitializer
+ %.idx32 = mul nuw nsw i32 %18, 136
+ %93 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx32
+ %94 = getelementptr bfloat, ptr addrspace(3) %93, i32 %28
+ store <8 x bfloat> %92, ptr addrspace(3) %94, align 8
+ %95 = select i1 %44, <8 x bfloat> %61, <8 x bfloat> zeroinitializer
+ %.idx34 = mul nuw nsw i32 %19, 264
+ %96 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx34
+ %97 = getelementptr bfloat, ptr addrspace(3) %96, i32 %29
+ store <8 x bfloat> %95, ptr addrspace(3) %97, align 8
+ %98 = select i1 %44, <8 x bfloat> %64, <8 x bfloat> zeroinitializer
+ %.idx36 = mul nuw nsw i32 %20, 264
+ %99 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx36
+ %100 = getelementptr bfloat, ptr addrspace(3) %99, i32 %29
+ store <8 x bfloat> %98, ptr addrspace(3) %100, align 8
+ %101 = select i1 %44, <8 x bfloat> %67, <8 x bfloat> zeroinitializer
+ %.idx38 = mul nuw nsw i32 %21, 264
+ %102 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx38
+ %103 = getelementptr bfloat, ptr addrspace(3) %102, i32 %29
+ store <8 x bfloat> %101, ptr addrspace(3) %103, align 8
+ %104 = select i1 %44, <8 x bfloat> %70, <8 x bfloat> zeroinitializer
+ %.idx40 = mul nuw nsw i32 %22, 264
+ %105 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx40
+ %106 = getelementptr bfloat, ptr addrspace(3) %105, i32 %29
+ store <8 x bfloat> %104, ptr addrspace(3) %106, align 8
+ %107 = select i1 %44, <8 x bfloat> %73, <8 x bfloat> zeroinitializer
+ %.idx42 = mul nuw nsw i32 %23, 264
+ %108 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx42
+ %109 = getelementptr bfloat, ptr addrspace(3) %108, i32 %29
+ store <8 x bfloat> %107, ptr addrspace(3) %109, align 8
+ %110 = select i1 %44, <8 x bfloat> %76, <8 x bfloat> zeroinitializer
+ %.idx44 = mul nuw nsw i32 %24, 264
+ %111 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx44
+ %112 = getelementptr bfloat, ptr addrspace(3) %111, i32 %29
+ store <8 x bfloat> %110, ptr addrspace(3) %112, align 8
+ %113 = select i1 %44, <8 x bfloat> %79, <8 x bfloat> zeroinitializer
+ %.idx46 = mul nuw nsw i32 %25, 264
+ %114 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx46
+ %115 = getelementptr bfloat, ptr addrspace(3) %114, i32 %29
+ store <8 x bfloat> %113, ptr addrspace(3) %115, align 8
+ %116 = select i1 %44, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
+ %.idx48 = mul nuw nsw i32 %26, 264
+ %117 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx48
+ %118 = getelementptr bfloat, ptr addrspace(3) %117, i32 %29
+ store <8 x bfloat> %116, ptr addrspace(3) %118, align 8
+ %119 = shl nuw nsw i32 %8, 5
+ %120 = or disjoint i32 %119, %9
+ %121 = or disjoint i32 %11, 1
+ %122 = or disjoint i32 %11, 2
+ %123 = or disjoint i32 %11, 3
+ %124 = or disjoint i32 %11, 16
+ %125 = or disjoint i32 %11, 17
+ %126 = or disjoint i32 %11, 18
+ %127 = or disjoint i32 %11, 19
+ %128 = or disjoint i32 %11, 32
+ %129 = or disjoint i32 %11, 33
+ %130 = or disjoint i32 %11, 34
+ %131 = or disjoint i32 %11, 35
+ %132 = or disjoint i32 %11, 48
+ %133 = or disjoint i32 %11, 49
+ %134 = or disjoint i32 %11, 50
+ %135 = or i32 %10, 51
+ %136 = or disjoint i32 %11, 64
+ %137 = or disjoint i32 %11, 65
+ %138 = or disjoint i32 %11, 66
+ %139 = or disjoint i32 %11, 67
+ %140 = or disjoint i32 %11, 80
+ %141 = or disjoint i32 %11, 81
+ %142 = or disjoint i32 %11, 82
+ %143 = or disjoint i32 %11, 83
+ %144 = or disjoint i32 %11, 96
+ %145 = or disjoint i32 %11, 97
+ %146 = or disjoint i32 %11, 98
+ %147 = or disjoint i32 %11, 99
+ %148 = or disjoint i32 %11, 112
+ %149 = or disjoint i32 %11, 113
+ %150 = or disjoint i32 %11, 114
+ %151 = or i32 %10, 115
+ %152 = lshr i32 %4, 1
+ %153 = and i32 %152, 64
+ %154 = or disjoint i32 %153, %9
+ %invariant.gep = getelementptr bfloat, ptr addrspace(7) %5, i32 %45
+ %invariant.gep111 = getelementptr bfloat, ptr addrspace(7) %6, i32 %58
+ %155 = mul nuw nsw i32 %11, 68
+ %156 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
+ %157 = getelementptr bfloat, ptr addrspace(3) %156, i32 %155
+ %158 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
+ %159 = getelementptr i8, ptr addrspace(3) %158, i32 32
+ %160 = getelementptr bfloat, ptr addrspace(3) %159, i32 %155
+ %161 = mul nuw nsw i32 %121, 68
+ %162 = getelementptr bfloat, ptr addrspace(3) %156, i32 %161
+ %163 = getelementptr bfloat, ptr addrspace(3) %159, i32 %161
+ %164 = mul nuw nsw i32 %122, 68
+ %165 = getelementptr bfloat, ptr addrspace(3) %156, i32 %164
+ %166 = getelementptr bfloat, ptr addrspace(3) %159, i32 %164
+ %167 = mul nuw nsw i32 %123, 68
+ %168 = getelementptr bfloat, ptr addrspace(3) %156, i32 %167
+ %169 = getelementptr bfloat, ptr addrspace(3) %159, i32 %167
+ %170 = mul nuw nsw i32 %124, 68
+ %171 = getelementptr bfloat, ptr addrspace(3) %156, i32 %170
+ %172 = getelementptr bfloat, ptr addrspace(3) %159, i32 %170
+ %173 = mul nuw nsw i32 %125, 68
+ %174 = getelementptr bfloat, ptr addrspace(3) %156, i32 %173
+ %175 = getelementptr bfloat, ptr addrspace(3) %159, i32 %173
+ %176 = mul nuw nsw i32 %126, 68
+ %177 = getelementptr bfloat, ptr addrspace(3) %156, i32 %176
+ %178 = getelementptr bfloat, ptr addrspace(3) %159, i32 %176
+ %179 = mul nuw nsw i32 %127, 68
+ %180 = getelementptr bfloat, ptr addrspace(3) %156, i32 %179
+ %181 = getelementptr bfloat, ptr addrspace(3) %159, i32 %179
+ %182 = mul nuw nsw i32 %128, 68
+ %183 = getelementptr bfloat, ptr addrspace(3) %156, i32 %182
+ %184 = getelementptr bfloat, ptr addrspace(3) %159, i32 %182
+ %185 = mul nuw nsw i32 %129, 68
+ %186 = getelementptr bfloat, ptr addrspace(3) %156, i32 %185
+ %187 = getelementptr bfloat, ptr addrspace(3) %159, i32 %185
+ %188 = mul nuw nsw i32 %130, 68
+ %189 = getelementptr bfloat, ptr addrspace(3) %156, i32 %188
+ %190 = getelementptr bfloat, ptr addrspace(3) %159, i32 %188
+ %191 = mul nuw nsw i32 %131, 68
+ %192 = getelementptr bfloat, ptr addrspace(3) %156, i32 %191
+ %193 = getelementptr bfloat, ptr addrspace(3) %159, i32 %191
+ %194 = mul nuw nsw i32 %132, 68
+ %195 = getelementptr bfloat, ptr addrspace(3) %156, i32 %194
+ %196 = getelementptr bfloat, ptr addrspace(3) %159, i32 %194
+ %197 = mul nuw nsw i32 %133, 68
+ %198 = getelementptr bfloat, ptr addrspace(3) %156, i32 %197
+ %199 = getelementptr bfloat, ptr addrspace(3) %159, i32 %197
+ %200 = mul nuw nsw i32 %134, 68
+ %201 = getelementptr bfloat, ptr addrspace(3) %156, i32 %200
+ %202 = getelementptr bfloat, ptr addrspace(3) %159, i32 %200
+ %203 = mul nuw nsw i32 %135, 68
+ %204 = getelementptr bfloat, ptr addrspace(3) %156, i32 %203
+ %205 = getelementptr bfloat, ptr addrspace(3) %159, i32 %203
+ %206 = mul nuw nsw i32 %136, 68
+ %207 = getelementptr bfloat, ptr addrspace(3) %156, i32 %206
+ %208 = getelementptr bfloat, ptr addrspace(3) %159, i32 %206
+ %209 = mul nuw nsw i32 %137, 68
+ %210 = getelementptr bfloat, ptr addrspace(3) %156, i32 %209
+ %211 = getelementptr bfloat, ptr addrspace(3) %159, i32 %209
+ %212 = mul nuw nsw i32 %138, 68
+ %213 = getelementptr bfloat, ptr addrspace(3) %156, i32 %212
+ %214 = getelementptr bfloat, ptr addrspace(3) %159, i32 %212
+ %215 = mul nuw nsw i32 %139, 68
+ %216 = getelementptr bfloat, ptr addrspace(3) %156, i32 %215
+ %217 = getelementptr bfloat, ptr addrspace(3) %159, i32 %215
+ %218 = mul nuw nsw i32 %140, 68
+ %219 = getelementptr bfloat, ptr addrspace(3) %156, i32 %218
+ %220 = getelementptr bfloat, ptr addrspace(3) %159, i32 %218
+ %221 = mul nuw nsw i32 %141, 68
+ %222 = getelementptr bfloat, ptr addrspace(3) %156, i32 %221
+ %223 = getelementptr bfloat, ptr addrspace(3) %159, i32 %221
+ %224 = mul nuw nsw i32 %142, 68
+ %225 = getelementptr bfloat, ptr addrspace(3) %156, i32 %224
+ %226 = getelementptr bfloat, ptr addrspace(3) %159, i32 %224
+ %227 = mul nuw nsw i32 %143, 68
+ %228 = getelementptr bfloat, ptr addrspace(3) %156, i32 %227
+ %229 = getelementptr bfloat, ptr addrspace(3) %159, i32 %227
+ %230 = mul nuw nsw i32 %144, 68
+ %231 = getelementptr bfloat, ptr addrspace(3) %156, i32 %230
+ %232 = getelementptr bfloat, ptr addrspace(3) %159, i32 %230
+ %233 = mul nuw nsw i32 %145, 68
+ %234 = getelementptr bfloat, ptr addrspace(3) %156, i32 %233
+ %235 = getelementptr bfloat, ptr addrspace(3) %159, i32 %233
+ %236 = mul nuw nsw i32 %146, 68
+ %237 = getelementptr bfloat, ptr addrspace(3) %156, i32 %236
+ %238 = getelementptr bfloat, ptr addrspace(3) %159, i32 %236
+ %239 = mul nuw nsw i32 %147, 68
+ %240 = getelementptr bfloat, ptr addrspace(3) %156, i32 %239
+ %241 = getelementptr bfloat, ptr addrspace(3) %159, i32 %239
+ %242 = mul nuw nsw i32 %148, 68
+ %243 = getelementptr bfloat, ptr addrspace(3) %156, i32 %242
+ %244 = getelementptr bfloat, ptr addrspace(3) %159, i32 %242
+ %245 = mul nuw nsw i32 %149, 68
+ %246 = getelementptr bfloat, ptr addrspace(3) %156, i32 %245
+ %247 = getelementptr bfloat, ptr addrspace(3) %159, i32 %245
+ %248 = mul nuw nsw i32 %150, 68
+ %249 = getelementptr bfloat, ptr addrspace(3) %156, i32 %248
+ %250 = getelementptr bfloat, ptr addrspace(3) %159, i32 %248
+ %251 = mul nuw nsw i32 %151, 68
+ %252 = getelementptr bfloat, ptr addrspace(3) %156, i32 %251
+ %253 = getelementptr bfloat, ptr addrspace(3) %159, i32 %251
+ %narrow = mul nuw nsw i32 %11, 132
+ %254 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %255 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow
+ %256 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %257 = getelementptr i8, ptr addrspace(3) %256, i32 32
+ %258 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow
+ %259 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %260 = getelementptr i8, ptr addrspace(3) %259, i32 64
+ %261 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow
+ %262 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %263 = getelementptr i8, ptr addrspace(3) %262, i32 96
+ %264 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow
+ %narrow74 = mul nuw nsw i32 %121, 132
+ %265 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow74
+ %266 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow74
+ %267 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow74
+ %268 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow74
+ %narrow75 = mul nuw nsw i32 %122, 132
+ %269 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow75
+ %270 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow75
+ %271 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow75
+ %272 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow75
+ %narrow76 = mul nuw nsw i32 %123, 132
+ %273 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow76
+ %274 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow76
+ %275 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow76
+ %276 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow76
+ %narrow77 = mul nuw nsw i32 %124, 132
+ %277 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow77
+ %278 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow77
+ %279 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow77
+ %280 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow77
+ %narrow78 = mul nuw nsw i32 %125, 132
+ %281 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow78
+ %282 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow78
+ %283 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow78
+ %284 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow78
+ %narrow79 = mul nuw nsw i32 %126, 132
+ %285 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow79
+ %286 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow79
+ %287 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow79
+ %288 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow79
+ %narrow80 = mul nuw nsw i32 %127, 132
+ %289 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow80
+ %290 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow80
+ %291 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow80
+ %292 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow80
+ %narrow81 = mul nuw nsw i32 %128, 132
+ %293 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow81
+ %294 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow81
+ %295 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow81
+ %296 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow81
+ %narrow82 = mul nuw nsw i32 %129, 132
+ %297 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow82
+ %298 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow82
+ %299 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow82
+ %300 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow82
+ %narrow83 = mul nuw nsw i32 %130, 132
+ %301 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow83
+ %302 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow83
+ %303 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow83
+ %304 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow83
+ %narrow84 = mul nuw nsw i32 %131, 132
+ %305 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow84
+ %306 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow84
+ %307 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow84
+ %308 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow84
+ %narrow85 = mul nuw nsw i32 %132, 132
+ %309 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow85
+ %310 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow85
+ %311 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow85
+ %312 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow85
+ %narrow86 = mul nuw nsw i32 %133, 132
+ %313 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow86
+ %314 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow86
+ %315 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow86
+ %316 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow86
+ %narrow87 = mul nuw nsw i32 %134, 132
+ %317 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow87
+ %318 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow87
+ %319 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow87
+ %320 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow87
+ %narrow88 = mul nuw nsw i32 %135, 132
+ %321 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow88
+ %322 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow88
+ %323 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow88
+ %324 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow88
+ %narrow89 = mul nuw nsw i32 %136, 132
+ %325 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow89
+ %326 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow89
+ %327 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow89
+ %328 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow89
+ %narrow90 = mul nuw nsw i32 %137, 132
+ %329 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow90
+ %330 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow90
+ %331 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow90
+ %332 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow90
+ %narrow91 = mul nuw nsw i32 %138, 132
+ %333 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow91
+ %334 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow91
+ %335 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow91
+ %336 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow91
+ %narrow92 = mul nuw nsw i32 %139, 132
+ %337 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow92
+ %338 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow92
+ %339 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow92
+ %340 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow92
+ %narrow93 = mul nuw nsw i32 %140, 132
+ %341 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow93
+ %342 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow93
+ %343 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow93
+ %344 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow93
+ %narrow94 = mul nuw nsw i32 %141, 132
+ %345 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow94
+ %346 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow94
+ %347 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow94
+ %348 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow94
+ %narrow95 = mul nuw nsw i32 %142, 132
+ %349 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow95
+ %350 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow95
+ %351 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow95
+ %352 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow95
+ %narrow96 = mul nuw nsw i32 %143, 132
+ %353 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow96
+ %354 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow96
+ %355 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow96
+ %356 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow96
+ %narrow97 = mul nuw nsw i32 %144, 132
+ %357 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow97
+ %358 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow97
+ %359 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow97
+ %360 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow97
+ %narrow98 = mul nuw nsw i32 %145, 132
+ %361 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow98
+ %362 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow98
+ %363 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow98
+ %364 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow98
+ %narrow99 = mul nuw nsw i32 %146, 132
+ %365 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow99
+ %366 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow99
+ %367 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow99
+ %368 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow99
+ %narrow100 = mul nuw nsw i32 %147, 132
+ %369 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow100
+ %370 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow100
+ %371 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow100
+ %372 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow100
+ %narrow101 = mul nuw nsw i32 %148, 132
+ %373 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow101
+ %374 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow101
+ %375 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow101
+ %376 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow101
+ %narrow102 = mul nuw nsw i32 %149, 132
+ %377 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow102
+ %378 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow102
+ %379 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow102
+ %380 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow102
+ %narrow103 = mul nuw nsw i32 %150, 132
+ %381 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow103
+ %382 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow103
+ %383 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow103
+ %384 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow103
+ %narrow104 = mul nuw nsw i32 %151, 132
+ %385 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow104
+ %386 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow104
+ %387 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow104
+ %388 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow104
+ br label %389
+
+389: ; preds = %3, %389
+ %390 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %3 ], [ %1015, %389 ]
+ %391 = phi i32 [ 0, %3 ], [ %392, %389 ]
+ %392 = add nuw nsw i32 %391, 8
+ %393 = shl nuw nsw i32 %392, 4
+ %394 = sub nuw nsw i32 1380, %393
+ %395 = tail call i32 @llvm.umin.i32(i32 %394, i32 128)
+ %396 = tail call i32 @llvm.umin.i32(i32 %395, i32 %12)
+ %397 = sub nsw i32 %395, %396
+ %398 = tail call i32 @llvm.smin.i32(i32 %397, i32 1)
+ %399 = or disjoint i32 %396, %393
+ %.idx50 = mul nuw nsw i32 %399, 4032
+ %gep = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx50
+ %400 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
+ %401 = tail call i32 @llvm.umin.i32(i32 %395, i32 %14)
+ %402 = sub nsw i32 %395, %401
+ %403 = tail call i32 @llvm.smin.i32(i32 %402, i32 1)
+ %404 = or disjoint i32 %401, %393
+ %.idx52 = mul nuw nsw i32 %404, 4032
+ %gep106 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx52
+ %405 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
+ %406 = tail call i32 @llvm.umin.i32(i32 %395, i32 %16)
+ %407 = sub nsw i32 %395, %406
+ %408 = tail call i32 @llvm.smin.i32(i32 %407, i32 1)
+ %409 = or disjoint i32 %406, %393
+ %.idx54 = mul nuw nsw i32 %409, 4032
+ %gep108 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx54
+ %410 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
+ %411 = tail call i32 @llvm.umin.i32(i32 %395, i32 %18)
+ %412 = sub nsw i32 %395, %411
+ %413 = tail call i32 @llvm.smin.i32(i32 %412, i32 1)
+ %414 = or disjoint i32 %411, %393
+ %.idx56 = mul nuw nsw i32 %414, 4032
+ %gep110 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx56
+ %415 = load <8 x bfloat>, ptr addrspace(7) %gep110, align 2
+ %416 = tail call i32 @llvm.umin.i32(i32 %395, i32 %19)
+ %417 = sub nsw i32 %395, %416
+ %418 = tail call i32 @llvm.smin.i32(i32 %417, i32 1)
+ %419 = or disjoint i32 %416, %393
+ %.idx58 = mul nuw nsw i32 %419, 4032
+ %gep112 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx58
+ %420 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
+ %421 = tail call i32 @llvm.umin.i32(i32 %395, i32 %20)
+ %422 = sub nsw i32 %395, %421
+ %423 = tail call i32 @llvm.smin.i32(i32 %422, i32 1)
+ %424 = or disjoint i32 %421, %393
+ %.idx60 = mul nuw nsw i32 %424, 4032
+ %gep114 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx60
+ %425 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
+ %426 = tail call i32 @llvm.umin.i32(i32 %395, i32 %21)
+ %427 = sub nsw i32 %395, %426
+ %428 = tail call i32 @llvm.smin.i32(i32 %427, i32 1)
+ %429 = or disjoint i32 %426, %393
+ %.idx62 = mul nuw nsw i32 %429, 4032
+ %gep116 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx62
+ %430 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
+ %431 = tail call i32 @llvm.umin.i32(i32 %395, i32 %22)
+ %432 = sub nsw i32 %395, %431
+ %433 = tail call i32 @llvm.smin.i32(i32 %432, i32 1)
+ %434 = or disjoint i32 %431, %393
+ %.idx64 = mul nuw nsw i32 %434, 4032
+ %gep118 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx64
+ %435 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
+ %436 = tail call i32 @llvm.umin.i32(i32 %395, i32 %23)
+ %437 = sub nsw i32 %395, %436
+ %438 = tail call i32 @llvm.smin.i32(i32 %437, i32 1)
+ %439 = or disjoint i32 %436, %393
+ %.idx66 = mul nuw nsw i32 %439, 4032
+ %gep120 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx66
+ %440 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
+ %441 = tail call i32 @llvm.umin.i32(i32 %395, i32 %24)
+ %442 = sub nsw i32 %395, %441
+ %443 = tail call i32 @llvm.smin.i32(i32 %442, i32 1)
+ %444 = or disjoint i32 %441, %393
+ %.idx68 = mul nuw nsw i32 %444, 4032
+ %gep122 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx68
+ %445 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
+ %446 = tail call i32 @llvm.umin.i32(i32 %395, i32 %25)
+ %447 = sub nsw i32 %395, %446
+ %448 = tail call i32 @llvm.smin.i32(i32 %447, i32 1)
+ %449 = or disjoint i32 %446, %393
+ %.idx70 = mul nuw nsw i32 %449, 4032
+ %gep124 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx70
+ %450 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
+ %451 = tail call i32 @llvm.umin.i32(i32 %395, i32 %26)
+ %452 = sub nsw i32 %395, %451
+ %453 = tail call i32 @llvm.smin.i32(i32 %452, i32 1)
+ %454 = or disjoint i32 %451, %393
+ %.idx72 = mul nuw nsw i32 %454, 4032
+ %gep126 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx72
+ %455 = load <8 x bfloat>, ptr addrspace(7) %gep126, align 2
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ %456 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
+ %457 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
+ %458 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
+ %459 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
+ %460 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
+ %461 = shufflevector <1 x bfloat> %460, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %462 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
+ %463 = shufflevector <1 x bfloat> %462, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %464 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
+ %465 = shufflevector <1 x bfloat> %464, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %466 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
+ %467 = shufflevector <1 x bfloat> %466, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %468 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
+ %469 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
+ %470 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
+ %471 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
+ %472 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
+ %473 = shufflevector <1 x bfloat> %472, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %474 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
+ %475 = shufflevector <1 x bfloat> %474, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %476 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
+ %477 = shufflevector <1 x bfloat> %476, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %478 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
+ %479 = shufflevector <1 x bfloat> %478, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %480 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
+ %481 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
+ %482 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
+ %483 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
+ %484 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
+ %485 = shufflevector <1 x bfloat> %484, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %486 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
+ %487 = shufflevector <1 x bfloat> %486, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %488 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
+ %489 = shufflevector <1 x bfloat> %488, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %490 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
+ %491 = shufflevector <1 x bfloat> %490, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %492 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
+ %493 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
+ %494 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
+ %495 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
+ %496 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
+ %497 = shufflevector <1 x bfloat> %496, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %498 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
+ %499 = shufflevector <1 x bfloat> %498, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %500 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
+ %501 = shufflevector <1 x bfloat> %500, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %502 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
+ %503 = shufflevector <1 x bfloat> %502, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %504 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
+ %505 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
+ %506 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
+ %507 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
+ %508 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
+ %509 = shufflevector <1 x bfloat> %508, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %510 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
+ %511 = shufflevector <1 x bfloat> %510, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %512 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
+ %513 = shufflevector <1 x bfloat> %512, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %514 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
+ %515 = shufflevector <1 x bfloat> %514, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %516 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
+ %517 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
+ %518 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
+ %519 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
+ %520 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
+ %521 = shufflevector <1 x bfloat> %520, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %522 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
+ %523 = shufflevector <1 x bfloat> %522, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %524 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
+ %525 = shufflevector <1 x bfloat> %524, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %526 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
+ %527 = shufflevector <1 x bfloat> %526, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %528 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
+ %529 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
+ %530 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
+ %531 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
+ %532 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
+ %533 = shufflevector <1 x bfloat> %532, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %534 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
+ %535 = shufflevector <1 x bfloat> %534, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %536 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
+ %537 = shufflevector <1 x bfloat> %536, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %538 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
+ %539 = shufflevector <1 x bfloat> %538, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %540 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
+ %541 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
+ %542 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
+ %543 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
+ %544 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
+ %545 = shufflevector <1 x bfloat> %544, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %546 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
+ %547 = shufflevector <1 x bfloat> %546, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %548 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
+ %549 = shufflevector <1 x bfloat> %548, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %550 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
+ %551 = shufflevector <1 x bfloat> %550, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %552 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
+ %553 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
+ %554 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
+ %555 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
+ %556 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
+ %557 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
+ %558 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
+ %559 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
+ %560 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
+ %561 = shufflevector <1 x bfloat> %560, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %562 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
+ %563 = shufflevector <1 x bfloat> %562, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %564 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
+ %565 = shufflevector <1 x bfloat> %564, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %566 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
+ %567 = shufflevector <1 x bfloat> %566, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %568 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
+ %569 = shufflevector <1 x bfloat> %568, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %570 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
+ %571 = shufflevector <1 x bfloat> %570, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %572 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
+ %573 = shufflevector <1 x bfloat> %572, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %574 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
+ %575 = shufflevector <1 x bfloat> %574, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %576 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
+ %577 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
+ %578 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
+ %579 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
+ %580 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
+ %581 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
+ %582 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
+ %583 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
+ %584 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
+ %585 = shufflevector <1 x bfloat> %584, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %586 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
+ %587 = shufflevector <1 x bfloat> %586, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %588 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
+ %589 = shufflevector <1 x bfloat> %588, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %590 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
+ %591 = shufflevector <1 x bfloat> %590, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %592 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
+ %593 = shufflevector <1 x bfloat> %592, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %594 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
+ %595 = shufflevector <1 x bfloat> %594, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %596 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
+ %597 = shufflevector <1 x bfloat> %596, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %598 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
+ %599 = shufflevector <1 x bfloat> %598, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %600 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
+ %601 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
+ %602 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
+ %603 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
+ %604 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
+ %605 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
+ %606 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
+ %607 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
+ %608 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
+ %609 = shufflevector <1 x bfloat> %608, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %610 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
+ %611 = shufflevector <1 x bfloat> %610, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %612 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
+ %613 = shufflevector <1 x bfloat> %612, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %614 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
+ %615 = shufflevector <1 x bfloat> %614, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %616 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
+ %617 = shufflevector <1 x bfloat> %616, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %618 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
+ %619 = shufflevector <1 x bfloat> %618, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %620 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
+ %621 = shufflevector <1 x bfloat> %620, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %622 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
+ %623 = shufflevector <1 x bfloat> %622, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %624 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
+ %625 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
+ %626 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
+ %627 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
+ %628 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
+ %629 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
+ %630 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
+ %631 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
+ %632 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
+ %633 = shufflevector <1 x bfloat> %632, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %634 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
+ %635 = shufflevector <1 x bfloat> %634, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %636 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
+ %637 = shufflevector <1 x bfloat> %636, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %638 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
+ %639 = shufflevector <1 x bfloat> %638, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %640 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
+ %641 = shufflevector <1 x bfloat> %640, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %642 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
+ %643 = shufflevector <1 x bfloat> %642, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %644 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
+ %645 = shufflevector <1 x bfloat> %644, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %646 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
+ %647 = shufflevector <1 x bfloat> %646, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %648 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
+ %649 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
+ %650 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
+ %651 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
+ %652 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
+ %653 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
+ %654 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
+ %655 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
+ %656 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
+ %657 = shufflevector <1 x bfloat> %656, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %658 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
+ %659 = shufflevector <1 x bfloat> %658, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %660 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
+ %661 = shufflevector <1 x bfloat> %660, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %662 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
+ %663 = shufflevector <1 x bfloat> %662, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %664 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
+ %665 = shufflevector <1 x bfloat> %664, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %666 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
+ %667 = shufflevector <1 x bfloat> %666, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %668 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
+ %669 = shufflevector <1 x bfloat> %668, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %670 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
+ %671 = shufflevector <1 x bfloat> %670, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %672 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
+ %673 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
+ %674 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
+ %675 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
+ %676 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
+ %677 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
+ %678 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
+ %679 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
+ %680 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
+ %681 = shufflevector <1 x bfloat> %680, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %682 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
+ %683 = shufflevector <1 x bfloat> %682, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %684 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
+ %685 = shufflevector <1 x bfloat> %684, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %686 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
+ %687 = shufflevector <1 x bfloat> %686, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %688 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
+ %689 = shufflevector <1 x bfloat> %688, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %690 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
+ %691 = shufflevector <1 x bfloat> %690, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %692 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
+ %693 = shufflevector <1 x bfloat> %692, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %694 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
+ %695 = shufflevector <1 x bfloat> %694, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %696 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
+ %697 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
+ %698 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
+ %699 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
+ %700 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
+ %701 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
+ %702 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
+ %703 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
+ %704 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
+ %705 = shufflevector <1 x bfloat> %704, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %706 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
+ %707 = shufflevector <1 x bfloat> %706, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %708 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
+ %709 = shufflevector <1 x bfloat> %708, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %710 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
+ %711 = shufflevector <1 x bfloat> %710, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %712 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
+ %713 = shufflevector <1 x bfloat> %712, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %714 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
+ %715 = shufflevector <1 x bfloat> %714, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %716 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
+ %717 = shufflevector <1 x bfloat> %716, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %718 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
+ %719 = shufflevector <1 x bfloat> %718, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %720 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
+ %721 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
+ %722 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
+ %723 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
+ %724 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
+ %725 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
+ %726 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
+ %727 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
+ %728 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
+ %729 = shufflevector <1 x bfloat> %728, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %730 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
+ %731 = shufflevector <1 x bfloat> %730, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %732 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
+ %733 = shufflevector <1 x bfloat> %732, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %734 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
+ %735 = shufflevector <1 x bfloat> %734, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %736 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
+ %737 = shufflevector <1 x bfloat> %736, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %738 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
+ %739 = shufflevector <1 x bfloat> %738, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %740 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
+ %741 = shufflevector <1 x bfloat> %740, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %742 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
+ %743 = shufflevector <1 x bfloat> %742, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %744 = shufflevector <1 x bfloat> %456, <1 x bfloat> %458, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %745 = shufflevector <4 x bfloat> %744, <4 x bfloat> %461, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %746 = shufflevector <4 x bfloat> %745, <4 x bfloat> %465, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %747 = shufflevector <1 x bfloat> %552, <1 x bfloat> %556, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %748 = shufflevector <4 x bfloat> %747, <4 x bfloat> %561, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %749 = shufflevector <4 x bfloat> %748, <4 x bfloat> %569, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %750 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 0, 0
+ %751 = bitcast <4 x bfloat> %746 to <4 x i16>
+ %752 = bitcast <4 x bfloat> %749 to <4 x i16>
+ %753 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %752, <4 x float> %750, i32 0, i32 0, i32 0)
+ %754 = shufflevector <1 x bfloat> %553, <1 x bfloat> %557, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %755 = shufflevector <4 x bfloat> %754, <4 x bfloat> %563, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %756 = shufflevector <4 x bfloat> %755, <4 x bfloat> %571, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %757 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 0, 0
+ %758 = bitcast <4 x bfloat> %756 to <4 x i16>
+ %759 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %758, <4 x float> %757, i32 0, i32 0, i32 0)
+ %760 = shufflevector <1 x bfloat> %554, <1 x bfloat> %558, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %761 = shufflevector <4 x bfloat> %760, <4 x bfloat> %565, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %762 = shufflevector <4 x bfloat> %761, <4 x bfloat> %573, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %763 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 0, 0
+ %764 = bitcast <4 x bfloat> %762 to <4 x i16>
+ %765 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %764, <4 x float> %763, i32 0, i32 0, i32 0)
+ %766 = shufflevector <1 x bfloat> %555, <1 x bfloat> %559, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %767 = shufflevector <4 x bfloat> %766, <4 x bfloat> %567, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %768 = shufflevector <4 x bfloat> %767, <4 x bfloat> %575, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %769 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 0, 0
+ %770 = bitcast <4 x bfloat> %768 to <4 x i16>
+ %771 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %770, <4 x float> %769, i32 0, i32 0, i32 0)
+ %772 = shufflevector <1 x bfloat> %457, <1 x bfloat> %459, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %773 = shufflevector <4 x bfloat> %772, <4 x bfloat> %463, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %774 = shufflevector <4 x bfloat> %773, <4 x bfloat> %467, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %775 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 1, 0
+ %776 = bitcast <4 x bfloat> %774 to <4 x i16>
+ %777 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %752, <4 x float> %775, i32 0, i32 0, i32 0)
+ %778 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 1, 0
+ %779 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %758, <4 x float> %778, i32 0, i32 0, i32 0)
+ %780 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 1, 0
+ %781 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %764, <4 x float> %780, i32 0, i32 0, i32 0)
+ %782 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 1, 0
+ %783 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %770, <4 x float> %782, i32 0, i32 0, i32 0)
+ %784 = shufflevector <1 x bfloat> %468, <1 x bfloat> %470, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %785 = shufflevector <4 x bfloat> %784, <4 x bfloat> %473, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %786 = shufflevector <4 x bfloat> %785, <4 x bfloat> %477, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %787 = shufflevector <1 x bfloat> %576, <1 x bfloat> %580, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %788 = shufflevector <4 x bfloat> %787, <4 x bfloat> %585, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %789 = shufflevector <4 x bfloat> %788, <4 x bfloat> %593, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %790 = bitcast <4 x bfloat> %786 to <4 x i16>
+ %791 = bitcast <4 x bfloat> %789 to <4 x i16>
+ %792 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %791, <4 x float> %753, i32 0, i32 0, i32 0)
+ %793 = shufflevector <1 x bfloat> %577, <1 x bfloat> %581, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %794 = shufflevector <4 x bfloat> %793, <4 x bfloat> %587, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %795 = shufflevector <4 x bfloat> %794, <4 x bfloat> %595, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %796 = bitcast <4 x bfloat> %795 to <4 x i16>
+ %797 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %796, <4 x float> %759, i32 0, i32 0, i32 0)
+ %798 = shufflevector <1 x bfloat> %578, <1 x bfloat> %582, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %799 = shufflevector <4 x bfloat> %798, <4 x bfloat> %589, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %800 = shufflevector <4 x bfloat> %799, <4 x bfloat> %597, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %801 = bitcast <4 x bfloat> %800 to <4 x i16>
+ %802 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %801, <4 x float> %765, i32 0, i32 0, i32 0)
+ %803 = shufflevector <1 x bfloat> %579, <1 x bfloat> %583, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %804 = shufflevector <4 x bfloat> %803, <4 x bfloat> %591, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %805 = shufflevector <4 x bfloat> %804, <4 x bfloat> %599, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %806 = bitcast <4 x bfloat> %805 to <4 x i16>
+ %807 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %806, <4 x float> %771, i32 0, i32 0, i32 0)
+ %808 = shufflevector <1 x bfloat> %469, <1 x bfloat> %471, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %809 = shufflevector <4 x bfloat> %808, <4 x bfloat> %475, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %810 = shufflevector <4 x bfloat> %809, <4 x bfloat> %479, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %811 = bitcast <4 x bfloat> %810 to <4 x i16>
+ %812 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %791, <4 x float> %777, i32 0, i32 0, i32 0)
+ %813 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %796, <4 x float> %779, i32 0, i32 0, i32 0)
+ %814 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %801, <4 x float> %781, i32 0, i32 0, i32 0)
+ %815 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %806, <4 x float> %783, i32 0, i32 0, i32 0)
+ %816 = shufflevector <1 x bfloat> %480, <1 x bfloat> %482, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %817 = shufflevector <4 x bfloat> %816, <4 x bfloat> %485, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %818 = shufflevector <4 x bfloat> %817, <4 x bfloat> %489, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %819 = shufflevector <1 x bfloat> %600, <1 x bfloat> %604, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %820 = shufflevector <4 x bfloat> %819, <4 x bfloat> %609, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %821 = shufflevector <4 x bfloat> %820, <4 x bfloat> %617, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %822 = bitcast <4 x bfloat> %818 to <4 x i16>
+ %823 = bitcast <4 x bfloat> %821 to <4 x i16>
+ %824 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %823, <4 x float> %792, i32 0, i32 0, i32 0)
+ %825 = shufflevector <1 x bfloat> %601, <1 x bfloat> %605, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %826 = shufflevector <4 x bfloat> %825, <4 x bfloat> %611, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %827 = shufflevector <4 x bfloat> %826, <4 x bfloat> %619, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %828 = bitcast <4 x bfloat> %827 to <4 x i16>
+ %829 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %828, <4 x float> %797, i32 0, i32 0, i32 0)
+ %830 = shufflevector <1 x bfloat> %602, <1 x bfloat> %606, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %831 = shufflevector <4 x bfloat> %830, <4 x bfloat> %613, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %832 = shufflevector <4 x bfloat> %831, <4 x bfloat> %621, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %833 = bitcast <4 x bfloat> %832 to <4 x i16>
+ %834 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %833, <4 x float> %802, i32 0, i32 0, i32 0)
+ %835 = shufflevector <1 x bfloat> %603, <1 x bfloat> %607, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %836 = shufflevector <4 x bfloat> %835, <4 x bfloat> %615, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %837 = shufflevector <4 x bfloat> %836, <4 x bfloat> %623, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %838 = bitcast <4 x bfloat> %837 to <4 x i16>
+ %839 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %838, <4 x float> %807, i32 0, i32 0, i32 0)
+ %840 = shufflevector <1 x bfloat> %481, <1 x bfloat> %483, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %841 = shufflevector <4 x bfloat> %840, <4 x bfloat> %487, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %842 = shufflevector <4 x bfloat> %841, <4 x bfloat> %491, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %843 = bitcast <4 x bfloat> %842 to <4 x i16>
+ %844 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %823, <4 x float> %812, i32 0, i32 0, i32 0)
+ %845 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %828, <4 x float> %813, i32 0, i32 0, i32 0)
+ %846 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %833, <4 x float> %814, i32 0, i32 0, i32 0)
+ %847 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %838, <4 x float> %815, i32 0, i32 0, i32 0)
+ %848 = shufflevector <1 x bfloat> %492, <1 x bfloat> %494, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %849 = shufflevector <4 x bfloat> %848, <4 x bfloat> %497, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %850 = shufflevector <4 x bfloat> %849, <4 x bfloat> %501, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %851 = shufflevector <1 x bfloat> %624, <1 x bfloat> %628, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %852 = shufflevector <4 x bfloat> %851, <4 x bfloat> %633, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %853 = shufflevector <4 x bfloat> %852, <4 x bfloat> %641, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %854 = bitcast <4 x bfloat> %850 to <4 x i16>
+ %855 = bitcast <4 x bfloat> %853 to <4 x i16>
+ %856 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %855, <4 x float> %824, i32 0, i32 0, i32 0)
+ %857 = shufflevector <1 x bfloat> %625, <1 x bfloat> %629, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %858 = shufflevector <4 x bfloat> %857, <4 x bfloat> %635, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %859 = shufflevector <4 x bfloat> %858, <4 x bfloat> %643, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %860 = bitcast <4 x bfloat> %859 to <4 x i16>
+ %861 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %860, <4 x float> %829, i32 0, i32 0, i32 0)
+ %862 = shufflevector <1 x bfloat> %626, <1 x bfloat> %630, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %863 = shufflevector <4 x bfloat> %862, <4 x bfloat> %637, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %864 = shufflevector <4 x bfloat> %863, <4 x bfloat> %645, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %865 = bitcast <4 x bfloat> %864 to <4 x i16>
+ %866 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %865, <4 x float> %834, i32 0, i32 0, i32 0)
+ %867 = shufflevector <1 x bfloat> %627, <1 x bfloat> %631, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %868 = shufflevector <4 x bfloat> %867, <4 x bfloat> %639, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %869 = shufflevector <4 x bfloat> %868, <4 x bfloat> %647, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %870 = bitcast <4 x bfloat> %869 to <4 x i16>
+ %871 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %870, <4 x float> %839, i32 0, i32 0, i32 0)
+ %872 = shufflevector <1 x bfloat> %493, <1 x bfloat> %495, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %873 = shufflevector <4 x bfloat> %872, <4 x bfloat> %499, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %874 = shufflevector <4 x bfloat> %873, <4 x bfloat> %503, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %875 = bitcast <4 x bfloat> %874 to <4 x i16>
+ %876 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %855, <4 x float> %844, i32 0, i32 0, i32 0)
+ %877 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %860, <4 x float> %845, i32 0, i32 0, i32 0)
+ %878 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %865, <4 x float> %846, i32 0, i32 0, i32 0)
+ %879 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %870, <4 x float> %847, i32 0, i32 0, i32 0)
+ %880 = shufflevector <1 x bfloat> %504, <1 x bfloat> %506, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %881 = shufflevector <4 x bfloat> %880, <4 x bfloat> %509, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %882 = shufflevector <4 x bfloat> %881, <4 x bfloat> %513, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %883 = shufflevector <1 x bfloat> %648, <1 x bfloat> %652, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %884 = shufflevector <4 x bfloat> %883, <4 x bfloat> %657, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %885 = shufflevector <4 x bfloat> %884, <4 x bfloat> %665, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %886 = bitcast <4 x bfloat> %882 to <4 x i16>
+ %887 = bitcast <4 x bfloat> %885 to <4 x i16>
+ %888 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %887, <4 x float> %856, i32 0, i32 0, i32 0)
+ %889 = shufflevector <1 x bfloat> %649, <1 x bfloat> %653, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %890 = shufflevector <4 x bfloat> %889, <4 x bfloat> %659, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %891 = shufflevector <4 x bfloat> %890, <4 x bfloat> %667, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %892 = bitcast <4 x bfloat> %891 to <4 x i16>
+ %893 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %892, <4 x float> %861, i32 0, i32 0, i32 0)
+ %894 = shufflevector <1 x bfloat> %650, <1 x bfloat> %654, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %895 = shufflevector <4 x bfloat> %894, <4 x bfloat> %661, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %896 = shufflevector <4 x bfloat> %895, <4 x bfloat> %669, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %897 = bitcast <4 x bfloat> %896 to <4 x i16>
+ %898 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %897, <4 x float> %866, i32 0, i32 0, i32 0)
+ %899 = shufflevector <1 x bfloat> %651, <1 x bfloat> %655, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %900 = shufflevector <4 x bfloat> %899, <4 x bfloat> %663, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %901 = shufflevector <4 x bfloat> %900, <4 x bfloat> %671, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %902 = bitcast <4 x bfloat> %901 to <4 x i16>
+ %903 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %902, <4 x float> %871, i32 0, i32 0, i32 0)
+ %904 = shufflevector <1 x bfloat> %505, <1 x bfloat> %507, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %905 = shufflevector <4 x bfloat> %904, <4 x bfloat> %511, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %906 = shufflevector <4 x bfloat> %905, <4 x bfloat> %515, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %907 = bitcast <4 x bfloat> %906 to <4 x i16>
+ %908 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %887, <4 x float> %876, i32 0, i32 0, i32 0)
+ %909 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %892, <4 x float> %877, i32 0, i32 0, i32 0)
+ %910 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %897, <4 x float> %878, i32 0, i32 0, i32 0)
+ %911 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %902, <4 x float> %879, i32 0, i32 0, i32 0)
+ %912 = shufflevector <1 x bfloat> %516, <1 x bfloat> %518, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %913 = shufflevector <4 x bfloat> %912, <4 x bfloat> %521, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %914 = shufflevector <4 x bfloat> %913, <4 x bfloat> %525, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %915 = shufflevector <1 x bfloat> %672, <1 x bfloat> %676, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %916 = shufflevector <4 x bfloat> %915, <4 x bfloat> %681, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %917 = shufflevector <4 x bfloat> %916, <4 x bfloat> %689, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %918 = bitcast <4 x bfloat> %914 to <4 x i16>
+ %919 = bitcast <4 x bfloat> %917 to <4 x i16>
+ %920 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %919, <4 x float> %888, i32 0, i32 0, i32 0)
+ %921 = shufflevector <1 x bfloat> %673, <1 x bfloat> %677, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %922 = shufflevector <4 x bfloat> %921, <4 x bfloat> %683, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %923 = shufflevector <4 x bfloat> %922, <4 x bfloat> %691, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %924 = bitcast <4 x bfloat> %923 to <4 x i16>
+ %925 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %924, <4 x float> %893, i32 0, i32 0, i32 0)
+ %926 = shufflevector <1 x bfloat> %674, <1 x bfloat> %678, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %927 = shufflevector <4 x bfloat> %926, <4 x bfloat> %685, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %928 = shufflevector <4 x bfloat> %927, <4 x bfloat> %693, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %929 = bitcast <4 x bfloat> %928 to <4 x i16>
+ %930 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %929, <4 x float> %898, i32 0, i32 0, i32 0)
+ %931 = shufflevector <1 x bfloat> %675, <1 x bfloat> %679, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %932 = shufflevector <4 x bfloat> %931, <4 x bfloat> %687, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %933 = shufflevector <4 x bfloat> %932, <4 x bfloat> %695, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %934 = bitcast <4 x bfloat> %933 to <4 x i16>
+ %935 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %934, <4 x float> %903, i32 0, i32 0, i32 0)
+ %936 = shufflevector <1 x bfloat> %517, <1 x bfloat> %519, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %937 = shufflevector <4 x bfloat> %936, <4 x bfloat> %523, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %938 = shufflevector <4 x bfloat> %937, <4 x bfloat> %527, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %939 = bitcast <4 x bfloat> %938 to <4 x i16>
+ %940 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %919, <4 x float> %908, i32 0, i32 0, i32 0)
+ %941 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %924, <4 x float> %909, i32 0, i32 0, i32 0)
+ %942 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %929, <4 x float> %910, i32 0, i32 0, i32 0)
+ %943 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %934, <4 x float> %911, i32 0, i32 0, i32 0)
+ %944 = shufflevector <1 x bfloat> %528, <1 x bfloat> %530, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %945 = shufflevector <4 x bfloat> %944, <4 x bfloat> %533, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %946 = shufflevector <4 x bfloat> %945, <4 x bfloat> %537, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %947 = shufflevector <1 x bfloat> %696, <1 x bfloat> %700, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %948 = shufflevector <4 x bfloat> %947, <4 x bfloat> %705, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %949 = shufflevector <4 x bfloat> %948, <4 x bfloat> %713, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %950 = bitcast <4 x bfloat> %946 to <4 x i16>
+ %951 = bitcast <4 x bfloat> %949 to <4 x i16>
+ %952 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %951, <4 x float> %920, i32 0, i32 0, i32 0)
+ %953 = shufflevector <1 x bfloat> %697, <1 x bfloat> %701, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %954 = shufflevector <4 x bfloat> %953, <4 x bfloat> %707, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %955 = shufflevector <4 x bfloat> %954, <4 x bfloat> %715, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %956 = bitcast <4 x bfloat> %955 to <4 x i16>
+ %957 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %956, <4 x float> %925, i32 0, i32 0, i32 0)
+ %958 = shufflevector <1 x bfloat> %698, <1 x bfloat> %702, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %959 = shufflevector <4 x bfloat> %958, <4 x bfloat> %709, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %960 = shufflevector <4 x bfloat> %959, <4 x bfloat> %717, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %961 = bitcast <4 x bfloat> %960 to <4 x i16>
+ %962 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %961, <4 x float> %930, i32 0, i32 0, i32 0)
+ %963 = shufflevector <1 x bfloat> %699, <1 x bfloat> %703, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %964 = shufflevector <4 x bfloat> %963, <4 x bfloat> %711, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %965 = shufflevector <4 x bfloat> %964, <4 x bfloat> %719, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %966 = bitcast <4 x bfloat> %965 to <4 x i16>
+ %967 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %966, <4 x float> %935, i32 0, i32 0, i32 0)
+ %968 = shufflevector <1 x bfloat> %529, <1 x bfloat> %531, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %969 = shufflevector <4 x bfloat> %968, <4 x bfloat> %535, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %970 = shufflevector <4 x bfloat> %969, <4 x bfloat> %539, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %971 = bitcast <4 x bfloat> %970 to <4 x i16>
+ %972 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %951, <4 x float> %940, i32 0, i32 0, i32 0)
+ %973 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %956, <4 x float> %941, i32 0, i32 0, i32 0)
+ %974 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %961, <4 x float> %942, i32 0, i32 0, i32 0)
+ %975 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %966, <4 x float> %943, i32 0, i32 0, i32 0)
+ %976 = shufflevector <1 x bfloat> %540, <1 x bfloat> %542, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %977 = shufflevector <4 x bfloat> %976, <4 x bfloat> %545, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %978 = shufflevector <4 x bfloat> %977, <4 x bfloat> %549, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %979 = shufflevector <1 x bfloat> %720, <1 x bfloat> %724, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %980 = shufflevector <4 x bfloat> %979, <4 x bfloat> %729, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %981 = shufflevector <4 x bfloat> %980, <4 x bfloat> %737, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %982 = bitcast <4 x bfloat> %978 to <4 x i16>
+ %983 = bitcast <4 x bfloat> %981 to <4 x i16>
+ %984 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %983, <4 x float> %952, i32 0, i32 0, i32 0)
+ %985 = shufflevector <1 x bfloat> %721, <1 x bfloat> %725, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %986 = shufflevector <4 x bfloat> %985, <4 x bfloat> %731, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %987 = shufflevector <4 x bfloat> %986, <4 x bfloat> %739, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %988 = bitcast <4 x bfloat> %987 to <4 x i16>
+ %989 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %988, <4 x float> %957, i32 0, i32 0, i32 0)
+ %990 = shufflevector <1 x bfloat> %722, <1 x bfloat> %726, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %991 = shufflevector <4 x bfloat> %990, <4 x bfloat> %733, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %992 = shufflevector <4 x bfloat> %991, <4 x bfloat> %741, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %993 = bitcast <4 x bfloat> %992 to <4 x i16>
+ %994 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %993, <4 x float> %962, i32 0, i32 0, i32 0)
+ %995 = shufflevector <1 x bfloat> %723, <1 x bfloat> %727, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %996 = shufflevector <4 x bfloat> %995, <4 x bfloat> %735, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %997 = shufflevector <4 x bfloat> %996, <4 x bfloat> %743, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %998 = bitcast <4 x bfloat> %997 to <4 x i16>
+ %999 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %998, <4 x float> %967, i32 0, i32 0, i32 0)
+ %1000 = shufflevector <1 x bfloat> %541, <1 x bfloat> %543, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1001 = shufflevector <4 x bfloat> %1000, <4 x bfloat> %547, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1002 = shufflevector <4 x bfloat> %1001, <4 x bfloat> %551, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1003 = bitcast <4 x bfloat> %1002 to <4 x i16>
+ %1004 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %983, <4 x float> %972, i32 0, i32 0, i32 0)
+ %1005 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %988, <4 x float> %973, i32 0, i32 0, i32 0)
+ %1006 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %993, <4 x float> %974, i32 0, i32 0, i32 0)
+ %1007 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %998, <4 x float> %975, i32 0, i32 0, i32 0)
+ %1008 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %984, 0, 0, 0
+ %1009 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1008, <4 x float> %989, 1, 0, 0
+ %1010 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1009, <4 x float> %994, 2, 0, 0
+ %1011 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1010, <4 x float> %999, 3, 0, 0
+ %1012 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1011, <4 x float> %1004, 0, 1, 0
+ %1013 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1012, <4 x float> %1005, 1, 1, 0
+ %1014 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1013, <4 x float> %1006, 2, 1, 0
+ %1015 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1014, <4 x float> %1007, 3, 1, 0
+ %1016 = trunc i32 %398 to i1
+ %1017 = and i1 %41, %1016
+ %1018 = select i1 %1017, <8 x bfloat> %400, <8 x bfloat> zeroinitializer
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ tail call void @llvm.amdgcn.sched.barrier(i32 0)
+ store <8 x bfloat> %1018, ptr addrspace(3) %85, align 8
+ %1019 = trunc i32 %403 to i1
+ %1020 = and i1 %41, %1019
+ %1021 = select i1 %1020, <8 x bfloat> %405, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1021, ptr addrspace(3) %88, align 8
+ %1022 = trunc i32 %408 to i1
+ %1023 = and i1 %41, %1022
+ %1024 = select i1 %1023, <8 x bfloat> %410, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1024, ptr addrspace(3) %91, align 8
+ %1025 = trunc i32 %413 to i1
+ %1026 = and i1 %41, %1025
+ %1027 = select i1 %1026, <8 x bfloat> %415, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1027, ptr addrspace(3) %94, align 8
+ %1028 = trunc i32 %418 to i1
+ %1029 = and i1 %44, %1028
+ %1030 = select i1 %1029, <8 x bfloat> %420, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1030, ptr addrspace(3) %97, align 8
+ %1031 = trunc i32 %423 to i1
+ %1032 = and i1 %44, %1031
+ %1033 = select i1 %1032, <8 x bfloat> %425, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1033, ptr addrspace(3) %100, align 8
+ %1034 = trunc i32 %428 to i1
+ %1035 = and i1 %44, %1034
+ %1036 = select i1 %1035, <8 x bfloat> %430, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1036, ptr addrspace(3) %103, align 8
+ %1037 = trunc i32 %433 to i1
+ %1038 = and i1 %44, %1037
+ %1039 = select i1 %1038, <8 x bfloat> %435, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1039, ptr addrspace(3) %106, align 8
+ %1040 = trunc i32 %438 to i1
+ %1041 = and i1 %44, %1040
+ %1042 = select i1 %1041, <8 x bfloat> %440, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1042, ptr addrspace(3) %109, align 8
+ %1043 = trunc i32 %443 to i1
+ %1044 = and i1 %44, %1043
+ %1045 = select i1 %1044, <8 x bfloat> %445, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1045, ptr addrspace(3) %112, align 8
+ %1046 = trunc i32 %448 to i1
+ %1047 = and i1 %44, %1046
+ %1048 = select i1 %1047, <8 x bfloat> %450, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1048, ptr addrspace(3) %115, align 8
+ %1049 = trunc i32 %453 to i1
+ %1050 = and i1 %44, %1049
+ %1051 = select i1 %1050, <8 x bfloat> %455, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1051, ptr addrspace(3) %118, align 8
+ %1052 = icmp samesign ult i32 %391, 72
+ br i1 %1052, label %389, label %1053
+
+1053: ; preds = %389
+ %1054 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %2, i16 0, i64 8128512, i32 159744)
+ %1055 = shl nuw nsw i32 %8, 5
+ %.scalar = or disjoint i32 %1055, %11
+ %1056 = insertelement <4 x i32> poison, i32 %.scalar, i64 0
+ %1057 = shufflevector <4 x i32> %1056, <4 x i32> poison, <4 x i32> zeroinitializer
+ %1058 = or disjoint <4 x i32> %1057, <i32 0, i32 1, i32 2, i32 3>
+ %1059 = or disjoint <4 x i32> %1057, <i32 16, i32 17, i32 18, i32 19>
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ %1060 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
+ %1061 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
+ %1062 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
+ %1063 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
+ %1064 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
+ %1065 = shufflevector <1 x bfloat> %1064, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1066 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
+ %1067 = shufflevector <1 x bfloat> %1066, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1068 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
+ %1069 = shufflevector <1 x bfloat> %1068, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1070 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
+ %1071 = shufflevector <1 x bfloat> %1070, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1072 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
+ %1073 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
+ %1074 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
+ %1075 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
+ %1076 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
+ %1077 = shufflevector <1 x bfloat> %1076, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1078 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
+ %1079 = shufflevector <1 x bfloat> %1078, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1080 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
+ %1081 = shufflevector <1 x bfloat> %1080, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1082 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
+ %1083 = shufflevector <1 x bfloat> %1082, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1084 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
+ %1085 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
+ %1086 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
+ %1087 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
+ %1088 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
+ %1089 = shufflevector <1 x bfloat> %1088, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1090 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
+ %1091 = shufflevector <1 x bfloat> %1090, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1092 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
+ %1093 = shufflevector <1 x bfloat> %1092, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1094 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
+ %1095 = shufflevector <1 x bfloat> %1094, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1096 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
+ %1097 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
+ %1098 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
+ %1099 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
+ %1100 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
+ %1101 = shufflevector <1 x bfloat> %1100, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1102 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
+ %1103 = shufflevector <1 x bfloat> %1102, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1104 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
+ %1105 = shufflevector <1 x bfloat> %1104, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1106 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
+ %1107 = shufflevector <1 x bfloat> %1106, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1108 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
+ %1109 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
+ %1110 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
+ %1111 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
+ %1112 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
+ %1113 = shufflevector <1 x bfloat> %1112, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1114 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
+ %1115 = shufflevector <1 x bfloat> %1114, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1116 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
+ %1117 = shufflevector <1 x bfloat> %1116, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1118 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
+ %1119 = shufflevector <1 x bfloat> %1118, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1120 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
+ %1121 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
+ %1122 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
+ %1123 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
+ %1124 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
+ %1125 = shufflevector <1 x bfloat> %1124, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1126 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
+ %1127 = shufflevector <1 x bfloat> %1126, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1128 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
+ %1129 = shufflevector <1 x bfloat> %1128, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1130 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
+ %1131 = shufflevector <1 x bfloat> %1130, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1132 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
+ %1133 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
+ %1134 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
+ %1135 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
+ %1136 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
+ %1137 = shufflevector <1 x bfloat> %1136, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1138 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
+ %1139 = shufflevector <1 x bfloat> %1138, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1140 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
+ %1141 = shufflevector <1 x bfloat> %1140, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1142 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
+ %1143 = shufflevector <1 x bfloat> %1142, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1144 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
+ %1145 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
+ %1146 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
+ %1147 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
+ %1148 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
+ %1149 = shufflevector <1 x bfloat> %1148, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1150 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
+ %1151 = shufflevector <1 x bfloat> %1150, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1152 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
+ %1153 = shufflevector <1 x bfloat> %1152, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1154 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
+ %1155 = shufflevector <1 x bfloat> %1154, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1156 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
+ %1157 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
+ %1158 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
+ %1159 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
+ %1160 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
+ %1161 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
+ %1162 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
+ %1163 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
+ %1164 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
+ %1165 = shufflevector <1 x bfloat> %1164, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1166 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
+ %1167 = shufflevector <1 x bfloat> %1166, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1168 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
+ %1169 = shufflevector <1 x bfloat> %1168, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1170 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
+ %1171 = shufflevector <1 x bfloat> %1170, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1172 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
+ %1173 = shufflevector <1 x bfloat> %1172, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1174 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
+ %1175 = shufflevector <1 x bfloat> %1174, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1176 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
+ %1177 = shufflevector <1 x bfloat> %1176, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1178 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
+ %1179 = shufflevector <1 x bfloat> %1178, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1180 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
+ %1181 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
+ %1182 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
+ %1183 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
+ %1184 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
+ %1185 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
+ %1186 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
+ %1187 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
+ %1188 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
+ %1189 = shufflevector <1 x bfloat> %1188, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1190 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
+ %1191 = shufflevector <1 x bfloat> %1190, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1192 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
+ %1193 = shufflevector <1 x bfloat> %1192, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1194 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
+ %1195 = shufflevector <1 x bfloat> %1194, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1196 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
+ %1197 = shufflevector <1 x bfloat> %1196, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1198 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
+ %1199 = shufflevector <1 x bfloat> %1198, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1200 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
+ %1201 = shufflevector <1 x bfloat> %1200, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1202 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
+ %1203 = shufflevector <1 x bfloat> %1202, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1204 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
+ %1205 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
+ %1206 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
+ %1207 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
+ %1208 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
+ %1209 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
+ %1210 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
+ %1211 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
+ %1212 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
+ %1213 = shufflevector <1 x bfloat> %1212, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1214 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
+ %1215 = shufflevector <1 x bfloat> %1214, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1216 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
+ %1217 = shufflevector <1 x bfloat> %1216, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1218 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
+ %1219 = shufflevector <1 x bfloat> %1218, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1220 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
+ %1221 = shufflevector <1 x bfloat> %1220, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1222 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
+ %1223 = shufflevector <1 x bfloat> %1222, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1224 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
+ %1225 = shufflevector <1 x bfloat> %1224, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1226 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
+ %1227 = shufflevector <1 x bfloat> %1226, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1228 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
+ %1229 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
+ %1230 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
+ %1231 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
+ %1232 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
+ %1233 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
+ %1234 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
+ %1235 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
+ %1236 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
+ %1237 = shufflevector <1 x bfloat> %1236, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1238 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
+ %1239 = shufflevector <1 x bfloat> %1238, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1240 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
+ %1241 = shufflevector <1 x bfloat> %1240, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1242 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
+ %1243 = shufflevector <1 x bfloat> %1242, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1244 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
+ %1245 = shufflevector <1 x bfloat> %1244, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1246 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
+ %1247 = shufflevector <1 x bfloat> %1246, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1248 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
+ %1249 = shufflevector <1 x bfloat> %1248, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1250 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
+ %1251 = shufflevector <1 x bfloat> %1250, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1252 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
+ %1253 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
+ %1254 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
+ %1255 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
+ %1256 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
+ %1257 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
+ %1258 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
+ %1259 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
+ %1260 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
+ %1261 = shufflevector <1 x bfloat> %1260, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1262 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
+ %1263 = shufflevector <1 x bfloat> %1262, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1264 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
+ %1265 = shufflevector <1 x bfloat> %1264, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1266 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
+ %1267 = shufflevector <1 x bfloat> %1266, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1268 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
+ %1269 = shufflevector <1 x bfloat> %1268, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1270 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
+ %1271 = shufflevector <1 x bfloat> %1270, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1272 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
+ %1273 = shufflevector <1 x bfloat> %1272, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1274 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
+ %1275 = shufflevector <1 x bfloat> %1274, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1276 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
+ %1277 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
+ %1278 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
+ %1279 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
+ %1280 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
+ %1281 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
+ %1282 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
+ %1283 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
+ %1284 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
+ %1285 = shufflevector <1 x bfloat> %1284, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1286 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
+ %1287 = shufflevector <1 x bfloat> %1286, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1288 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
+ %1289 = shufflevector <1 x bfloat> %1288, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1290 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
+ %1291 = shufflevector <1 x bfloat> %1290, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1292 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
+ %1293 = shufflevector <1 x bfloat> %1292, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1294 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
+ %1295 = shufflevector <1 x bfloat> %1294, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1296 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
+ %1297 = shufflevector <1 x bfloat> %1296, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1298 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
+ %1299 = shufflevector <1 x bfloat> %1298, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1300 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
+ %1301 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
+ %1302 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
+ %1303 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
+ %1304 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
+ %1305 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
+ %1306 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
+ %1307 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
+ %1308 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
+ %1309 = shufflevector <1 x bfloat> %1308, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1310 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
+ %1311 = shufflevector <1 x bfloat> %1310, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1312 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
+ %1313 = shufflevector <1 x bfloat> %1312, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1314 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
+ %1315 = shufflevector <1 x bfloat> %1314, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1316 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
+ %1317 = shufflevector <1 x bfloat> %1316, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1318 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
+ %1319 = shufflevector <1 x bfloat> %1318, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1320 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
+ %1321 = shufflevector <1 x bfloat> %1320, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1322 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
+ %1323 = shufflevector <1 x bfloat> %1322, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1324 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
+ %1325 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
+ %1326 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
+ %1327 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
+ %1328 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
+ %1329 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
+ %1330 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
+ %1331 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
+ %1332 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
+ %1333 = shufflevector <1 x bfloat> %1332, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1334 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
+ %1335 = shufflevector <1 x bfloat> %1334, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1336 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
+ %1337 = shufflevector <1 x bfloat> %1336, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1338 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
+ %1339 = shufflevector <1 x bfloat> %1338, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1340 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
+ %1341 = shufflevector <1 x bfloat> %1340, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1342 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
+ %1343 = shufflevector <1 x bfloat> %1342, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1344 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
+ %1345 = shufflevector <1 x bfloat> %1344, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1346 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
+ %1347 = shufflevector <1 x bfloat> %1346, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1348 = shufflevector <1 x bfloat> %1060, <1 x bfloat> %1062, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1349 = shufflevector <4 x bfloat> %1348, <4 x bfloat> %1065, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1350 = shufflevector <4 x bfloat> %1349, <4 x bfloat> %1069, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1351 = shufflevector <1 x bfloat> %1156, <1 x bfloat> %1160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1352 = shufflevector <4 x bfloat> %1351, <4 x bfloat> %1165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1353 = shufflevector <4 x bfloat> %1352, <4 x bfloat> %1173, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1354 = bitcast <4 x bfloat> %1350 to <4 x i16>
+ %1355 = bitcast <4 x bfloat> %1353 to <4 x i16>
+ %1356 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1355, <4 x float> %984, i32 0, i32 0, i32 0)
+ %1357 = shufflevector <1 x bfloat> %1157, <1 x bfloat> %1161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1358 = shufflevector <4 x bfloat> %1357, <4 x bfloat> %1167, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1359 = shufflevector <4 x bfloat> %1358, <4 x bfloat> %1175, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1360 = bitcast <4 x bfloat> %1359 to <4 x i16>
+ %1361 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1360, <4 x float> %989, i32 0, i32 0, i32 0)
+ %1362 = shufflevector <1 x bfloat> %1158, <1 x bfloat> %1162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1363 = shufflevector <4 x bfloat> %1362, <4 x bfloat> %1169, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1364 = shufflevector <4 x bfloat> %1363, <4 x bfloat> %1177, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1365 = bitcast <4 x bfloat> %1364 to <4 x i16>
+ %1366 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1365, <4 x float> %994, i32 0, i32 0, i32 0)
+ %1367 = shufflevector <1 x bfloat> %1159, <1 x bfloat> %1163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1368 = shufflevector <4 x bfloat> %1367, <4 x bfloat> %1171, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1369 = shufflevector <4 x bfloat> %1368, <4 x bfloat> %1179, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1370 = bitcast <4 x bfloat> %1369 to <4 x i16>
+ %1371 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1370, <4 x float> %999, i32 0, i32 0, i32 0)
+ %1372 = shufflevector <1 x bfloat> %1061, <1 x bfloat> %1063, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1373 = shufflevector <4 x bfloat> %1372, <4 x bfloat> %1067, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1374 = shufflevector <4 x bfloat> %1373, <4 x bfloat> %1071, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1375 = bitcast <4 x bfloat> %1374 to <4 x i16>
+ %1376 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1355, <4 x float> %1004, i32 0, i32 0, i32 0)
+ %1377 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1360, <4 x float> %1005, i32 0, i32 0, i32 0)
+ %1378 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1365, <4 x float> %1006, i32 0, i32 0, i32 0)
+ %1379 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1370, <4 x float> %1007, i32 0, i32 0, i32 0)
+ %1380 = shufflevector <1 x bfloat> %1072, <1 x bfloat> %1074, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1381 = shufflevector <4 x bfloat> %1380, <4 x bfloat> %1077, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1382 = shufflevector <4 x bfloat> %1381, <4 x bfloat> %1081, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1383 = shufflevector <1 x bfloat> %1180, <1 x bfloat> %1184, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1384 = shufflevector <4 x bfloat> %1383, <4 x bfloat> %1189, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1385 = shufflevector <4 x bfloat> %1384, <4 x bfloat> %1197, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1386 = bitcast <4 x bfloat> %1382 to <4 x i16>
+ %1387 = bitcast <4 x bfloat> %1385 to <4 x i16>
+ %1388 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1387, <4 x float> %1356, i32 0, i32 0, i32 0)
+ %1389 = shufflevector <1 x bfloat> %1181, <1 x bfloat> %1185, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1390 = shufflevector <4 x bfloat> %1389, <4 x bfloat> %1191, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1391 = shufflevector <4 x bfloat> %1390, <4 x bfloat> %1199, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1392 = bitcast <4 x bfloat> %1391 to <4 x i16>
+ %1393 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1392, <4 x float> %1361, i32 0, i32 0, i32 0)
+ %1394 = shufflevector <1 x bfloat> %1182, <1 x bfloat> %1186, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1395 = shufflevector <4 x bfloat> %1394, <4 x bfloat> %1193, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1396 = shufflevector <4 x bfloat> %1395, <4 x bfloat> %1201, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1397 = bitcast <4 x bfloat> %1396 to <4 x i16>
+ %1398 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1397, <4 x float> %1366, i32 0, i32 0, i32 0)
+ %1399 = shufflevector <1 x bfloat> %1183, <1 x bfloat> %1187, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1400 = shufflevector <4 x bfloat> %1399, <4 x bfloat> %1195, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1401 = shufflevector <4 x bfloat> %1400, <4 x bfloat> %1203, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1402 = bitcast <4 x bfloat> %1401 to <4 x i16>
+ %1403 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1402, <4 x float> %1371, i32 0, i32 0, i32 0)
+ %1404 = shufflevector <1 x bfloat> %1073, <1 x bfloat> %1075, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1405 = shufflevector <4 x bfloat> %1404, <4 x bfloat> %1079, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1406 = shufflevector <4 x bfloat> %1405, <4 x bfloat> %1083, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1407 = bitcast <4 x bfloat> %1406 to <4 x i16>
+ %1408 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1387, <4 x float> %1376, i32 0, i32 0, i32 0)
+ %1409 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1392, <4 x float> %1377, i32 0, i32 0, i32 0)
+ %1410 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1397, <4 x float> %1378, i32 0, i32 0, i32 0)
+ %1411 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1402, <4 x float> %1379, i32 0, i32 0, i32 0)
+ %1412 = shufflevector <1 x bfloat> %1084, <1 x bfloat> %1086, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1413 = shufflevector <4 x bfloat> %1412, <4 x bfloat> %1089, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1414 = shufflevector <4 x bfloat> %1413, <4 x bfloat> %1093, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1415 = shufflevector <1 x bfloat> %1204, <1 x bfloat> %1208, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1416 = shufflevector <4 x bfloat> %1415, <4 x bfloat> %1213, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1417 = shufflevector <4 x bfloat> %1416, <4 x bfloat> %1221, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1418 = bitcast <4 x bfloat> %1414 to <4 x i16>
+ %1419 = bitcast <4 x bfloat> %1417 to <4 x i16>
+ %1420 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1419, <4 x float> %1388, i32 0, i32 0, i32 0)
+ %1421 = shufflevector <1 x bfloat> %1205, <1 x bfloat> %1209, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1422 = shufflevector <4 x bfloat> %1421, <4 x bfloat> %1215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1423 = shufflevector <4 x bfloat> %1422, <4 x bfloat> %1223, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1424 = bitcast <4 x bfloat> %1423 to <4 x i16>
+ %1425 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1424, <4 x float> %1393, i32 0, i32 0, i32 0)
+ %1426 = shufflevector <1 x bfloat> %1206, <1 x bfloat> %1210, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1427 = shufflevector <4 x bfloat> %1426, <4 x bfloat> %1217, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1428 = shufflevector <4 x bfloat> %1427, <4 x bfloat> %1225, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1429 = bitcast <4 x bfloat> %1428 to <4 x i16>
+ %1430 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1429, <4 x float> %1398, i32 0, i32 0, i32 0)
+ %1431 = shufflevector <1 x bfloat> %1207, <1 x bfloat> %1211, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1432 = shufflevector <4 x bfloat> %1431, <4 x bfloat> %1219, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1433 = shufflevector <4 x bfloat> %1432, <4 x bfloat> %1227, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1434 = bitcast <4 x bfloat> %1433 to <4 x i16>
+ %1435 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1434, <4 x float> %1403, i32 0, i32 0, i32 0)
+ %1436 = shufflevector <1 x bfloat> %1085, <1 x bfloat> %1087, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1437 = shufflevector <4 x bfloat> %1436, <4 x bfloat> %1091, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1438 = shufflevector <4 x bfloat> %1437, <4 x bfloat> %1095, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1439 = bitcast <4 x bfloat> %1438 to <4 x i16>
+ %1440 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1419, <4 x float> %1408, i32 0, i32 0, i32 0)
+ %1441 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1424, <4 x float> %1409, i32 0, i32 0, i32 0)
+ %1442 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1429, <4 x float> %1410, i32 0, i32 0, i32 0)
+ %1443 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1434, <4 x float> %1411, i32 0, i32 0, i32 0)
+ %1444 = shufflevector <1 x bfloat> %1096, <1 x bfloat> %1098, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1445 = shufflevector <4 x bfloat> %1444, <4 x bfloat> %1101, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1446 = shufflevector <4 x bfloat> %1445, <4 x bfloat> %1105, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1447 = shufflevector <1 x bfloat> %1228, <1 x bfloat> %1232, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1448 = shufflevector <4 x bfloat> %1447, <4 x bfloat> %1237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1449 = shufflevector <4 x bfloat> %1448, <4 x bfloat> %1245, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1450 = bitcast <4 x bfloat> %1446 to <4 x i16>
+ %1451 = bitcast <4 x bfloat> %1449 to <4 x i16>
+ %1452 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1451, <4 x float> %1420, i32 0, i32 0, i32 0)
+ %1453 = shufflevector <1 x bfloat> %1229, <1 x bfloat> %1233, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1454 = shufflevector <4 x bfloat> %1453, <4 x bfloat> %1239, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1455 = shufflevector <4 x bfloat> %1454, <4 x bfloat> %1247, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1456 = bitcast <4 x bfloat> %1455 to <4 x i16>
+ %1457 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1456, <4 x float> %1425, i32 0, i32 0, i32 0)
+ %1458 = shufflevector <1 x bfloat> %1230, <1 x bfloat> %1234, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1459 = shufflevector <4 x bfloat> %1458, <4 x bfloat> %1241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1460 = shufflevector <4 x bfloat> %1459, <4 x bfloat> %1249, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1461 = bitcast <4 x bfloat> %1460 to <4 x i16>
+ %1462 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1461, <4 x float> %1430, i32 0, i32 0, i32 0)
+ %1463 = shufflevector <1 x bfloat> %1231, <1 x bfloat> %1235, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1464 = shufflevector <4 x bfloat> %1463, <4 x bfloat> %1243, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1465 = shufflevector <4 x bfloat> %1464, <4 x bfloat> %1251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1466 = bitcast <4 x bfloat> %1465 to <4 x i16>
+ %1467 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1466, <4 x float> %1435, i32 0, i32 0, i32 0)
+ %1468 = shufflevector <1 x bfloat> %1097, <1 x bfloat> %1099, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1469 = shufflevector <4 x bfloat> %1468, <4 x bfloat> %1103, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1470 = shufflevector <4 x bfloat> %1469, <4 x bfloat> %1107, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1471 = bitcast <4 x bfloat> %1470 to <4 x i16>
+ %1472 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1451, <4 x float> %1440, i32 0, i32 0, i32 0)
+ %1473 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1456, <4 x float> %1441, i32 0, i32 0, i32 0)
+ %1474 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1461, <4 x float> %1442, i32 0, i32 0, i32 0)
+ %1475 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1466, <4 x float> %1443, i32 0, i32 0, i32 0)
+ %1476 = shufflevector <1 x bfloat> %1108, <1 x bfloat> %1110, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1477 = shufflevector <4 x bfloat> %1476, <4 x bfloat> %1113, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1478 = shufflevector <4 x bfloat> %1477, <4 x bfloat> %1117, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1479 = shufflevector <1 x bfloat> %1252, <1 x bfloat> %1256, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1480 = shufflevector <4 x bfloat> %1479, <4 x bfloat> %1261, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1481 = shufflevector <4 x bfloat> %1480, <4 x bfloat> %1269, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1482 = bitcast <4 x bfloat> %1478 to <4 x i16>
+ %1483 = bitcast <4 x bfloat> %1481 to <4 x i16>
+ %1484 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1483, <4 x float> %1452, i32 0, i32 0, i32 0)
+ %1485 = shufflevector <1 x bfloat> %1253, <1 x bfloat> %1257, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1486 = shufflevector <4 x bfloat> %1485, <4 x bfloat> %1263, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1487 = shufflevector <4 x bfloat> %1486, <4 x bfloat> %1271, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1488 = bitcast <4 x bfloat> %1487 to <4 x i16>
+ %1489 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1488, <4 x float> %1457, i32 0, i32 0, i32 0)
+ %1490 = shufflevector <1 x bfloat> %1254, <1 x bfloat> %1258, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1491 = shufflevector <4 x bfloat> %1490, <4 x bfloat> %1265, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1492 = shufflevector <4 x bfloat> %1491, <4 x bfloat> %1273, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1493 = bitcast <4 x bfloat> %1492 to <4 x i16>
+ %1494 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1493, <4 x float> %1462, i32 0, i32 0, i32 0)
+ %1495 = shufflevector <1 x bfloat> %1255, <1 x bfloat> %1259, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1496 = shufflevector <4 x bfloat> %1495, <4 x bfloat> %1267, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1497 = shufflevector <4 x bfloat> %1496, <4 x bfloat> %1275, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1498 = bitcast <4 x bfloat> %1497 to <4 x i16>
+ %1499 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1498, <4 x float> %1467, i32 0, i32 0, i32 0)
+ %1500 = shufflevector <1 x bfloat> %1109, <1 x bfloat> %1111, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1501 = shufflevector <4 x bfloat> %1500, <4 x bfloat> %1115, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1502 = shufflevector <4 x bfloat> %1501, <4 x bfloat> %1119, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1503 = bitcast <4 x bfloat> %1502 to <4 x i16>
+ %1504 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1483, <4 x float> %1472, i32 0, i32 0, i32 0)
+ %1505 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1488, <4 x float> %1473, i32 0, i32 0, i32 0)
+ %1506 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1493, <4 x float> %1474, i32 0, i32 0, i32 0)
+ %1507 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1498, <4 x float> %1475, i32 0, i32 0, i32 0)
+ %1508 = shufflevector <1 x bfloat> %1120, <1 x bfloat> %1122, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1509 = shufflevector <4 x bfloat> %1508, <4 x bfloat> %1125, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1510 = shufflevector <4 x bfloat> %1509, <4 x bfloat> %1129, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1511 = shufflevector <1 x bfloat> %1276, <1 x bfloat> %1280, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1512 = shufflevector <4 x bfloat> %1511, <4 x bfloat> %1285, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1513 = shufflevector <4 x bfloat> %1512, <4 x bfloat> %1293, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1514 = bitcast <4 x bfloat> %1510 to <4 x i16>
+ %1515 = bitcast <4 x bfloat> %1513 to <4 x i16>
+ %1516 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1515, <4 x float> %1484, i32 0, i32 0, i32 0)
+ %1517 = shufflevector <1 x bfloat> %1277, <1 x bfloat> %1281, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1518 = shufflevector <4 x bfloat> %1517, <4 x bfloat> %1287, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1519 = shufflevector <4 x bfloat> %1518, <4 x bfloat> %1295, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1520 = bitcast <4 x bfloat> %1519 to <4 x i16>
+ %1521 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1520, <4 x float> %1489, i32 0, i32 0, i32 0)
+ %1522 = shufflevector <1 x bfloat> %1278, <1 x bfloat> %1282, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1523 = shufflevector <4 x bfloat> %1522, <4 x bfloat> %1289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1524 = shufflevector <4 x bfloat> %1523, <4 x bfloat> %1297, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1525 = bitcast <4 x bfloat> %1524 to <4 x i16>
+ %1526 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1525, <4 x float> %1494, i32 0, i32 0, i32 0)
+ %1527 = shufflevector <1 x bfloat> %1279, <1 x bfloat> %1283, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1528 = shufflevector <4 x bfloat> %1527, <4 x bfloat> %1291, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1529 = shufflevector <4 x bfloat> %1528, <4 x bfloat> %1299, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1530 = bitcast <4 x bfloat> %1529 to <4 x i16>
+ %1531 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1530, <4 x float> %1499, i32 0, i32 0, i32 0)
+ %1532 = shufflevector <1 x bfloat> %1121, <1 x bfloat> %1123, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1533 = shufflevector <4 x bfloat> %1532, <4 x bfloat> %1127, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1534 = shufflevector <4 x bfloat> %1533, <4 x bfloat> %1131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1535 = bitcast <4 x bfloat> %1534 to <4 x i16>
+ %1536 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1515, <4 x float> %1504, i32 0, i32 0, i32 0)
+ %1537 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1520, <4 x float> %1505, i32 0, i32 0, i32 0)
+ %1538 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1525, <4 x float> %1506, i32 0, i32 0, i32 0)
+ %1539 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1530, <4 x float> %1507, i32 0, i32 0, i32 0)
+ %1540 = shufflevector <1 x bfloat> %1132, <1 x bfloat> %1134, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1541 = shufflevector <4 x bfloat> %1540, <4 x bfloat> %1137, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1542 = shufflevector <4 x bfloat> %1541, <4 x bfloat> %1141, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1543 = shufflevector <1 x bfloat> %1300, <1 x bfloat> %1304, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1544 = shufflevector <4 x bfloat> %1543, <4 x bfloat> %1309, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1545 = shufflevector <4 x bfloat> %1544, <4 x bfloat> %1317, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1546 = bitcast <4 x bfloat> %1542 to <4 x i16>
+ %1547 = bitcast <4 x bfloat> %1545 to <4 x i16>
+ %1548 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1547, <4 x float> %1516, i32 0, i32 0, i32 0)
+ %1549 = shufflevector <1 x bfloat> %1301, <1 x bfloat> %1305, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1550 = shufflevector <4 x bfloat> %1549, <4 x bfloat> %1311, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1551 = shufflevector <4 x bfloat> %1550, <4 x bfloat> %1319, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1552 = bitcast <4 x bfloat> %1551 to <4 x i16>
+ %1553 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1552, <4 x float> %1521, i32 0, i32 0, i32 0)
+ %1554 = shufflevector <1 x bfloat> %1302, <1 x bfloat> %1306, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1555 = shufflevector <4 x bfloat> %1554, <4 x bfloat> %1313, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1556 = shufflevector <4 x bfloat> %1555, <4 x bfloat> %1321, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1557 = bitcast <4 x bfloat> %1556 to <4 x i16>
+ %1558 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1557, <4 x float> %1526, i32 0, i32 0, i32 0)
+ %1559 = shufflevector <1 x bfloat> %1303, <1 x bfloat> %1307, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1560 = shufflevector <4 x bfloat> %1559, <4 x bfloat> %1315, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1561 = shufflevector <4 x bfloat> %1560, <4 x bfloat> %1323, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1562 = bitcast <4 x bfloat> %1561 to <4 x i16>
+ %1563 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1562, <4 x float> %1531, i32 0, i32 0, i32 0)
+ %1564 = shufflevector <1 x bfloat> %1133, <1 x bfloat> %1135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1565 = shufflevector <4 x bfloat> %1564, <4 x bfloat> %1139, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1566 = shufflevector <4 x bfloat> %1565, <4 x bfloat> %1143, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1567 = bitcast <4 x bfloat> %1566 to <4 x i16>
+ %1568 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1547, <4 x float> %1536, i32 0, i32 0, i32 0)
+ %1569 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1552, <4 x float> %1537, i32 0, i32 0, i32 0)
+ %1570 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1557, <4 x float> %1538, i32 0, i32 0, i32 0)
+ %1571 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1562, <4 x float> %1539, i32 0, i32 0, i32 0)
+ %1572 = shufflevector <1 x bfloat> %1144, <1 x bfloat> %1146, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1573 = shufflevector <4 x bfloat> %1572, <4 x bfloat> %1149, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1574 = shufflevector <4 x bfloat> %1573, <4 x bfloat> %1153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1575 = shufflevector <1 x bfloat> %1324, <1 x bfloat> %1328, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1576 = shufflevector <4 x bfloat> %1575, <4 x bfloat> %1333, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1577 = shufflevector <4 x bfloat> %1576, <4 x bfloat> %1341, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1578 = bitcast <4 x bfloat> %1574 to <4 x i16>
+ %1579 = bitcast <4 x bfloat> %1577 to <4 x i16>
+ %1580 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1579, <4 x float> %1548, i32 0, i32 0, i32 0)
+ %1581 = shufflevector <1 x bfloat> %1325, <1 x bfloat> %1329, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1582 = shufflevector <4 x bfloat> %1581, <4 x bfloat> %1335, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1583 = shufflevector <4 x bfloat> %1582, <4 x bfloat> %1343, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1584 = bitcast <4 x bfloat> %1583 to <4 x i16>
+ %1585 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1584, <4 x float> %1553, i32 0, i32 0, i32 0)
+ %1586 = shufflevector <4 x float> %1585, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1587 = shufflevector <1 x bfloat> %1326, <1 x bfloat> %1330, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1588 = shufflevector <4 x bfloat> %1587, <4 x bfloat> %1337, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1589 = shufflevector <4 x bfloat> %1588, <4 x bfloat> %1345, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1590 = bitcast <4 x bfloat> %1589 to <4 x i16>
+ %1591 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1590, <4 x float> %1558, i32 0, i32 0, i32 0)
+ %1592 = shufflevector <4 x float> %1591, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1593 = shufflevector <1 x bfloat> %1327, <1 x bfloat> %1331, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1594 = shufflevector <4 x bfloat> %1593, <4 x bfloat> %1339, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1595 = shufflevector <4 x bfloat> %1594, <4 x bfloat> %1347, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1596 = bitcast <4 x bfloat> %1595 to <4 x i16>
+ %1597 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1596, <4 x float> %1563, i32 0, i32 0, i32 0)
+ %1598 = shufflevector <4 x float> %1597, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1599 = shufflevector <1 x bfloat> %1145, <1 x bfloat> %1147, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1600 = shufflevector <4 x bfloat> %1599, <4 x bfloat> %1151, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1601 = shufflevector <4 x bfloat> %1600, <4 x bfloat> %1155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1602 = bitcast <4 x bfloat> %1601 to <4 x i16>
+ %1603 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1579, <4 x float> %1568, i32 0, i32 0, i32 0)
+ %1604 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1584, <4 x float> %1569, i32 0, i32 0, i32 0)
+ %1605 = shufflevector <4 x float> %1604, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1606 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1590, <4 x float> %1570, i32 0, i32 0, i32 0)
+ %1607 = shufflevector <4 x float> %1606, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1608 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1596, <4 x float> %1571, i32 0, i32 0, i32 0)
+ %1609 = shufflevector <4 x float> %1608, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1610 = shufflevector <4 x float> %1580, <4 x float> %1603, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1611 = shufflevector <32 x float> %1610, <32 x float> %1586, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1612 = shufflevector <32 x float> %1611, <32 x float> %1605, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1613 = shufflevector <32 x float> %1612, <32 x float> %1592, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1614 = shufflevector <32 x float> %1613, <32 x float> %1607, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1615 = shufflevector <32 x float> %1614, <32 x float> %1598, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1616 = shufflevector <32 x float> %1615, <32 x float> %1609, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
+ %1617 = insertelement <4 x i32> poison, i32 %38, i64 0
+ %1618 = insertelement <4 x i32> poison, i32 %32, i64 0
+ %1619 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1620 = shufflevector <4 x i32> %1618, <4 x i32> poison, <8 x i32> zeroinitializer
+ %1621 = or disjoint <8 x i32> %1619, %1620
+ %1622 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1623 = shufflevector <4 x i32> %1617, <4 x i32> poison, <8 x i32> zeroinitializer
+ %1624 = icmp samesign ult <8 x i32> %1622, %1623
+ br label %1625
+
+1625: ; preds = %1053, %1625
+ %1626 = phi i32 [ 0, %1053 ], [ %1676, %1625 ]
+ %1627 = shl nuw nsw i32 %1626, 3
+ %1628 = extractelement <32 x float> %1616, i32 %1627
+ %1629 = insertelement <4 x float> poison, float %1628, i64 0
+ %1630 = or disjoint i32 %1627, 1
+ %1631 = extractelement <32 x float> %1616, i32 %1630
+ %1632 = insertelement <4 x float> %1629, float %1631, i64 1
+ %1633 = or disjoint i32 %1627, 2
+ %1634 = extractelement <32 x float> %1616, i32 %1633
+ %1635 = insertelement <4 x float> %1632, float %1634, i64 2
+ %1636 = or disjoint i32 %1627, 3
+ %1637 = extractelement <32 x float> %1616, i32 %1636
+ %1638 = insertelement <4 x float> %1635, float %1637, i64 3
+ %1639 = or disjoint i32 %1627, 4
+ %1640 = extractelement <32 x float> %1616, i32 %1639
+ %1641 = insertelement <4 x float> poison, float %1640, i64 0
+ %1642 = or disjoint i32 %1627, 5
+ %1643 = extractelement <32 x float> %1616, i32 %1642
+ %1644 = insertelement <4 x float> %1641, float %1643, i64 1
+ %1645 = or disjoint i32 %1627, 6
+ %1646 = extractelement <32 x float> %1616, i32 %1645
+ %1647 = insertelement <4 x float> %1644, float %1646, i64 2
+ %1648 = or disjoint i32 %1627, 7
+ %1649 = extractelement <32 x float> %1616, i32 %1648
+ %1650 = insertelement <4 x float> %1647, float %1649, i64 3
+ %1651 = bitcast <4 x float> %1638 to <4 x i32>
+ %1652 = bitcast <4 x float> %1650 to <4 x i32>
+ %1653 = shl nuw nsw i32 %1626, 4
+ %.reass = add nuw nsw i32 %1653, %154
+ %1654 = icmp samesign ult i32 %.reass, %36
+ %1655 = insertelement <4 x i1> poison, i1 %1654, i64 0
+ %1656 = add nuw nsw i32 %.reass, %34
+ %1657 = mul nuw nsw i32 %1656, 2016
+ %1658 = insertelement <4 x i32> poison, i32 %1657, i64 0
+ %1659 = shufflevector <4 x i32> %1658, <4 x i32> poison, <8 x i32> zeroinitializer
+ %1660 = add nuw nsw <8 x i32> %1621, %1659
+ %1661 = shufflevector <4 x i1> %1655, <4 x i1> poison, <8 x i32> zeroinitializer
+ %1662 = and <8 x i1> %1624, %1661
+ %1663 = shufflevector <4 x float> %1638, <4 x float> %1650, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1664 = fcmp uno <8 x float> %1663, zeroinitializer
+ %1665 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1666 = add <8 x i32> %1665, splat (i32 32767)
+ %1667 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1668 = lshr <8 x i32> %1667, splat (i32 16)
+ %1669 = and <8 x i32> %1668, splat (i32 1)
+ %1670 = add <8 x i32> %1666, %1669
+ %1671 = lshr <8 x i32> %1670, splat (i32 16)
+ %1672 = trunc nuw <8 x i32> %1671 to <8 x i16>
+ %1673 = bitcast <8 x i16> %1672 to <8 x bfloat>
+ %1674 = select <8 x i1> %1664, <8 x bfloat> splat (bfloat 0xR7FC0), <8 x bfloat> %1673
+ %1675 = getelementptr bfloat, ptr addrspace(7) %1054, <8 x i32> %1660
+ tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %1674, <8 x ptr addrspace(7)> align 2 %1675, <8 x i1> %1662)
+ %1676 = add nuw nsw i32 %1626, 1
+ %exitcond.not = icmp eq i32 %1676, 4
+ br i1 %exitcond.not, label %1677, label %1625
+
+1677: ; preds = %1625
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ ret void
+}
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef) #2
+
+; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) readnone, i16, i64, i32) #3
+
+; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.umin.i32(i32, i32) #3
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
+
+; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.smin.i32(i32, i32) #3
+
+; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.s.barrier() #4
+
+; Function Attrs: alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16>, <4 x i16>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #5
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write)
+declare void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat>, <8 x ptr addrspace(7)>, <8 x i1>) #6
+
+; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.sched.barrier(i32 immarg) #4
+
+attributes #0 = { alwaysinline nofree norecurse nounwind "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="256,256" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
+attributes #1 = { alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #3 = { alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #4 = { alwaysinline convergent mustprogress nocallback nofree nounwind willreturn }
+attributes #5 = { alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
+attributes #6 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write) }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"amdhsa_code_object_version", i32 500}
+!2 = !{i32 256, i32 1, i32 1}
+!3 = !{!"amdgpu-synchronize-as", !"local"}
>From a2fa3e2da1ca009c18e526e1831b928154c6c393 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:20:29 -0600
Subject: [PATCH 04/17] Remove unnecessary debug code.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 12526862927ed..0479fb04be8bd 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2303,11 +2303,8 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
SpillCost *= (int64_t)RelativeFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0) {
- LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite rejected.\n");
+ if (SpillCost > 0)
return SpillCost;
- }
- LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite successful.\n");
if (SpillCost < BestSpillCost)
BestSpillCost = SpillCost;
>From 4d042f5a0ad1e3007326a2a27fa63035f5f7b9de Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:39:36 -0600
Subject: [PATCH 05/17] Fix typo
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 0479fb04be8bd..b6a6707f6dc83 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2240,7 +2240,7 @@ bool RewriteMFMAFormStage::initHeuristics(
// registers to be different types even though the actual operand is
// the same size.
const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *AUseRC = SRI->getEquivalentVGPRClass(VUseRC);
+ const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
}
Changed = true;
>From 1ff91746a4ec502c0c72ba0c7feb4beff0517e82 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 18:09:47 -0600
Subject: [PATCH 06/17] Get rid of use of undef in test.
---
llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
index c8f444c175be8..38d3f566bf4de 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -2,8 +2,8 @@
target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
- at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] undef, align 16
- at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] undef, align 16
+ at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] poison, align 16
+ at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] poison, align 16
; Function Attrs: alwaysinline nofree norecurse nounwind
define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %0, ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %1, ptr addrspace(1) inreg noalias noundef nonnull align 16 %2) local_unnamed_addr #0 !reqd_work_group_size !2 {
>From f7cfe35aade97451b6c9e3b1b9033ad80fa672e8 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 11:01:23 -0600
Subject: [PATCH 07/17] Restore blank line.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b6a6707f6dc83..277ac7d4ecf4c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,6 +2184,7 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
+
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
>From dda30a79f662031aed2d4f66c88afcc96a04de3a Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 11:45:20 -0600
Subject: [PATCH 08/17] Reduce test case size
---
.../AMDGPU/sched_mfma_rewrite_reject.ll | 2160 +++--------------
1 file changed, 301 insertions(+), 1859 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
index 38d3f566bf4de..8f431e2e9a46c 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -1,1867 +1,309 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s | FileCheck %s
-target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
-
- at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] poison, align 16
- at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] poison, align 16
-
-; Function Attrs: alwaysinline nofree norecurse nounwind
-define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %0, ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %1, ptr addrspace(1) inreg noalias noundef nonnull align 16 %2) local_unnamed_addr #0 !reqd_work_group_size !2 {
+define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(i32 %0, ptr addrspace(3) %1, ptr addrspace(3) %2, ptr addrspace(3) %3, ptr addrspace(3) %4, ptr addrspace(3) %5, ptr addrspace(3) %6, ptr addrspace(3) %7, ptr addrspace(3) %8, ptr addrspace(3) %9, ptr addrspace(3) %10, ptr addrspace(3) %11, ptr addrspace(3) %12, ptr addrspace(7) %invariant.gep, ptr addrspace(7) %gep, ptr addrspace(7) %gep106, ptr addrspace(7) %gep108, ptr addrspace(7) %gep112, ptr addrspace(7) %gep114, ptr addrspace(7) %gep116, ptr addrspace(7) %gep118, ptr addrspace(7) %gep120, ptr addrspace(7) %gep122, ptr addrspace(7) %gep124, ptr addrspace(3) %13, ptr addrspace(3) %14, ptr addrspace(3) %15, ptr addrspace(3) %16, ptr addrspace(3) %17, ptr addrspace(3) %18, ptr addrspace(3) %19, ptr addrspace(3) %20, ptr addrspace(3) %21, ptr addrspace(3) %22, ptr addrspace(3) %23, ptr addrspace(3) %24, ptr addrspace(3) %25, ptr addrspace(3) %26, ptr addrspace(3) %27, ptr addrspace(3) %28, ptr addrspace(3) %29, ptr addrspace(3) %30, ptr addrspace(3) %31, ptr addrspace(3) %32, ptr addrspace(3) %33, ptr addrspace(3) %34, ptr addrspace(3) %35, ptr addrspace(3) %36, ptr addrspace(3) %37, ptr addrspace(3) %38, ptr addrspace(3) %39, ptr addrspace(3) %40, ptr addrspace(3) %41, ptr addrspace(3) %42, ptr addrspace(3) %43, ptr addrspace(3) %44, ptr addrspace(3) %45, ptr addrspace(3) %46, ptr addrspace(3) %47, ptr addrspace(3) %48, ptr addrspace(3) %49, ptr addrspace(3) %50, ptr addrspace(3) %51, ptr addrspace(3) %52, ptr addrspace(3) %53, ptr addrspace(3) %54, ptr addrspace(3) %55, ptr addrspace(3) %56, ptr addrspace(3) %57, ptr addrspace(3) %58, ptr addrspace(3) %59, <4 x bfloat> %60, <4 x i16> %61, <4 x i16> %62, <4 x i16> %63, <4 x i16> %64, <4 x i16> %65, <8 x i1> %66, <4 x i16> %67, <4 x i16> %68, <4 x i16> %69, <4 x i16> %70, <8 x i1> %71) #0 {
; This should be rejected for MFMA rewrite by the scheduler so we should see
; no AGPR operands.
; CHECK-NOT: v_mfma_f32_16x16x16_bf16 a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}]
- %4 = tail call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
- call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %0, i64 64) ]
- %5 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %0, i16 0, i64 5564160, i32 159744)
- call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %1, i64 64) ]
- %6 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %1, i16 0, i64 5564160, i32 159744)
- call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %2, i64 64) ]
- %7 = lshr i32 %4, 6
- %8 = and i32 %7, 1
- %9 = and i32 %4, 15
- %10 = lshr i32 %4, 2
- %11 = and i32 %10, 12
- %12 = lshr i32 %4, 3
- %13 = or disjoint i32 %4, 256
- %14 = lshr i32 %13, 3
- %15 = or disjoint i32 %4, 512
- %16 = lshr i32 %15, 3
- %17 = or disjoint i32 %4, 768
- %18 = lshr i32 %17, 3
- %19 = lshr i32 %4, 4
- %20 = lshr i32 %13, 4
- %21 = lshr i32 %15, 4
- %22 = lshr i32 %17, 4
- %23 = or disjoint i32 %19, 64
- %24 = or disjoint i32 %19, 80
- %25 = or disjoint i32 %19, 96
- %26 = or disjoint i32 %19, 112
- %27 = shl nuw nsw i32 %4, 3
- %28 = and i32 %27, 56
- %29 = and i32 %27, 120
- %30 = tail call range(i32 0, 512) i32 @llvm.amdgcn.workgroup.id.x()
- %31 = shl nuw nsw i32 %30, 6
- %32 = and i32 %31, 1984
- %33 = shl nuw nsw i32 %30, 2
- %34 = and i32 %33, 1920
- %35 = sub nuw nsw i32 2016, %34
- %36 = tail call i32 @llvm.umin.i32(i32 %35, i32 128)
- %37 = sub nuw nsw i32 2016, %32
- %38 = tail call i32 @llvm.umin.i32(i32 %37, i32 64)
- %39 = tail call i32 @llvm.umin.i32(i32 %38, i32 %28)
- %40 = sub nsw i32 %38, %39
- %41 = icmp sgt i32 %40, 7
- %42 = tail call i32 @llvm.umin.i32(i32 %36, i32 %29)
- %43 = sub nsw i32 %36, %42
- %44 = icmp sgt i32 %43, 7
- %45 = or disjoint i32 %39, %32
- %.idx = mul nuw nsw i32 %12, 4032
- %46 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx
- %47 = getelementptr bfloat, ptr addrspace(7) %46, i32 %45
- %48 = load <8 x bfloat>, ptr addrspace(7) %47, align 2
- %.idx4 = mul nuw nsw i32 %14, 4032
- %49 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx4
- %50 = getelementptr bfloat, ptr addrspace(7) %49, i32 %45
- %51 = load <8 x bfloat>, ptr addrspace(7) %50, align 2
- %.idx6 = mul nuw nsw i32 %16, 4032
- %52 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx6
- %53 = getelementptr bfloat, ptr addrspace(7) %52, i32 %45
- %54 = load <8 x bfloat>, ptr addrspace(7) %53, align 2
- %.idx8 = mul nuw nsw i32 %18, 4032
- %55 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx8
- %56 = getelementptr bfloat, ptr addrspace(7) %55, i32 %45
- %57 = load <8 x bfloat>, ptr addrspace(7) %56, align 2
- %58 = or disjoint i32 %42, %34
- %.idx10 = mul nuw nsw i32 %19, 4032
- %59 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx10
- %60 = getelementptr bfloat, ptr addrspace(7) %59, i32 %58
- %61 = load <8 x bfloat>, ptr addrspace(7) %60, align 2
- %.idx12 = mul nuw nsw i32 %20, 4032
- %62 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx12
- %63 = getelementptr bfloat, ptr addrspace(7) %62, i32 %58
- %64 = load <8 x bfloat>, ptr addrspace(7) %63, align 2
- %.idx14 = mul nuw nsw i32 %21, 4032
- %65 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx14
- %66 = getelementptr bfloat, ptr addrspace(7) %65, i32 %58
- %67 = load <8 x bfloat>, ptr addrspace(7) %66, align 2
- %.idx16 = mul nuw nsw i32 %22, 4032
- %68 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx16
- %69 = getelementptr bfloat, ptr addrspace(7) %68, i32 %58
- %70 = load <8 x bfloat>, ptr addrspace(7) %69, align 2
- %.idx18 = mul nuw nsw i32 %23, 4032
- %71 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx18
- %72 = getelementptr bfloat, ptr addrspace(7) %71, i32 %58
- %73 = load <8 x bfloat>, ptr addrspace(7) %72, align 2
- %.idx20 = mul nuw nsw i32 %24, 4032
- %74 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx20
- %75 = getelementptr bfloat, ptr addrspace(7) %74, i32 %58
- %76 = load <8 x bfloat>, ptr addrspace(7) %75, align 2
- %.idx22 = mul nuw nsw i32 %25, 4032
- %77 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx22
- %78 = getelementptr bfloat, ptr addrspace(7) %77, i32 %58
- %79 = load <8 x bfloat>, ptr addrspace(7) %78, align 2
- %.idx24 = mul nuw nsw i32 %26, 4032
- %80 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx24
- %81 = getelementptr bfloat, ptr addrspace(7) %80, i32 %58
- %82 = load <8 x bfloat>, ptr addrspace(7) %81, align 2
- %83 = select i1 %41, <8 x bfloat> %48, <8 x bfloat> zeroinitializer
- %.idx26 = mul nuw nsw i32 %12, 136
- %84 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx26
- %85 = getelementptr bfloat, ptr addrspace(3) %84, i32 %28
- store <8 x bfloat> %83, ptr addrspace(3) %85, align 8
- %86 = select i1 %41, <8 x bfloat> %51, <8 x bfloat> zeroinitializer
- %.idx28 = mul nuw nsw i32 %14, 136
- %87 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx28
- %88 = getelementptr bfloat, ptr addrspace(3) %87, i32 %28
- store <8 x bfloat> %86, ptr addrspace(3) %88, align 8
- %89 = select i1 %41, <8 x bfloat> %54, <8 x bfloat> zeroinitializer
- %.idx30 = mul nuw nsw i32 %16, 136
- %90 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx30
- %91 = getelementptr bfloat, ptr addrspace(3) %90, i32 %28
- store <8 x bfloat> %89, ptr addrspace(3) %91, align 8
- %92 = select i1 %41, <8 x bfloat> %57, <8 x bfloat> zeroinitializer
- %.idx32 = mul nuw nsw i32 %18, 136
- %93 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx32
- %94 = getelementptr bfloat, ptr addrspace(3) %93, i32 %28
- store <8 x bfloat> %92, ptr addrspace(3) %94, align 8
- %95 = select i1 %44, <8 x bfloat> %61, <8 x bfloat> zeroinitializer
- %.idx34 = mul nuw nsw i32 %19, 264
- %96 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx34
- %97 = getelementptr bfloat, ptr addrspace(3) %96, i32 %29
- store <8 x bfloat> %95, ptr addrspace(3) %97, align 8
- %98 = select i1 %44, <8 x bfloat> %64, <8 x bfloat> zeroinitializer
- %.idx36 = mul nuw nsw i32 %20, 264
- %99 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx36
- %100 = getelementptr bfloat, ptr addrspace(3) %99, i32 %29
- store <8 x bfloat> %98, ptr addrspace(3) %100, align 8
- %101 = select i1 %44, <8 x bfloat> %67, <8 x bfloat> zeroinitializer
- %.idx38 = mul nuw nsw i32 %21, 264
- %102 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx38
- %103 = getelementptr bfloat, ptr addrspace(3) %102, i32 %29
- store <8 x bfloat> %101, ptr addrspace(3) %103, align 8
- %104 = select i1 %44, <8 x bfloat> %70, <8 x bfloat> zeroinitializer
- %.idx40 = mul nuw nsw i32 %22, 264
- %105 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx40
- %106 = getelementptr bfloat, ptr addrspace(3) %105, i32 %29
- store <8 x bfloat> %104, ptr addrspace(3) %106, align 8
- %107 = select i1 %44, <8 x bfloat> %73, <8 x bfloat> zeroinitializer
- %.idx42 = mul nuw nsw i32 %23, 264
- %108 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx42
- %109 = getelementptr bfloat, ptr addrspace(3) %108, i32 %29
- store <8 x bfloat> %107, ptr addrspace(3) %109, align 8
- %110 = select i1 %44, <8 x bfloat> %76, <8 x bfloat> zeroinitializer
- %.idx44 = mul nuw nsw i32 %24, 264
- %111 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx44
- %112 = getelementptr bfloat, ptr addrspace(3) %111, i32 %29
- store <8 x bfloat> %110, ptr addrspace(3) %112, align 8
- %113 = select i1 %44, <8 x bfloat> %79, <8 x bfloat> zeroinitializer
- %.idx46 = mul nuw nsw i32 %25, 264
- %114 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx46
- %115 = getelementptr bfloat, ptr addrspace(3) %114, i32 %29
- store <8 x bfloat> %113, ptr addrspace(3) %115, align 8
- %116 = select i1 %44, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
- %.idx48 = mul nuw nsw i32 %26, 264
- %117 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx48
- %118 = getelementptr bfloat, ptr addrspace(3) %117, i32 %29
- store <8 x bfloat> %116, ptr addrspace(3) %118, align 8
- %119 = shl nuw nsw i32 %8, 5
- %120 = or disjoint i32 %119, %9
- %121 = or disjoint i32 %11, 1
- %122 = or disjoint i32 %11, 2
- %123 = or disjoint i32 %11, 3
- %124 = or disjoint i32 %11, 16
- %125 = or disjoint i32 %11, 17
- %126 = or disjoint i32 %11, 18
- %127 = or disjoint i32 %11, 19
- %128 = or disjoint i32 %11, 32
- %129 = or disjoint i32 %11, 33
- %130 = or disjoint i32 %11, 34
- %131 = or disjoint i32 %11, 35
- %132 = or disjoint i32 %11, 48
- %133 = or disjoint i32 %11, 49
- %134 = or disjoint i32 %11, 50
- %135 = or i32 %10, 51
- %136 = or disjoint i32 %11, 64
- %137 = or disjoint i32 %11, 65
- %138 = or disjoint i32 %11, 66
- %139 = or disjoint i32 %11, 67
- %140 = or disjoint i32 %11, 80
- %141 = or disjoint i32 %11, 81
- %142 = or disjoint i32 %11, 82
- %143 = or disjoint i32 %11, 83
- %144 = or disjoint i32 %11, 96
- %145 = or disjoint i32 %11, 97
- %146 = or disjoint i32 %11, 98
- %147 = or disjoint i32 %11, 99
- %148 = or disjoint i32 %11, 112
- %149 = or disjoint i32 %11, 113
- %150 = or disjoint i32 %11, 114
- %151 = or i32 %10, 115
- %152 = lshr i32 %4, 1
- %153 = and i32 %152, 64
- %154 = or disjoint i32 %153, %9
- %invariant.gep = getelementptr bfloat, ptr addrspace(7) %5, i32 %45
- %invariant.gep111 = getelementptr bfloat, ptr addrspace(7) %6, i32 %58
- %155 = mul nuw nsw i32 %11, 68
- %156 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
- %157 = getelementptr bfloat, ptr addrspace(3) %156, i32 %155
- %158 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
- %159 = getelementptr i8, ptr addrspace(3) %158, i32 32
- %160 = getelementptr bfloat, ptr addrspace(3) %159, i32 %155
- %161 = mul nuw nsw i32 %121, 68
- %162 = getelementptr bfloat, ptr addrspace(3) %156, i32 %161
- %163 = getelementptr bfloat, ptr addrspace(3) %159, i32 %161
- %164 = mul nuw nsw i32 %122, 68
- %165 = getelementptr bfloat, ptr addrspace(3) %156, i32 %164
- %166 = getelementptr bfloat, ptr addrspace(3) %159, i32 %164
- %167 = mul nuw nsw i32 %123, 68
- %168 = getelementptr bfloat, ptr addrspace(3) %156, i32 %167
- %169 = getelementptr bfloat, ptr addrspace(3) %159, i32 %167
- %170 = mul nuw nsw i32 %124, 68
- %171 = getelementptr bfloat, ptr addrspace(3) %156, i32 %170
- %172 = getelementptr bfloat, ptr addrspace(3) %159, i32 %170
- %173 = mul nuw nsw i32 %125, 68
- %174 = getelementptr bfloat, ptr addrspace(3) %156, i32 %173
- %175 = getelementptr bfloat, ptr addrspace(3) %159, i32 %173
- %176 = mul nuw nsw i32 %126, 68
- %177 = getelementptr bfloat, ptr addrspace(3) %156, i32 %176
- %178 = getelementptr bfloat, ptr addrspace(3) %159, i32 %176
- %179 = mul nuw nsw i32 %127, 68
- %180 = getelementptr bfloat, ptr addrspace(3) %156, i32 %179
- %181 = getelementptr bfloat, ptr addrspace(3) %159, i32 %179
- %182 = mul nuw nsw i32 %128, 68
- %183 = getelementptr bfloat, ptr addrspace(3) %156, i32 %182
- %184 = getelementptr bfloat, ptr addrspace(3) %159, i32 %182
- %185 = mul nuw nsw i32 %129, 68
- %186 = getelementptr bfloat, ptr addrspace(3) %156, i32 %185
- %187 = getelementptr bfloat, ptr addrspace(3) %159, i32 %185
- %188 = mul nuw nsw i32 %130, 68
- %189 = getelementptr bfloat, ptr addrspace(3) %156, i32 %188
- %190 = getelementptr bfloat, ptr addrspace(3) %159, i32 %188
- %191 = mul nuw nsw i32 %131, 68
- %192 = getelementptr bfloat, ptr addrspace(3) %156, i32 %191
- %193 = getelementptr bfloat, ptr addrspace(3) %159, i32 %191
- %194 = mul nuw nsw i32 %132, 68
- %195 = getelementptr bfloat, ptr addrspace(3) %156, i32 %194
- %196 = getelementptr bfloat, ptr addrspace(3) %159, i32 %194
- %197 = mul nuw nsw i32 %133, 68
- %198 = getelementptr bfloat, ptr addrspace(3) %156, i32 %197
- %199 = getelementptr bfloat, ptr addrspace(3) %159, i32 %197
- %200 = mul nuw nsw i32 %134, 68
- %201 = getelementptr bfloat, ptr addrspace(3) %156, i32 %200
- %202 = getelementptr bfloat, ptr addrspace(3) %159, i32 %200
- %203 = mul nuw nsw i32 %135, 68
- %204 = getelementptr bfloat, ptr addrspace(3) %156, i32 %203
- %205 = getelementptr bfloat, ptr addrspace(3) %159, i32 %203
- %206 = mul nuw nsw i32 %136, 68
- %207 = getelementptr bfloat, ptr addrspace(3) %156, i32 %206
- %208 = getelementptr bfloat, ptr addrspace(3) %159, i32 %206
- %209 = mul nuw nsw i32 %137, 68
- %210 = getelementptr bfloat, ptr addrspace(3) %156, i32 %209
- %211 = getelementptr bfloat, ptr addrspace(3) %159, i32 %209
- %212 = mul nuw nsw i32 %138, 68
- %213 = getelementptr bfloat, ptr addrspace(3) %156, i32 %212
- %214 = getelementptr bfloat, ptr addrspace(3) %159, i32 %212
- %215 = mul nuw nsw i32 %139, 68
- %216 = getelementptr bfloat, ptr addrspace(3) %156, i32 %215
- %217 = getelementptr bfloat, ptr addrspace(3) %159, i32 %215
- %218 = mul nuw nsw i32 %140, 68
- %219 = getelementptr bfloat, ptr addrspace(3) %156, i32 %218
- %220 = getelementptr bfloat, ptr addrspace(3) %159, i32 %218
- %221 = mul nuw nsw i32 %141, 68
- %222 = getelementptr bfloat, ptr addrspace(3) %156, i32 %221
- %223 = getelementptr bfloat, ptr addrspace(3) %159, i32 %221
- %224 = mul nuw nsw i32 %142, 68
- %225 = getelementptr bfloat, ptr addrspace(3) %156, i32 %224
- %226 = getelementptr bfloat, ptr addrspace(3) %159, i32 %224
- %227 = mul nuw nsw i32 %143, 68
- %228 = getelementptr bfloat, ptr addrspace(3) %156, i32 %227
- %229 = getelementptr bfloat, ptr addrspace(3) %159, i32 %227
- %230 = mul nuw nsw i32 %144, 68
- %231 = getelementptr bfloat, ptr addrspace(3) %156, i32 %230
- %232 = getelementptr bfloat, ptr addrspace(3) %159, i32 %230
- %233 = mul nuw nsw i32 %145, 68
- %234 = getelementptr bfloat, ptr addrspace(3) %156, i32 %233
- %235 = getelementptr bfloat, ptr addrspace(3) %159, i32 %233
- %236 = mul nuw nsw i32 %146, 68
- %237 = getelementptr bfloat, ptr addrspace(3) %156, i32 %236
- %238 = getelementptr bfloat, ptr addrspace(3) %159, i32 %236
- %239 = mul nuw nsw i32 %147, 68
- %240 = getelementptr bfloat, ptr addrspace(3) %156, i32 %239
- %241 = getelementptr bfloat, ptr addrspace(3) %159, i32 %239
- %242 = mul nuw nsw i32 %148, 68
- %243 = getelementptr bfloat, ptr addrspace(3) %156, i32 %242
- %244 = getelementptr bfloat, ptr addrspace(3) %159, i32 %242
- %245 = mul nuw nsw i32 %149, 68
- %246 = getelementptr bfloat, ptr addrspace(3) %156, i32 %245
- %247 = getelementptr bfloat, ptr addrspace(3) %159, i32 %245
- %248 = mul nuw nsw i32 %150, 68
- %249 = getelementptr bfloat, ptr addrspace(3) %156, i32 %248
- %250 = getelementptr bfloat, ptr addrspace(3) %159, i32 %248
- %251 = mul nuw nsw i32 %151, 68
- %252 = getelementptr bfloat, ptr addrspace(3) %156, i32 %251
- %253 = getelementptr bfloat, ptr addrspace(3) %159, i32 %251
- %narrow = mul nuw nsw i32 %11, 132
- %254 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %255 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow
- %256 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %257 = getelementptr i8, ptr addrspace(3) %256, i32 32
- %258 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow
- %259 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %260 = getelementptr i8, ptr addrspace(3) %259, i32 64
- %261 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow
- %262 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %263 = getelementptr i8, ptr addrspace(3) %262, i32 96
- %264 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow
- %narrow74 = mul nuw nsw i32 %121, 132
- %265 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow74
- %266 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow74
- %267 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow74
- %268 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow74
- %narrow75 = mul nuw nsw i32 %122, 132
- %269 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow75
- %270 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow75
- %271 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow75
- %272 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow75
- %narrow76 = mul nuw nsw i32 %123, 132
- %273 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow76
- %274 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow76
- %275 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow76
- %276 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow76
- %narrow77 = mul nuw nsw i32 %124, 132
- %277 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow77
- %278 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow77
- %279 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow77
- %280 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow77
- %narrow78 = mul nuw nsw i32 %125, 132
- %281 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow78
- %282 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow78
- %283 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow78
- %284 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow78
- %narrow79 = mul nuw nsw i32 %126, 132
- %285 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow79
- %286 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow79
- %287 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow79
- %288 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow79
- %narrow80 = mul nuw nsw i32 %127, 132
- %289 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow80
- %290 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow80
- %291 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow80
- %292 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow80
- %narrow81 = mul nuw nsw i32 %128, 132
- %293 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow81
- %294 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow81
- %295 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow81
- %296 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow81
- %narrow82 = mul nuw nsw i32 %129, 132
- %297 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow82
- %298 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow82
- %299 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow82
- %300 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow82
- %narrow83 = mul nuw nsw i32 %130, 132
- %301 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow83
- %302 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow83
- %303 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow83
- %304 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow83
- %narrow84 = mul nuw nsw i32 %131, 132
- %305 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow84
- %306 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow84
- %307 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow84
- %308 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow84
- %narrow85 = mul nuw nsw i32 %132, 132
- %309 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow85
- %310 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow85
- %311 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow85
- %312 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow85
- %narrow86 = mul nuw nsw i32 %133, 132
- %313 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow86
- %314 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow86
- %315 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow86
- %316 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow86
- %narrow87 = mul nuw nsw i32 %134, 132
- %317 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow87
- %318 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow87
- %319 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow87
- %320 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow87
- %narrow88 = mul nuw nsw i32 %135, 132
- %321 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow88
- %322 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow88
- %323 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow88
- %324 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow88
- %narrow89 = mul nuw nsw i32 %136, 132
- %325 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow89
- %326 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow89
- %327 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow89
- %328 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow89
- %narrow90 = mul nuw nsw i32 %137, 132
- %329 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow90
- %330 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow90
- %331 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow90
- %332 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow90
- %narrow91 = mul nuw nsw i32 %138, 132
- %333 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow91
- %334 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow91
- %335 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow91
- %336 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow91
- %narrow92 = mul nuw nsw i32 %139, 132
- %337 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow92
- %338 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow92
- %339 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow92
- %340 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow92
- %narrow93 = mul nuw nsw i32 %140, 132
- %341 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow93
- %342 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow93
- %343 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow93
- %344 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow93
- %narrow94 = mul nuw nsw i32 %141, 132
- %345 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow94
- %346 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow94
- %347 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow94
- %348 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow94
- %narrow95 = mul nuw nsw i32 %142, 132
- %349 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow95
- %350 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow95
- %351 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow95
- %352 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow95
- %narrow96 = mul nuw nsw i32 %143, 132
- %353 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow96
- %354 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow96
- %355 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow96
- %356 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow96
- %narrow97 = mul nuw nsw i32 %144, 132
- %357 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow97
- %358 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow97
- %359 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow97
- %360 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow97
- %narrow98 = mul nuw nsw i32 %145, 132
- %361 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow98
- %362 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow98
- %363 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow98
- %364 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow98
- %narrow99 = mul nuw nsw i32 %146, 132
- %365 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow99
- %366 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow99
- %367 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow99
- %368 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow99
- %narrow100 = mul nuw nsw i32 %147, 132
- %369 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow100
- %370 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow100
- %371 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow100
- %372 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow100
- %narrow101 = mul nuw nsw i32 %148, 132
- %373 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow101
- %374 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow101
- %375 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow101
- %376 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow101
- %narrow102 = mul nuw nsw i32 %149, 132
- %377 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow102
- %378 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow102
- %379 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow102
- %380 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow102
- %narrow103 = mul nuw nsw i32 %150, 132
- %381 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow103
- %382 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow103
- %383 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow103
- %384 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow103
- %narrow104 = mul nuw nsw i32 %151, 132
- %385 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow104
- %386 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow104
- %387 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow104
- %388 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow104
- br label %389
-
-389: ; preds = %3, %389
- %390 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %3 ], [ %1015, %389 ]
- %391 = phi i32 [ 0, %3 ], [ %392, %389 ]
- %392 = add nuw nsw i32 %391, 8
- %393 = shl nuw nsw i32 %392, 4
- %394 = sub nuw nsw i32 1380, %393
- %395 = tail call i32 @llvm.umin.i32(i32 %394, i32 128)
- %396 = tail call i32 @llvm.umin.i32(i32 %395, i32 %12)
- %397 = sub nsw i32 %395, %396
- %398 = tail call i32 @llvm.smin.i32(i32 %397, i32 1)
- %399 = or disjoint i32 %396, %393
- %.idx50 = mul nuw nsw i32 %399, 4032
- %gep = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx50
- %400 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
- %401 = tail call i32 @llvm.umin.i32(i32 %395, i32 %14)
- %402 = sub nsw i32 %395, %401
- %403 = tail call i32 @llvm.smin.i32(i32 %402, i32 1)
- %404 = or disjoint i32 %401, %393
- %.idx52 = mul nuw nsw i32 %404, 4032
- %gep106 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx52
- %405 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
- %406 = tail call i32 @llvm.umin.i32(i32 %395, i32 %16)
- %407 = sub nsw i32 %395, %406
- %408 = tail call i32 @llvm.smin.i32(i32 %407, i32 1)
- %409 = or disjoint i32 %406, %393
- %.idx54 = mul nuw nsw i32 %409, 4032
- %gep108 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx54
- %410 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
- %411 = tail call i32 @llvm.umin.i32(i32 %395, i32 %18)
- %412 = sub nsw i32 %395, %411
- %413 = tail call i32 @llvm.smin.i32(i32 %412, i32 1)
- %414 = or disjoint i32 %411, %393
- %.idx56 = mul nuw nsw i32 %414, 4032
- %gep110 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx56
- %415 = load <8 x bfloat>, ptr addrspace(7) %gep110, align 2
- %416 = tail call i32 @llvm.umin.i32(i32 %395, i32 %19)
- %417 = sub nsw i32 %395, %416
- %418 = tail call i32 @llvm.smin.i32(i32 %417, i32 1)
- %419 = or disjoint i32 %416, %393
- %.idx58 = mul nuw nsw i32 %419, 4032
- %gep112 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx58
- %420 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
- %421 = tail call i32 @llvm.umin.i32(i32 %395, i32 %20)
- %422 = sub nsw i32 %395, %421
- %423 = tail call i32 @llvm.smin.i32(i32 %422, i32 1)
- %424 = or disjoint i32 %421, %393
- %.idx60 = mul nuw nsw i32 %424, 4032
- %gep114 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx60
- %425 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
- %426 = tail call i32 @llvm.umin.i32(i32 %395, i32 %21)
- %427 = sub nsw i32 %395, %426
- %428 = tail call i32 @llvm.smin.i32(i32 %427, i32 1)
- %429 = or disjoint i32 %426, %393
- %.idx62 = mul nuw nsw i32 %429, 4032
- %gep116 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx62
- %430 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
- %431 = tail call i32 @llvm.umin.i32(i32 %395, i32 %22)
- %432 = sub nsw i32 %395, %431
- %433 = tail call i32 @llvm.smin.i32(i32 %432, i32 1)
- %434 = or disjoint i32 %431, %393
- %.idx64 = mul nuw nsw i32 %434, 4032
- %gep118 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx64
- %435 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
- %436 = tail call i32 @llvm.umin.i32(i32 %395, i32 %23)
- %437 = sub nsw i32 %395, %436
- %438 = tail call i32 @llvm.smin.i32(i32 %437, i32 1)
- %439 = or disjoint i32 %436, %393
- %.idx66 = mul nuw nsw i32 %439, 4032
- %gep120 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx66
- %440 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
- %441 = tail call i32 @llvm.umin.i32(i32 %395, i32 %24)
- %442 = sub nsw i32 %395, %441
- %443 = tail call i32 @llvm.smin.i32(i32 %442, i32 1)
- %444 = or disjoint i32 %441, %393
- %.idx68 = mul nuw nsw i32 %444, 4032
- %gep122 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx68
- %445 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
- %446 = tail call i32 @llvm.umin.i32(i32 %395, i32 %25)
- %447 = sub nsw i32 %395, %446
- %448 = tail call i32 @llvm.smin.i32(i32 %447, i32 1)
- %449 = or disjoint i32 %446, %393
- %.idx70 = mul nuw nsw i32 %449, 4032
- %gep124 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx70
- %450 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
- %451 = tail call i32 @llvm.umin.i32(i32 %395, i32 %26)
- %452 = sub nsw i32 %395, %451
- %453 = tail call i32 @llvm.smin.i32(i32 %452, i32 1)
- %454 = or disjoint i32 %451, %393
- %.idx72 = mul nuw nsw i32 %454, 4032
- %gep126 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx72
- %455 = load <8 x bfloat>, ptr addrspace(7) %gep126, align 2
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
- %456 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
- %457 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
- %458 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
- %459 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
- %460 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
- %461 = shufflevector <1 x bfloat> %460, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %462 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
- %463 = shufflevector <1 x bfloat> %462, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %464 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
- %465 = shufflevector <1 x bfloat> %464, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %466 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
- %467 = shufflevector <1 x bfloat> %466, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %468 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
- %469 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
- %470 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
- %471 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
- %472 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
- %473 = shufflevector <1 x bfloat> %472, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %474 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
- %475 = shufflevector <1 x bfloat> %474, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %476 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
- %477 = shufflevector <1 x bfloat> %476, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %478 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
- %479 = shufflevector <1 x bfloat> %478, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %480 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
- %481 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
- %482 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
- %483 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
- %484 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
- %485 = shufflevector <1 x bfloat> %484, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %486 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
- %487 = shufflevector <1 x bfloat> %486, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %488 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
- %489 = shufflevector <1 x bfloat> %488, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %490 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
- %491 = shufflevector <1 x bfloat> %490, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %492 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
- %493 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
- %494 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
- %495 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
- %496 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
- %497 = shufflevector <1 x bfloat> %496, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %498 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
- %499 = shufflevector <1 x bfloat> %498, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %500 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
- %501 = shufflevector <1 x bfloat> %500, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %502 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
- %503 = shufflevector <1 x bfloat> %502, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %504 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
- %505 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
- %506 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
- %507 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
- %508 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
- %509 = shufflevector <1 x bfloat> %508, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %510 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
- %511 = shufflevector <1 x bfloat> %510, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %512 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
- %513 = shufflevector <1 x bfloat> %512, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %514 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
- %515 = shufflevector <1 x bfloat> %514, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %516 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
- %517 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
- %518 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
- %519 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
- %520 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
- %521 = shufflevector <1 x bfloat> %520, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %522 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
- %523 = shufflevector <1 x bfloat> %522, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %524 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
- %525 = shufflevector <1 x bfloat> %524, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %526 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
- %527 = shufflevector <1 x bfloat> %526, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %528 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
- %529 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
- %530 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
- %531 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
- %532 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
- %533 = shufflevector <1 x bfloat> %532, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %534 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
- %535 = shufflevector <1 x bfloat> %534, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %536 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
- %537 = shufflevector <1 x bfloat> %536, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %538 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
- %539 = shufflevector <1 x bfloat> %538, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %540 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
- %541 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
- %542 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
- %543 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
- %544 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
- %545 = shufflevector <1 x bfloat> %544, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %546 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
- %547 = shufflevector <1 x bfloat> %546, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %548 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
- %549 = shufflevector <1 x bfloat> %548, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %550 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
- %551 = shufflevector <1 x bfloat> %550, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %552 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
- %553 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
- %554 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
- %555 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
- %556 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
- %557 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
- %558 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
- %559 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
- %560 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
- %561 = shufflevector <1 x bfloat> %560, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %562 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
- %563 = shufflevector <1 x bfloat> %562, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %564 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
- %565 = shufflevector <1 x bfloat> %564, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %566 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
- %567 = shufflevector <1 x bfloat> %566, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %568 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
- %569 = shufflevector <1 x bfloat> %568, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %570 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
- %571 = shufflevector <1 x bfloat> %570, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %572 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
- %573 = shufflevector <1 x bfloat> %572, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %574 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
- %575 = shufflevector <1 x bfloat> %574, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %576 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
- %577 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
- %578 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
- %579 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
- %580 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
- %581 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
- %582 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
- %583 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
- %584 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
- %585 = shufflevector <1 x bfloat> %584, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %586 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
- %587 = shufflevector <1 x bfloat> %586, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %588 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
- %589 = shufflevector <1 x bfloat> %588, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %590 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
- %591 = shufflevector <1 x bfloat> %590, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %592 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
- %593 = shufflevector <1 x bfloat> %592, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %594 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
- %595 = shufflevector <1 x bfloat> %594, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %596 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
- %597 = shufflevector <1 x bfloat> %596, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %598 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
- %599 = shufflevector <1 x bfloat> %598, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %600 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
- %601 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
- %602 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
- %603 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
- %604 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
- %605 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
- %606 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
- %607 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
- %608 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
- %609 = shufflevector <1 x bfloat> %608, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %610 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
- %611 = shufflevector <1 x bfloat> %610, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %612 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
- %613 = shufflevector <1 x bfloat> %612, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %614 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
- %615 = shufflevector <1 x bfloat> %614, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %616 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
- %617 = shufflevector <1 x bfloat> %616, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %618 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
- %619 = shufflevector <1 x bfloat> %618, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %620 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
- %621 = shufflevector <1 x bfloat> %620, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %622 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
- %623 = shufflevector <1 x bfloat> %622, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %624 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
- %625 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
- %626 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
- %627 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
- %628 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
- %629 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
- %630 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
- %631 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
- %632 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
- %633 = shufflevector <1 x bfloat> %632, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %634 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
- %635 = shufflevector <1 x bfloat> %634, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %636 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
- %637 = shufflevector <1 x bfloat> %636, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %638 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
- %639 = shufflevector <1 x bfloat> %638, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %640 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
- %641 = shufflevector <1 x bfloat> %640, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %642 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
- %643 = shufflevector <1 x bfloat> %642, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %644 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
- %645 = shufflevector <1 x bfloat> %644, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %646 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
- %647 = shufflevector <1 x bfloat> %646, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %648 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
- %649 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
- %650 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
- %651 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
- %652 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
- %653 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
- %654 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
- %655 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
- %656 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
- %657 = shufflevector <1 x bfloat> %656, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %658 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
- %659 = shufflevector <1 x bfloat> %658, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %660 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
- %661 = shufflevector <1 x bfloat> %660, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %662 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
- %663 = shufflevector <1 x bfloat> %662, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %664 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
- %665 = shufflevector <1 x bfloat> %664, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %666 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
- %667 = shufflevector <1 x bfloat> %666, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %668 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
- %669 = shufflevector <1 x bfloat> %668, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %670 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
- %671 = shufflevector <1 x bfloat> %670, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %672 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
- %673 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
- %674 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
- %675 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
- %676 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
- %677 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
- %678 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
- %679 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
- %680 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
- %681 = shufflevector <1 x bfloat> %680, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %682 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
- %683 = shufflevector <1 x bfloat> %682, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %684 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
- %685 = shufflevector <1 x bfloat> %684, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %686 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
- %687 = shufflevector <1 x bfloat> %686, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %688 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
- %689 = shufflevector <1 x bfloat> %688, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %690 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
- %691 = shufflevector <1 x bfloat> %690, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %692 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
- %693 = shufflevector <1 x bfloat> %692, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %694 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
- %695 = shufflevector <1 x bfloat> %694, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %696 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
- %697 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
- %698 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
- %699 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
- %700 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
- %701 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
- %702 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
- %703 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
- %704 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
- %705 = shufflevector <1 x bfloat> %704, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %706 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
- %707 = shufflevector <1 x bfloat> %706, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %708 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
- %709 = shufflevector <1 x bfloat> %708, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %710 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
- %711 = shufflevector <1 x bfloat> %710, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %712 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
- %713 = shufflevector <1 x bfloat> %712, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %714 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
- %715 = shufflevector <1 x bfloat> %714, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %716 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
- %717 = shufflevector <1 x bfloat> %716, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %718 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
- %719 = shufflevector <1 x bfloat> %718, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %720 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
- %721 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
- %722 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
- %723 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
- %724 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
- %725 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
- %726 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
- %727 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
- %728 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
- %729 = shufflevector <1 x bfloat> %728, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %730 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
- %731 = shufflevector <1 x bfloat> %730, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %732 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
- %733 = shufflevector <1 x bfloat> %732, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %734 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
- %735 = shufflevector <1 x bfloat> %734, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %736 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
- %737 = shufflevector <1 x bfloat> %736, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %738 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
- %739 = shufflevector <1 x bfloat> %738, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %740 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
- %741 = shufflevector <1 x bfloat> %740, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %742 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
- %743 = shufflevector <1 x bfloat> %742, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %744 = shufflevector <1 x bfloat> %456, <1 x bfloat> %458, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %745 = shufflevector <4 x bfloat> %744, <4 x bfloat> %461, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %746 = shufflevector <4 x bfloat> %745, <4 x bfloat> %465, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %747 = shufflevector <1 x bfloat> %552, <1 x bfloat> %556, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %748 = shufflevector <4 x bfloat> %747, <4 x bfloat> %561, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %749 = shufflevector <4 x bfloat> %748, <4 x bfloat> %569, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %750 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 0, 0
- %751 = bitcast <4 x bfloat> %746 to <4 x i16>
- %752 = bitcast <4 x bfloat> %749 to <4 x i16>
- %753 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %752, <4 x float> %750, i32 0, i32 0, i32 0)
- %754 = shufflevector <1 x bfloat> %553, <1 x bfloat> %557, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %755 = shufflevector <4 x bfloat> %754, <4 x bfloat> %563, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %756 = shufflevector <4 x bfloat> %755, <4 x bfloat> %571, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %757 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 0, 0
- %758 = bitcast <4 x bfloat> %756 to <4 x i16>
- %759 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %758, <4 x float> %757, i32 0, i32 0, i32 0)
- %760 = shufflevector <1 x bfloat> %554, <1 x bfloat> %558, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %761 = shufflevector <4 x bfloat> %760, <4 x bfloat> %565, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %762 = shufflevector <4 x bfloat> %761, <4 x bfloat> %573, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %763 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 0, 0
- %764 = bitcast <4 x bfloat> %762 to <4 x i16>
- %765 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %764, <4 x float> %763, i32 0, i32 0, i32 0)
- %766 = shufflevector <1 x bfloat> %555, <1 x bfloat> %559, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %767 = shufflevector <4 x bfloat> %766, <4 x bfloat> %567, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %768 = shufflevector <4 x bfloat> %767, <4 x bfloat> %575, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %769 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 0, 0
- %770 = bitcast <4 x bfloat> %768 to <4 x i16>
- %771 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %770, <4 x float> %769, i32 0, i32 0, i32 0)
- %772 = shufflevector <1 x bfloat> %457, <1 x bfloat> %459, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %773 = shufflevector <4 x bfloat> %772, <4 x bfloat> %463, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %774 = shufflevector <4 x bfloat> %773, <4 x bfloat> %467, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %775 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 1, 0
- %776 = bitcast <4 x bfloat> %774 to <4 x i16>
- %777 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %752, <4 x float> %775, i32 0, i32 0, i32 0)
- %778 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 1, 0
- %779 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %758, <4 x float> %778, i32 0, i32 0, i32 0)
- %780 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 1, 0
- %781 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %764, <4 x float> %780, i32 0, i32 0, i32 0)
- %782 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 1, 0
- %783 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %770, <4 x float> %782, i32 0, i32 0, i32 0)
- %784 = shufflevector <1 x bfloat> %468, <1 x bfloat> %470, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %785 = shufflevector <4 x bfloat> %784, <4 x bfloat> %473, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %786 = shufflevector <4 x bfloat> %785, <4 x bfloat> %477, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %787 = shufflevector <1 x bfloat> %576, <1 x bfloat> %580, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %788 = shufflevector <4 x bfloat> %787, <4 x bfloat> %585, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %789 = shufflevector <4 x bfloat> %788, <4 x bfloat> %593, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %790 = bitcast <4 x bfloat> %786 to <4 x i16>
- %791 = bitcast <4 x bfloat> %789 to <4 x i16>
- %792 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %791, <4 x float> %753, i32 0, i32 0, i32 0)
- %793 = shufflevector <1 x bfloat> %577, <1 x bfloat> %581, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %794 = shufflevector <4 x bfloat> %793, <4 x bfloat> %587, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %795 = shufflevector <4 x bfloat> %794, <4 x bfloat> %595, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %796 = bitcast <4 x bfloat> %795 to <4 x i16>
- %797 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %796, <4 x float> %759, i32 0, i32 0, i32 0)
- %798 = shufflevector <1 x bfloat> %578, <1 x bfloat> %582, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %799 = shufflevector <4 x bfloat> %798, <4 x bfloat> %589, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %800 = shufflevector <4 x bfloat> %799, <4 x bfloat> %597, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %801 = bitcast <4 x bfloat> %800 to <4 x i16>
- %802 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %801, <4 x float> %765, i32 0, i32 0, i32 0)
- %803 = shufflevector <1 x bfloat> %579, <1 x bfloat> %583, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %804 = shufflevector <4 x bfloat> %803, <4 x bfloat> %591, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %805 = shufflevector <4 x bfloat> %804, <4 x bfloat> %599, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %806 = bitcast <4 x bfloat> %805 to <4 x i16>
- %807 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %806, <4 x float> %771, i32 0, i32 0, i32 0)
- %808 = shufflevector <1 x bfloat> %469, <1 x bfloat> %471, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %809 = shufflevector <4 x bfloat> %808, <4 x bfloat> %475, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %810 = shufflevector <4 x bfloat> %809, <4 x bfloat> %479, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %811 = bitcast <4 x bfloat> %810 to <4 x i16>
- %812 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %791, <4 x float> %777, i32 0, i32 0, i32 0)
- %813 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %796, <4 x float> %779, i32 0, i32 0, i32 0)
- %814 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %801, <4 x float> %781, i32 0, i32 0, i32 0)
- %815 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %806, <4 x float> %783, i32 0, i32 0, i32 0)
- %816 = shufflevector <1 x bfloat> %480, <1 x bfloat> %482, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %817 = shufflevector <4 x bfloat> %816, <4 x bfloat> %485, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %818 = shufflevector <4 x bfloat> %817, <4 x bfloat> %489, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %819 = shufflevector <1 x bfloat> %600, <1 x bfloat> %604, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %820 = shufflevector <4 x bfloat> %819, <4 x bfloat> %609, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %821 = shufflevector <4 x bfloat> %820, <4 x bfloat> %617, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %822 = bitcast <4 x bfloat> %818 to <4 x i16>
- %823 = bitcast <4 x bfloat> %821 to <4 x i16>
- %824 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %823, <4 x float> %792, i32 0, i32 0, i32 0)
- %825 = shufflevector <1 x bfloat> %601, <1 x bfloat> %605, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %826 = shufflevector <4 x bfloat> %825, <4 x bfloat> %611, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %827 = shufflevector <4 x bfloat> %826, <4 x bfloat> %619, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %828 = bitcast <4 x bfloat> %827 to <4 x i16>
- %829 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %828, <4 x float> %797, i32 0, i32 0, i32 0)
- %830 = shufflevector <1 x bfloat> %602, <1 x bfloat> %606, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %831 = shufflevector <4 x bfloat> %830, <4 x bfloat> %613, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %832 = shufflevector <4 x bfloat> %831, <4 x bfloat> %621, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %833 = bitcast <4 x bfloat> %832 to <4 x i16>
- %834 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %833, <4 x float> %802, i32 0, i32 0, i32 0)
- %835 = shufflevector <1 x bfloat> %603, <1 x bfloat> %607, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %836 = shufflevector <4 x bfloat> %835, <4 x bfloat> %615, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %837 = shufflevector <4 x bfloat> %836, <4 x bfloat> %623, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %838 = bitcast <4 x bfloat> %837 to <4 x i16>
- %839 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %838, <4 x float> %807, i32 0, i32 0, i32 0)
- %840 = shufflevector <1 x bfloat> %481, <1 x bfloat> %483, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %841 = shufflevector <4 x bfloat> %840, <4 x bfloat> %487, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %842 = shufflevector <4 x bfloat> %841, <4 x bfloat> %491, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %843 = bitcast <4 x bfloat> %842 to <4 x i16>
- %844 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %823, <4 x float> %812, i32 0, i32 0, i32 0)
- %845 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %828, <4 x float> %813, i32 0, i32 0, i32 0)
- %846 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %833, <4 x float> %814, i32 0, i32 0, i32 0)
- %847 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %838, <4 x float> %815, i32 0, i32 0, i32 0)
- %848 = shufflevector <1 x bfloat> %492, <1 x bfloat> %494, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %849 = shufflevector <4 x bfloat> %848, <4 x bfloat> %497, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %850 = shufflevector <4 x bfloat> %849, <4 x bfloat> %501, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %851 = shufflevector <1 x bfloat> %624, <1 x bfloat> %628, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %852 = shufflevector <4 x bfloat> %851, <4 x bfloat> %633, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %853 = shufflevector <4 x bfloat> %852, <4 x bfloat> %641, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %854 = bitcast <4 x bfloat> %850 to <4 x i16>
- %855 = bitcast <4 x bfloat> %853 to <4 x i16>
- %856 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %855, <4 x float> %824, i32 0, i32 0, i32 0)
- %857 = shufflevector <1 x bfloat> %625, <1 x bfloat> %629, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %858 = shufflevector <4 x bfloat> %857, <4 x bfloat> %635, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %859 = shufflevector <4 x bfloat> %858, <4 x bfloat> %643, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %860 = bitcast <4 x bfloat> %859 to <4 x i16>
- %861 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %860, <4 x float> %829, i32 0, i32 0, i32 0)
- %862 = shufflevector <1 x bfloat> %626, <1 x bfloat> %630, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %863 = shufflevector <4 x bfloat> %862, <4 x bfloat> %637, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %864 = shufflevector <4 x bfloat> %863, <4 x bfloat> %645, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %865 = bitcast <4 x bfloat> %864 to <4 x i16>
- %866 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %865, <4 x float> %834, i32 0, i32 0, i32 0)
- %867 = shufflevector <1 x bfloat> %627, <1 x bfloat> %631, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %868 = shufflevector <4 x bfloat> %867, <4 x bfloat> %639, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %869 = shufflevector <4 x bfloat> %868, <4 x bfloat> %647, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %870 = bitcast <4 x bfloat> %869 to <4 x i16>
- %871 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %870, <4 x float> %839, i32 0, i32 0, i32 0)
- %872 = shufflevector <1 x bfloat> %493, <1 x bfloat> %495, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %873 = shufflevector <4 x bfloat> %872, <4 x bfloat> %499, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %874 = shufflevector <4 x bfloat> %873, <4 x bfloat> %503, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %875 = bitcast <4 x bfloat> %874 to <4 x i16>
- %876 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %855, <4 x float> %844, i32 0, i32 0, i32 0)
- %877 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %860, <4 x float> %845, i32 0, i32 0, i32 0)
- %878 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %865, <4 x float> %846, i32 0, i32 0, i32 0)
- %879 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %870, <4 x float> %847, i32 0, i32 0, i32 0)
- %880 = shufflevector <1 x bfloat> %504, <1 x bfloat> %506, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %881 = shufflevector <4 x bfloat> %880, <4 x bfloat> %509, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %882 = shufflevector <4 x bfloat> %881, <4 x bfloat> %513, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %883 = shufflevector <1 x bfloat> %648, <1 x bfloat> %652, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %884 = shufflevector <4 x bfloat> %883, <4 x bfloat> %657, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %885 = shufflevector <4 x bfloat> %884, <4 x bfloat> %665, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %886 = bitcast <4 x bfloat> %882 to <4 x i16>
- %887 = bitcast <4 x bfloat> %885 to <4 x i16>
- %888 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %887, <4 x float> %856, i32 0, i32 0, i32 0)
- %889 = shufflevector <1 x bfloat> %649, <1 x bfloat> %653, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %890 = shufflevector <4 x bfloat> %889, <4 x bfloat> %659, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %891 = shufflevector <4 x bfloat> %890, <4 x bfloat> %667, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %892 = bitcast <4 x bfloat> %891 to <4 x i16>
- %893 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %892, <4 x float> %861, i32 0, i32 0, i32 0)
- %894 = shufflevector <1 x bfloat> %650, <1 x bfloat> %654, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %895 = shufflevector <4 x bfloat> %894, <4 x bfloat> %661, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %896 = shufflevector <4 x bfloat> %895, <4 x bfloat> %669, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %897 = bitcast <4 x bfloat> %896 to <4 x i16>
- %898 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %897, <4 x float> %866, i32 0, i32 0, i32 0)
- %899 = shufflevector <1 x bfloat> %651, <1 x bfloat> %655, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %900 = shufflevector <4 x bfloat> %899, <4 x bfloat> %663, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %901 = shufflevector <4 x bfloat> %900, <4 x bfloat> %671, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %902 = bitcast <4 x bfloat> %901 to <4 x i16>
- %903 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %902, <4 x float> %871, i32 0, i32 0, i32 0)
- %904 = shufflevector <1 x bfloat> %505, <1 x bfloat> %507, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %905 = shufflevector <4 x bfloat> %904, <4 x bfloat> %511, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %906 = shufflevector <4 x bfloat> %905, <4 x bfloat> %515, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %907 = bitcast <4 x bfloat> %906 to <4 x i16>
- %908 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %887, <4 x float> %876, i32 0, i32 0, i32 0)
- %909 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %892, <4 x float> %877, i32 0, i32 0, i32 0)
- %910 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %897, <4 x float> %878, i32 0, i32 0, i32 0)
- %911 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %902, <4 x float> %879, i32 0, i32 0, i32 0)
- %912 = shufflevector <1 x bfloat> %516, <1 x bfloat> %518, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %913 = shufflevector <4 x bfloat> %912, <4 x bfloat> %521, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %914 = shufflevector <4 x bfloat> %913, <4 x bfloat> %525, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %915 = shufflevector <1 x bfloat> %672, <1 x bfloat> %676, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %916 = shufflevector <4 x bfloat> %915, <4 x bfloat> %681, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %917 = shufflevector <4 x bfloat> %916, <4 x bfloat> %689, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %918 = bitcast <4 x bfloat> %914 to <4 x i16>
- %919 = bitcast <4 x bfloat> %917 to <4 x i16>
- %920 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %919, <4 x float> %888, i32 0, i32 0, i32 0)
- %921 = shufflevector <1 x bfloat> %673, <1 x bfloat> %677, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %922 = shufflevector <4 x bfloat> %921, <4 x bfloat> %683, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %923 = shufflevector <4 x bfloat> %922, <4 x bfloat> %691, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %924 = bitcast <4 x bfloat> %923 to <4 x i16>
- %925 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %924, <4 x float> %893, i32 0, i32 0, i32 0)
- %926 = shufflevector <1 x bfloat> %674, <1 x bfloat> %678, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %927 = shufflevector <4 x bfloat> %926, <4 x bfloat> %685, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %928 = shufflevector <4 x bfloat> %927, <4 x bfloat> %693, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %929 = bitcast <4 x bfloat> %928 to <4 x i16>
- %930 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %929, <4 x float> %898, i32 0, i32 0, i32 0)
- %931 = shufflevector <1 x bfloat> %675, <1 x bfloat> %679, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %932 = shufflevector <4 x bfloat> %931, <4 x bfloat> %687, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %933 = shufflevector <4 x bfloat> %932, <4 x bfloat> %695, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %934 = bitcast <4 x bfloat> %933 to <4 x i16>
- %935 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %934, <4 x float> %903, i32 0, i32 0, i32 0)
- %936 = shufflevector <1 x bfloat> %517, <1 x bfloat> %519, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %937 = shufflevector <4 x bfloat> %936, <4 x bfloat> %523, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %938 = shufflevector <4 x bfloat> %937, <4 x bfloat> %527, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %939 = bitcast <4 x bfloat> %938 to <4 x i16>
- %940 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %919, <4 x float> %908, i32 0, i32 0, i32 0)
- %941 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %924, <4 x float> %909, i32 0, i32 0, i32 0)
- %942 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %929, <4 x float> %910, i32 0, i32 0, i32 0)
- %943 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %934, <4 x float> %911, i32 0, i32 0, i32 0)
- %944 = shufflevector <1 x bfloat> %528, <1 x bfloat> %530, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %945 = shufflevector <4 x bfloat> %944, <4 x bfloat> %533, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %946 = shufflevector <4 x bfloat> %945, <4 x bfloat> %537, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %947 = shufflevector <1 x bfloat> %696, <1 x bfloat> %700, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %948 = shufflevector <4 x bfloat> %947, <4 x bfloat> %705, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %949 = shufflevector <4 x bfloat> %948, <4 x bfloat> %713, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %950 = bitcast <4 x bfloat> %946 to <4 x i16>
- %951 = bitcast <4 x bfloat> %949 to <4 x i16>
- %952 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %951, <4 x float> %920, i32 0, i32 0, i32 0)
- %953 = shufflevector <1 x bfloat> %697, <1 x bfloat> %701, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %954 = shufflevector <4 x bfloat> %953, <4 x bfloat> %707, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %955 = shufflevector <4 x bfloat> %954, <4 x bfloat> %715, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %956 = bitcast <4 x bfloat> %955 to <4 x i16>
- %957 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %956, <4 x float> %925, i32 0, i32 0, i32 0)
- %958 = shufflevector <1 x bfloat> %698, <1 x bfloat> %702, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %959 = shufflevector <4 x bfloat> %958, <4 x bfloat> %709, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %960 = shufflevector <4 x bfloat> %959, <4 x bfloat> %717, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %961 = bitcast <4 x bfloat> %960 to <4 x i16>
- %962 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %961, <4 x float> %930, i32 0, i32 0, i32 0)
- %963 = shufflevector <1 x bfloat> %699, <1 x bfloat> %703, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %964 = shufflevector <4 x bfloat> %963, <4 x bfloat> %711, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %965 = shufflevector <4 x bfloat> %964, <4 x bfloat> %719, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %966 = bitcast <4 x bfloat> %965 to <4 x i16>
- %967 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %966, <4 x float> %935, i32 0, i32 0, i32 0)
- %968 = shufflevector <1 x bfloat> %529, <1 x bfloat> %531, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %969 = shufflevector <4 x bfloat> %968, <4 x bfloat> %535, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %970 = shufflevector <4 x bfloat> %969, <4 x bfloat> %539, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %971 = bitcast <4 x bfloat> %970 to <4 x i16>
- %972 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %951, <4 x float> %940, i32 0, i32 0, i32 0)
- %973 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %956, <4 x float> %941, i32 0, i32 0, i32 0)
- %974 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %961, <4 x float> %942, i32 0, i32 0, i32 0)
- %975 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %966, <4 x float> %943, i32 0, i32 0, i32 0)
- %976 = shufflevector <1 x bfloat> %540, <1 x bfloat> %542, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %977 = shufflevector <4 x bfloat> %976, <4 x bfloat> %545, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %978 = shufflevector <4 x bfloat> %977, <4 x bfloat> %549, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %979 = shufflevector <1 x bfloat> %720, <1 x bfloat> %724, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %980 = shufflevector <4 x bfloat> %979, <4 x bfloat> %729, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %981 = shufflevector <4 x bfloat> %980, <4 x bfloat> %737, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %982 = bitcast <4 x bfloat> %978 to <4 x i16>
- %983 = bitcast <4 x bfloat> %981 to <4 x i16>
- %984 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %983, <4 x float> %952, i32 0, i32 0, i32 0)
- %985 = shufflevector <1 x bfloat> %721, <1 x bfloat> %725, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %986 = shufflevector <4 x bfloat> %985, <4 x bfloat> %731, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %987 = shufflevector <4 x bfloat> %986, <4 x bfloat> %739, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %988 = bitcast <4 x bfloat> %987 to <4 x i16>
- %989 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %988, <4 x float> %957, i32 0, i32 0, i32 0)
- %990 = shufflevector <1 x bfloat> %722, <1 x bfloat> %726, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %991 = shufflevector <4 x bfloat> %990, <4 x bfloat> %733, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %992 = shufflevector <4 x bfloat> %991, <4 x bfloat> %741, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %993 = bitcast <4 x bfloat> %992 to <4 x i16>
- %994 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %993, <4 x float> %962, i32 0, i32 0, i32 0)
- %995 = shufflevector <1 x bfloat> %723, <1 x bfloat> %727, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %996 = shufflevector <4 x bfloat> %995, <4 x bfloat> %735, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %997 = shufflevector <4 x bfloat> %996, <4 x bfloat> %743, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %998 = bitcast <4 x bfloat> %997 to <4 x i16>
- %999 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %998, <4 x float> %967, i32 0, i32 0, i32 0)
- %1000 = shufflevector <1 x bfloat> %541, <1 x bfloat> %543, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1001 = shufflevector <4 x bfloat> %1000, <4 x bfloat> %547, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1002 = shufflevector <4 x bfloat> %1001, <4 x bfloat> %551, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1003 = bitcast <4 x bfloat> %1002 to <4 x i16>
- %1004 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %983, <4 x float> %972, i32 0, i32 0, i32 0)
- %1005 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %988, <4 x float> %973, i32 0, i32 0, i32 0)
- %1006 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %993, <4 x float> %974, i32 0, i32 0, i32 0)
- %1007 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %998, <4 x float> %975, i32 0, i32 0, i32 0)
- %1008 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %984, 0, 0, 0
- %1009 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1008, <4 x float> %989, 1, 0, 0
- %1010 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1009, <4 x float> %994, 2, 0, 0
- %1011 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1010, <4 x float> %999, 3, 0, 0
- %1012 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1011, <4 x float> %1004, 0, 1, 0
- %1013 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1012, <4 x float> %1005, 1, 1, 0
- %1014 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1013, <4 x float> %1006, 2, 1, 0
- %1015 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1014, <4 x float> %1007, 3, 1, 0
- %1016 = trunc i32 %398 to i1
- %1017 = and i1 %41, %1016
- %1018 = select i1 %1017, <8 x bfloat> %400, <8 x bfloat> zeroinitializer
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
+ br label %73
+73: ; preds = %73, %72
+ %74 = phi <1 x bfloat> [ zeroinitializer, %72 ], [ %339, %73 ]
+ %75 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %72 ], [ %304, %73 ]
+ %76 = phi i32 [ 0, %72 ], [ %77, %73 ]
+ %77 = add i32 %76, 1
+ %78 = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %79 = tail call i32 @llvm.umin.i32(i32 %77, i32 %78)
+ %80 = sub i32 0, %79
+ %81 = tail call i32 @llvm.smin.i32(i32 %80, i32 0)
+ %82 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
+ %83 = sub i32 0, %76
+ %84 = tail call i32 @llvm.umin.i32(i32 %83, i32 1)
+ %85 = tail call i32 @llvm.umin.i32(i32 %84, i32 %0)
+ %86 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
+ %87 = or i32 %78, 512
+ %88 = lshr i32 %87, 1
+ %89 = tail call i32 @llvm.umin.i32(i32 %83, i32 %88)
+ %90 = sub i32 0, %89
+ %91 = tail call i32 @llvm.smin.i32(i32 %90, i32 0)
+ %92 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
+ %93 = or i32 %78, 768
+ %94 = lshr i32 %93, 1
+ %95 = tail call i32 @llvm.umin.i32(i32 %83, i32 %94)
+ %96 = sub i32 0, %95
+ %97 = tail call i32 @llvm.smin.i32(i32 %96, i32 0)
+ %98 = load <8 x bfloat>, ptr addrspace(7) %invariant.gep, align 2
+ %99 = lshr i32 %78, 1
+ %100 = tail call i32 @llvm.umin.i32(i32 %77, i32 %99)
+ %101 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
+ %102 = or i32 %78, 256
+ %103 = tail call i32 @llvm.umin.i32(i32 %83, i32 %102)
+ %104 = sub i32 0, %103
+ %105 = tail call i32 @llvm.smin.i32(i32 %104, i32 0)
+ %106 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
+ %107 = tail call i32 @llvm.umin.i32(i32 %83, i32 %87)
+ %108 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
+ %109 = tail call i32 @llvm.umin.i32(i32 %83, i32 %93)
+ %110 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
+ %111 = or i32 %78, 64
+ %112 = tail call i32 @llvm.umin.i32(i32 %83, i32 %111)
+ %113 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
+ %114 = or i32 %78, 80
+ %115 = tail call i32 @llvm.umin.i32(i32 %83, i32 %114)
+ %116 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
+ %117 = or i32 %78, 96
+ %118 = tail call i32 @llvm.umin.i32(i32 %83, i32 %117)
+ %119 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
+ %120 = or i32 %78, 1
+ %121 = tail call i32 @llvm.umin.i32(i32 %77, i32 %120)
+ %122 = load <8 x bfloat>, ptr addrspace(7) null, align 2
+ fence syncscope("workgroup") release
+ %123 = load <1 x bfloat>, ptr addrspace(3) %13, align 2
+ %124 = load <1 x bfloat>, ptr addrspace(3) %14, align 2
+ %125 = load <1 x bfloat>, ptr addrspace(3) %15, align 2
+ %126 = load <1 x bfloat>, ptr addrspace(3) %16, align 2
+ %127 = load <1 x bfloat>, ptr addrspace(3) %17, align 2
+ %128 = shufflevector <1 x bfloat> %127, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %129 = load <1 x bfloat>, ptr addrspace(3) %18, align 2
+ %130 = load <1 x bfloat>, ptr addrspace(3) %19, align 2
+ %131 = shufflevector <1 x bfloat> %130, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %132 = load <1 x bfloat>, ptr addrspace(3) %20, align 2
+ %133 = load <1 x bfloat>, ptr addrspace(3) %21, align 2
+ %134 = load <1 x bfloat>, ptr addrspace(3) %22, align 2
+ %135 = load <1 x bfloat>, ptr addrspace(3) %23, align 2
+ %136 = load <1 x bfloat>, ptr addrspace(3) %24, align 2
+ %137 = load <1 x bfloat>, ptr addrspace(3) %40, align 2
+ %138 = load <1 x bfloat>, ptr addrspace(3) %25, align 2
+ %139 = shufflevector <1 x bfloat> %138, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %140 = load <1 x bfloat>, ptr addrspace(3) %26, align 2
+ %141 = load <1 x bfloat>, ptr addrspace(3) %27, align 2
+ %142 = load <1 x bfloat>, ptr addrspace(3) %28, align 2
+ %143 = load <1 x bfloat>, ptr addrspace(3) %29, align 2
+ %144 = load <1 x bfloat>, ptr addrspace(3) %31, align 2
+ %145 = load <1 x bfloat>, ptr addrspace(3) %32, align 2
+ %146 = load <1 x bfloat>, ptr addrspace(3) %33, align 2
+ %147 = load <1 x bfloat>, ptr addrspace(3) %34, align 2
+ %148 = load <1 x bfloat>, ptr addrspace(3) %35, align 2
+ %149 = load <1 x bfloat>, ptr addrspace(3) %36, align 2
+ %150 = load <1 x bfloat>, ptr addrspace(3) %37, align 2
+ %151 = shufflevector <1 x bfloat> %150, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %152 = load <1 x bfloat>, ptr addrspace(3) %38, align 2
+ %153 = shufflevector <1 x bfloat> %152, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %154 = load <1 x bfloat>, ptr addrspace(3) %39, align 2
+ %155 = shufflevector <1 x bfloat> %154, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %156 = load <1 x bfloat>, ptr addrspace(3) %9, align 2
+ %157 = load <1 x bfloat>, ptr addrspace(3) %43, align 2
+ %158 = load <1 x bfloat>, ptr addrspace(3) %44, align 2
+ %159 = load <1 x bfloat>, ptr addrspace(3) %45, align 2
+ %160 = load <1 x bfloat>, ptr addrspace(3) %46, align 2
+ %161 = load <1 x bfloat>, ptr addrspace(3) %47, align 2
+ %162 = load <1 x bfloat>, ptr addrspace(3) %48, align 2
+ %163 = load <1 x bfloat>, ptr addrspace(3) %49, align 2
+ %164 = load <1 x bfloat>, ptr addrspace(3) %50, align 2
+ %165 = shufflevector <1 x bfloat> %164, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %166 = load <1 x bfloat>, ptr addrspace(3) %51, align 2
+ %167 = load <1 x bfloat>, ptr addrspace(3) %7, align 2
+ %168 = shufflevector <1 x bfloat> %167, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %169 = load <1 x bfloat>, ptr addrspace(3) %53, align 2
+ %170 = load <1 x bfloat>, ptr addrspace(3) %54, align 2
+ %171 = load <1 x bfloat>, ptr addrspace(3) %55, align 2
+ %172 = load <1 x bfloat>, ptr addrspace(3) %56, align 2
+ %173 = load <1 x bfloat>, ptr addrspace(3) %3, align 2
+ %174 = load <1 x bfloat>, ptr addrspace(3) %6, align 2
+ %175 = load <1 x bfloat>, ptr addrspace(3) %5, align 2
+ %176 = load <1 x bfloat>, ptr addrspace(3) %57, align 2
+ %177 = load <1 x bfloat>, ptr addrspace(3) %8, align 2
+ %178 = load <1 x bfloat>, ptr addrspace(3) %11, align 2
+ %179 = load <1 x bfloat>, ptr addrspace(3) %30, align 2
+ %180 = load <1 x bfloat>, ptr addrspace(3) %59, align 2
+ %181 = load <1 x bfloat>, ptr addrspace(3) %58, align 2
+ %182 = load <1 x bfloat>, ptr addrspace(3) %52, align 2
+ %183 = load <1 x bfloat>, ptr addrspace(3) %42, align 2
+ %184 = load <1 x bfloat>, ptr addrspace(3) %1, align 2
+ %185 = load <1 x bfloat>, ptr addrspace(3) %4, align 2
+ %186 = load <1 x bfloat>, ptr addrspace(3) %10, align 2
+ %187 = load <1 x bfloat>, ptr addrspace(3) %41, align 2
+ %188 = load <1 x bfloat>, ptr addrspace(3) %2, align 2
+ %189 = load <1 x bfloat>, ptr addrspace(3) %12, align 2
+ %190 = shufflevector <1 x bfloat> %74, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %191 = shufflevector <4 x bfloat> zeroinitializer, <4 x bfloat> %190, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %192 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 3, 1, 0
+ %193 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %67, <4 x float> %192, i32 0, i32 0, i32 0)
+ %194 = shufflevector <1 x bfloat> %123, <1 x bfloat> %125, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %195 = shufflevector <4 x bfloat> %194, <4 x bfloat> %128, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %196 = shufflevector <1 x bfloat> %156, <1 x bfloat> %160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %197 = shufflevector <4 x bfloat> %196, <4 x bfloat> %165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %198 = shufflevector <1 x bfloat> %170, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %199 = shufflevector <4 x bfloat> %195, <4 x bfloat> %131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %200 = bitcast <4 x bfloat> %199 to <4 x i16>
+ %201 = shufflevector <4 x bfloat> %197, <4 x bfloat> %198, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %202 = bitcast <4 x bfloat> %201 to <4 x i16>
+ %203 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 0, 0, 0
+ %204 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %61, <4 x float> %203, i32 0, i32 0, i32 0)
+ %205 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %200, <4 x i16> %202, <4 x float> %204, i32 0, i32 0, i32 0)
+ %206 = shufflevector <1 x bfloat> %157, <1 x bfloat> %161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %207 = shufflevector <1 x bfloat> %166, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %208 = shufflevector <4 x bfloat> %206, <4 x bfloat> %207, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %209 = shufflevector <1 x bfloat> %171, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %210 = shufflevector <4 x bfloat> %208, <4 x bfloat> %209, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %211 = shufflevector <1 x bfloat> %158, <1 x bfloat> %162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %212 = shufflevector <4 x bfloat> %211, <4 x bfloat> %168, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %213 = shufflevector <1 x bfloat> %172, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %214 = shufflevector <1 x bfloat> %159, <1 x bfloat> %163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %215 = shufflevector <1 x bfloat> %169, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %216 = shufflevector <4 x bfloat> %214, <4 x bfloat> %215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %217 = shufflevector <1 x bfloat> %173, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %218 = shufflevector <4 x bfloat> %216, <4 x bfloat> %217, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %219 = bitcast <4 x bfloat> %218 to <4 x i16>
+ %220 = shufflevector <1 x bfloat> %124, <1 x bfloat> %126, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %221 = shufflevector <1 x bfloat> %129, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %222 = bitcast <4 x bfloat> %210 to <4 x i16>
+ %223 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 1, 1, 0
+ %224 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %222, <4 x float> %223, i32 0, i32 0, i32 0)
+ %225 = shufflevector <4 x bfloat> %220, <4 x bfloat> %221, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %226 = shufflevector <1 x bfloat> %132, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %227 = shufflevector <4 x bfloat> %225, <4 x bfloat> %226, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %228 = bitcast <4 x bfloat> %227 to <4 x i16>
+ %229 = shufflevector <4 x bfloat> %212, <4 x bfloat> %213, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %230 = bitcast <4 x bfloat> %229 to <4 x i16>
+ %231 = bitcast <4 x bfloat> %191 to <4 x i16>
+ %232 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 2, 1, 0
+ %233 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %62, <4 x i16> %231, <4 x float> %232, i32 0, i32 0, i32 0)
+ %234 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %228, <4 x i16> %230, <4 x float> %233, i32 0, i32 0, i32 0)
+ %235 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %219, <4 x float> %193, i32 0, i32 0, i32 0)
+ %236 = shufflevector <1 x bfloat> %133, <1 x bfloat> %135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %237 = shufflevector <1 x bfloat> %137, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %238 = shufflevector <4 x bfloat> %236, <4 x bfloat> %237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %239 = shufflevector <4 x bfloat> %238, <4 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %240 = shufflevector <1 x bfloat> %174, <1 x bfloat> %178, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %241 = shufflevector <1 x bfloat> %182, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %242 = shufflevector <4 x bfloat> %240, <4 x bfloat> %241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %243 = shufflevector <1 x bfloat> %185, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %244 = shufflevector <4 x bfloat> %242, <4 x bfloat> %243, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %245 = bitcast <4 x bfloat> %239 to <4 x i16>
+ %246 = bitcast <4 x bfloat> %244 to <4 x i16>
+ %247 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %245, <4 x i16> %246, <4 x float> %205, i32 0, i32 0, i32 0)
+ %248 = shufflevector <1 x bfloat> %175, <1 x bfloat> %179, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %249 = shufflevector <1 x bfloat> %183, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %250 = shufflevector <4 x bfloat> %248, <4 x bfloat> %249, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %251 = shufflevector <1 x bfloat> %186, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %252 = shufflevector <4 x bfloat> %250, <4 x bfloat> %251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %253 = bitcast <4 x bfloat> %252 to <4 x i16>
+ %254 = shufflevector <1 x bfloat> %176, <1 x bfloat> %180, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %255 = shufflevector <4 x bfloat> %254, <4 x bfloat> %60, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %256 = shufflevector <1 x bfloat> %187, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %257 = shufflevector <4 x bfloat> %255, <4 x bfloat> %256, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %258 = bitcast <4 x bfloat> %257 to <4 x i16>
+ %259 = shufflevector <1 x bfloat> %177, <1 x bfloat> %181, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %260 = shufflevector <1 x bfloat> %184, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %261 = shufflevector <4 x bfloat> %259, <4 x bfloat> %260, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %262 = shufflevector <1 x bfloat> %188, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %263 = shufflevector <4 x bfloat> %261, <4 x bfloat> %262, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %264 = bitcast <4 x bfloat> %263 to <4 x i16>
+ %265 = shufflevector <1 x bfloat> %134, <1 x bfloat> %136, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %266 = shufflevector <4 x bfloat> %265, <4 x bfloat> %139, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %267 = bitcast <4 x bfloat> %266 to <4 x i16>
+ %268 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %267, <4 x i16> %253, <4 x float> %224, i32 0, i32 0, i32 0)
+ %269 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %258, <4 x float> %234, i32 0, i32 0, i32 0)
+ %270 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %264, <4 x float> %235, i32 0, i32 0, i32 0)
+ %271 = shufflevector <1 x bfloat> %140, <1 x bfloat> %141, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %272 = shufflevector <1 x bfloat> %142, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %273 = shufflevector <4 x bfloat> %271, <4 x bfloat> %272, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %274 = shufflevector <1 x bfloat> %143, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %275 = shufflevector <4 x bfloat> %273, <4 x bfloat> %274, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %276 = shufflevector <1 x bfloat> splat (bfloat 0xR3F80), <1 x bfloat> %189, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %277 = bitcast <4 x bfloat> %275 to <4 x i16>
+ %278 = bitcast <4 x bfloat> %276 to <4 x i16>
+ %279 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %277, <4 x i16> %278, <4 x float> %247, i32 0, i32 0, i32 0)
+ %280 = shufflevector <1 x bfloat> %144, <1 x bfloat> %145, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %281 = shufflevector <1 x bfloat> %146, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %282 = shufflevector <4 x bfloat> %280, <4 x bfloat> %281, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %283 = shufflevector <1 x bfloat> %147, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %284 = shufflevector <4 x bfloat> %282, <4 x bfloat> %283, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %285 = bitcast <4 x bfloat> %284 to <4 x i16>
+ %286 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %285, <4 x i16> %63, <4 x float> %279, i32 0, i32 0, i32 0)
+ %287 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %69, <4 x float> %268, i32 0, i32 0, i32 0)
+ %288 = shufflevector <1 x bfloat> %148, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %289 = shufflevector <1 x bfloat> %149, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %290 = shufflevector <4 x bfloat> %288, <4 x bfloat> %289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %291 = shufflevector <4 x bfloat> %290, <4 x bfloat> %153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %292 = bitcast <4 x bfloat> %291 to <4 x i16>
+ %293 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %292, <4 x i16> splat (i16 1), <4 x float> %286, i32 0, i32 0, i32 0)
+ %294 = shufflevector <4 x bfloat> %151, <4 x bfloat> %155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %295 = bitcast <4 x bfloat> %294 to <4 x i16>
+ %296 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %70, <4 x i16> zeroinitializer, <4 x float> %293, i32 0, i32 0, i32 0)
+ %297 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %65, <4 x i16> zeroinitializer, <4 x float> %296, i32 0, i32 0, i32 0)
+ %298 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %297, 0, 0, 0
+ %299 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %295, <4 x i16> zeroinitializer, <4 x float> %287, i32 0, i32 0, i32 0)
+ %300 = insertvalue [4 x [2 x [1 x <4 x float>]]] %298, <4 x float> %299, 1, 1, 0
+ %301 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %64, <4 x float> %269, i32 0, i32 0, i32 0)
+ %302 = insertvalue [4 x [2 x [1 x <4 x float>]]] %300, <4 x float> %301, 2, 1, 0
+ %303 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %68, <4 x i16> zeroinitializer, <4 x float> %270, i32 0, i32 0, i32 0)
+ %304 = insertvalue [4 x [2 x [1 x <4 x float>]]] %302, <4 x float> %303, 3, 1, 0
+ %305 = trunc i32 %81 to i1
+ %306 = select i1 %305, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
tail call void @llvm.amdgcn.sched.barrier(i32 0)
- store <8 x bfloat> %1018, ptr addrspace(3) %85, align 8
- %1019 = trunc i32 %403 to i1
- %1020 = and i1 %41, %1019
- %1021 = select i1 %1020, <8 x bfloat> %405, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1021, ptr addrspace(3) %88, align 8
- %1022 = trunc i32 %408 to i1
- %1023 = and i1 %41, %1022
- %1024 = select i1 %1023, <8 x bfloat> %410, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1024, ptr addrspace(3) %91, align 8
- %1025 = trunc i32 %413 to i1
- %1026 = and i1 %41, %1025
- %1027 = select i1 %1026, <8 x bfloat> %415, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1027, ptr addrspace(3) %94, align 8
- %1028 = trunc i32 %418 to i1
- %1029 = and i1 %44, %1028
- %1030 = select i1 %1029, <8 x bfloat> %420, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1030, ptr addrspace(3) %97, align 8
- %1031 = trunc i32 %423 to i1
- %1032 = and i1 %44, %1031
- %1033 = select i1 %1032, <8 x bfloat> %425, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1033, ptr addrspace(3) %100, align 8
- %1034 = trunc i32 %428 to i1
- %1035 = and i1 %44, %1034
- %1036 = select i1 %1035, <8 x bfloat> %430, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1036, ptr addrspace(3) %103, align 8
- %1037 = trunc i32 %433 to i1
- %1038 = and i1 %44, %1037
- %1039 = select i1 %1038, <8 x bfloat> %435, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1039, ptr addrspace(3) %106, align 8
- %1040 = trunc i32 %438 to i1
- %1041 = and i1 %44, %1040
- %1042 = select i1 %1041, <8 x bfloat> %440, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1042, ptr addrspace(3) %109, align 8
- %1043 = trunc i32 %443 to i1
- %1044 = and i1 %44, %1043
- %1045 = select i1 %1044, <8 x bfloat> %445, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1045, ptr addrspace(3) %112, align 8
- %1046 = trunc i32 %448 to i1
- %1047 = and i1 %44, %1046
- %1048 = select i1 %1047, <8 x bfloat> %450, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1048, ptr addrspace(3) %115, align 8
- %1049 = trunc i32 %453 to i1
- %1050 = and i1 %44, %1049
- %1051 = select i1 %1050, <8 x bfloat> %455, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1051, ptr addrspace(3) %118, align 8
- %1052 = icmp samesign ult i32 %391, 72
- br i1 %1052, label %389, label %1053
-
-1053: ; preds = %389
- %1054 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %2, i16 0, i64 8128512, i32 159744)
- %1055 = shl nuw nsw i32 %8, 5
- %.scalar = or disjoint i32 %1055, %11
- %1056 = insertelement <4 x i32> poison, i32 %.scalar, i64 0
- %1057 = shufflevector <4 x i32> %1056, <4 x i32> poison, <4 x i32> zeroinitializer
- %1058 = or disjoint <4 x i32> %1057, <i32 0, i32 1, i32 2, i32 3>
- %1059 = or disjoint <4 x i32> %1057, <i32 16, i32 17, i32 18, i32 19>
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
- %1060 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
- %1061 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
- %1062 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
- %1063 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
- %1064 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
- %1065 = shufflevector <1 x bfloat> %1064, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1066 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
- %1067 = shufflevector <1 x bfloat> %1066, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1068 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
- %1069 = shufflevector <1 x bfloat> %1068, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1070 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
- %1071 = shufflevector <1 x bfloat> %1070, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1072 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
- %1073 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
- %1074 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
- %1075 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
- %1076 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
- %1077 = shufflevector <1 x bfloat> %1076, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1078 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
- %1079 = shufflevector <1 x bfloat> %1078, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1080 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
- %1081 = shufflevector <1 x bfloat> %1080, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1082 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
- %1083 = shufflevector <1 x bfloat> %1082, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1084 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
- %1085 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
- %1086 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
- %1087 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
- %1088 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
- %1089 = shufflevector <1 x bfloat> %1088, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1090 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
- %1091 = shufflevector <1 x bfloat> %1090, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1092 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
- %1093 = shufflevector <1 x bfloat> %1092, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1094 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
- %1095 = shufflevector <1 x bfloat> %1094, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1096 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
- %1097 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
- %1098 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
- %1099 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
- %1100 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
- %1101 = shufflevector <1 x bfloat> %1100, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1102 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
- %1103 = shufflevector <1 x bfloat> %1102, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1104 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
- %1105 = shufflevector <1 x bfloat> %1104, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1106 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
- %1107 = shufflevector <1 x bfloat> %1106, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1108 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
- %1109 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
- %1110 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
- %1111 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
- %1112 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
- %1113 = shufflevector <1 x bfloat> %1112, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1114 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
- %1115 = shufflevector <1 x bfloat> %1114, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1116 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
- %1117 = shufflevector <1 x bfloat> %1116, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1118 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
- %1119 = shufflevector <1 x bfloat> %1118, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1120 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
- %1121 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
- %1122 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
- %1123 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
- %1124 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
- %1125 = shufflevector <1 x bfloat> %1124, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1126 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
- %1127 = shufflevector <1 x bfloat> %1126, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1128 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
- %1129 = shufflevector <1 x bfloat> %1128, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1130 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
- %1131 = shufflevector <1 x bfloat> %1130, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1132 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
- %1133 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
- %1134 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
- %1135 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
- %1136 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
- %1137 = shufflevector <1 x bfloat> %1136, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1138 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
- %1139 = shufflevector <1 x bfloat> %1138, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1140 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
- %1141 = shufflevector <1 x bfloat> %1140, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1142 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
- %1143 = shufflevector <1 x bfloat> %1142, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1144 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
- %1145 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
- %1146 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
- %1147 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
- %1148 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
- %1149 = shufflevector <1 x bfloat> %1148, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1150 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
- %1151 = shufflevector <1 x bfloat> %1150, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1152 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
- %1153 = shufflevector <1 x bfloat> %1152, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1154 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
- %1155 = shufflevector <1 x bfloat> %1154, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1156 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
- %1157 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
- %1158 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
- %1159 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
- %1160 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
- %1161 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
- %1162 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
- %1163 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
- %1164 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
- %1165 = shufflevector <1 x bfloat> %1164, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1166 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
- %1167 = shufflevector <1 x bfloat> %1166, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1168 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
- %1169 = shufflevector <1 x bfloat> %1168, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1170 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
- %1171 = shufflevector <1 x bfloat> %1170, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1172 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
- %1173 = shufflevector <1 x bfloat> %1172, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1174 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
- %1175 = shufflevector <1 x bfloat> %1174, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1176 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
- %1177 = shufflevector <1 x bfloat> %1176, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1178 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
- %1179 = shufflevector <1 x bfloat> %1178, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1180 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
- %1181 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
- %1182 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
- %1183 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
- %1184 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
- %1185 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
- %1186 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
- %1187 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
- %1188 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
- %1189 = shufflevector <1 x bfloat> %1188, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1190 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
- %1191 = shufflevector <1 x bfloat> %1190, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1192 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
- %1193 = shufflevector <1 x bfloat> %1192, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1194 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
- %1195 = shufflevector <1 x bfloat> %1194, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1196 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
- %1197 = shufflevector <1 x bfloat> %1196, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1198 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
- %1199 = shufflevector <1 x bfloat> %1198, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1200 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
- %1201 = shufflevector <1 x bfloat> %1200, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1202 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
- %1203 = shufflevector <1 x bfloat> %1202, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1204 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
- %1205 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
- %1206 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
- %1207 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
- %1208 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
- %1209 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
- %1210 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
- %1211 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
- %1212 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
- %1213 = shufflevector <1 x bfloat> %1212, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1214 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
- %1215 = shufflevector <1 x bfloat> %1214, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1216 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
- %1217 = shufflevector <1 x bfloat> %1216, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1218 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
- %1219 = shufflevector <1 x bfloat> %1218, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1220 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
- %1221 = shufflevector <1 x bfloat> %1220, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1222 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
- %1223 = shufflevector <1 x bfloat> %1222, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1224 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
- %1225 = shufflevector <1 x bfloat> %1224, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1226 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
- %1227 = shufflevector <1 x bfloat> %1226, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1228 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
- %1229 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
- %1230 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
- %1231 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
- %1232 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
- %1233 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
- %1234 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
- %1235 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
- %1236 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
- %1237 = shufflevector <1 x bfloat> %1236, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1238 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
- %1239 = shufflevector <1 x bfloat> %1238, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1240 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
- %1241 = shufflevector <1 x bfloat> %1240, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1242 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
- %1243 = shufflevector <1 x bfloat> %1242, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1244 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
- %1245 = shufflevector <1 x bfloat> %1244, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1246 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
- %1247 = shufflevector <1 x bfloat> %1246, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1248 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
- %1249 = shufflevector <1 x bfloat> %1248, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1250 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
- %1251 = shufflevector <1 x bfloat> %1250, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1252 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
- %1253 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
- %1254 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
- %1255 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
- %1256 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
- %1257 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
- %1258 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
- %1259 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
- %1260 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
- %1261 = shufflevector <1 x bfloat> %1260, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1262 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
- %1263 = shufflevector <1 x bfloat> %1262, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1264 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
- %1265 = shufflevector <1 x bfloat> %1264, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1266 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
- %1267 = shufflevector <1 x bfloat> %1266, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1268 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
- %1269 = shufflevector <1 x bfloat> %1268, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1270 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
- %1271 = shufflevector <1 x bfloat> %1270, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1272 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
- %1273 = shufflevector <1 x bfloat> %1272, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1274 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
- %1275 = shufflevector <1 x bfloat> %1274, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1276 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
- %1277 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
- %1278 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
- %1279 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
- %1280 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
- %1281 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
- %1282 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
- %1283 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
- %1284 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
- %1285 = shufflevector <1 x bfloat> %1284, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1286 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
- %1287 = shufflevector <1 x bfloat> %1286, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1288 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
- %1289 = shufflevector <1 x bfloat> %1288, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1290 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
- %1291 = shufflevector <1 x bfloat> %1290, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1292 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
- %1293 = shufflevector <1 x bfloat> %1292, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1294 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
- %1295 = shufflevector <1 x bfloat> %1294, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1296 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
- %1297 = shufflevector <1 x bfloat> %1296, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1298 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
- %1299 = shufflevector <1 x bfloat> %1298, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1300 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
- %1301 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
- %1302 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
- %1303 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
- %1304 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
- %1305 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
- %1306 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
- %1307 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
- %1308 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
- %1309 = shufflevector <1 x bfloat> %1308, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1310 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
- %1311 = shufflevector <1 x bfloat> %1310, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1312 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
- %1313 = shufflevector <1 x bfloat> %1312, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1314 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
- %1315 = shufflevector <1 x bfloat> %1314, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1316 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
- %1317 = shufflevector <1 x bfloat> %1316, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1318 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
- %1319 = shufflevector <1 x bfloat> %1318, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1320 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
- %1321 = shufflevector <1 x bfloat> %1320, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1322 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
- %1323 = shufflevector <1 x bfloat> %1322, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1324 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
- %1325 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
- %1326 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
- %1327 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
- %1328 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
- %1329 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
- %1330 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
- %1331 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
- %1332 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
- %1333 = shufflevector <1 x bfloat> %1332, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1334 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
- %1335 = shufflevector <1 x bfloat> %1334, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1336 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
- %1337 = shufflevector <1 x bfloat> %1336, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1338 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
- %1339 = shufflevector <1 x bfloat> %1338, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1340 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
- %1341 = shufflevector <1 x bfloat> %1340, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1342 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
- %1343 = shufflevector <1 x bfloat> %1342, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1344 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
- %1345 = shufflevector <1 x bfloat> %1344, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1346 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
- %1347 = shufflevector <1 x bfloat> %1346, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1348 = shufflevector <1 x bfloat> %1060, <1 x bfloat> %1062, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1349 = shufflevector <4 x bfloat> %1348, <4 x bfloat> %1065, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1350 = shufflevector <4 x bfloat> %1349, <4 x bfloat> %1069, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1351 = shufflevector <1 x bfloat> %1156, <1 x bfloat> %1160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1352 = shufflevector <4 x bfloat> %1351, <4 x bfloat> %1165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1353 = shufflevector <4 x bfloat> %1352, <4 x bfloat> %1173, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1354 = bitcast <4 x bfloat> %1350 to <4 x i16>
- %1355 = bitcast <4 x bfloat> %1353 to <4 x i16>
- %1356 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1355, <4 x float> %984, i32 0, i32 0, i32 0)
- %1357 = shufflevector <1 x bfloat> %1157, <1 x bfloat> %1161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1358 = shufflevector <4 x bfloat> %1357, <4 x bfloat> %1167, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1359 = shufflevector <4 x bfloat> %1358, <4 x bfloat> %1175, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1360 = bitcast <4 x bfloat> %1359 to <4 x i16>
- %1361 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1360, <4 x float> %989, i32 0, i32 0, i32 0)
- %1362 = shufflevector <1 x bfloat> %1158, <1 x bfloat> %1162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1363 = shufflevector <4 x bfloat> %1362, <4 x bfloat> %1169, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1364 = shufflevector <4 x bfloat> %1363, <4 x bfloat> %1177, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1365 = bitcast <4 x bfloat> %1364 to <4 x i16>
- %1366 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1365, <4 x float> %994, i32 0, i32 0, i32 0)
- %1367 = shufflevector <1 x bfloat> %1159, <1 x bfloat> %1163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1368 = shufflevector <4 x bfloat> %1367, <4 x bfloat> %1171, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1369 = shufflevector <4 x bfloat> %1368, <4 x bfloat> %1179, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1370 = bitcast <4 x bfloat> %1369 to <4 x i16>
- %1371 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1370, <4 x float> %999, i32 0, i32 0, i32 0)
- %1372 = shufflevector <1 x bfloat> %1061, <1 x bfloat> %1063, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1373 = shufflevector <4 x bfloat> %1372, <4 x bfloat> %1067, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1374 = shufflevector <4 x bfloat> %1373, <4 x bfloat> %1071, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1375 = bitcast <4 x bfloat> %1374 to <4 x i16>
- %1376 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1355, <4 x float> %1004, i32 0, i32 0, i32 0)
- %1377 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1360, <4 x float> %1005, i32 0, i32 0, i32 0)
- %1378 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1365, <4 x float> %1006, i32 0, i32 0, i32 0)
- %1379 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1370, <4 x float> %1007, i32 0, i32 0, i32 0)
- %1380 = shufflevector <1 x bfloat> %1072, <1 x bfloat> %1074, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1381 = shufflevector <4 x bfloat> %1380, <4 x bfloat> %1077, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1382 = shufflevector <4 x bfloat> %1381, <4 x bfloat> %1081, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1383 = shufflevector <1 x bfloat> %1180, <1 x bfloat> %1184, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1384 = shufflevector <4 x bfloat> %1383, <4 x bfloat> %1189, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1385 = shufflevector <4 x bfloat> %1384, <4 x bfloat> %1197, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1386 = bitcast <4 x bfloat> %1382 to <4 x i16>
- %1387 = bitcast <4 x bfloat> %1385 to <4 x i16>
- %1388 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1387, <4 x float> %1356, i32 0, i32 0, i32 0)
- %1389 = shufflevector <1 x bfloat> %1181, <1 x bfloat> %1185, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1390 = shufflevector <4 x bfloat> %1389, <4 x bfloat> %1191, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1391 = shufflevector <4 x bfloat> %1390, <4 x bfloat> %1199, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1392 = bitcast <4 x bfloat> %1391 to <4 x i16>
- %1393 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1392, <4 x float> %1361, i32 0, i32 0, i32 0)
- %1394 = shufflevector <1 x bfloat> %1182, <1 x bfloat> %1186, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1395 = shufflevector <4 x bfloat> %1394, <4 x bfloat> %1193, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1396 = shufflevector <4 x bfloat> %1395, <4 x bfloat> %1201, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1397 = bitcast <4 x bfloat> %1396 to <4 x i16>
- %1398 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1397, <4 x float> %1366, i32 0, i32 0, i32 0)
- %1399 = shufflevector <1 x bfloat> %1183, <1 x bfloat> %1187, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1400 = shufflevector <4 x bfloat> %1399, <4 x bfloat> %1195, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1401 = shufflevector <4 x bfloat> %1400, <4 x bfloat> %1203, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1402 = bitcast <4 x bfloat> %1401 to <4 x i16>
- %1403 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1402, <4 x float> %1371, i32 0, i32 0, i32 0)
- %1404 = shufflevector <1 x bfloat> %1073, <1 x bfloat> %1075, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1405 = shufflevector <4 x bfloat> %1404, <4 x bfloat> %1079, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1406 = shufflevector <4 x bfloat> %1405, <4 x bfloat> %1083, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1407 = bitcast <4 x bfloat> %1406 to <4 x i16>
- %1408 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1387, <4 x float> %1376, i32 0, i32 0, i32 0)
- %1409 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1392, <4 x float> %1377, i32 0, i32 0, i32 0)
- %1410 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1397, <4 x float> %1378, i32 0, i32 0, i32 0)
- %1411 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1402, <4 x float> %1379, i32 0, i32 0, i32 0)
- %1412 = shufflevector <1 x bfloat> %1084, <1 x bfloat> %1086, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1413 = shufflevector <4 x bfloat> %1412, <4 x bfloat> %1089, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1414 = shufflevector <4 x bfloat> %1413, <4 x bfloat> %1093, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1415 = shufflevector <1 x bfloat> %1204, <1 x bfloat> %1208, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1416 = shufflevector <4 x bfloat> %1415, <4 x bfloat> %1213, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1417 = shufflevector <4 x bfloat> %1416, <4 x bfloat> %1221, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1418 = bitcast <4 x bfloat> %1414 to <4 x i16>
- %1419 = bitcast <4 x bfloat> %1417 to <4 x i16>
- %1420 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1419, <4 x float> %1388, i32 0, i32 0, i32 0)
- %1421 = shufflevector <1 x bfloat> %1205, <1 x bfloat> %1209, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1422 = shufflevector <4 x bfloat> %1421, <4 x bfloat> %1215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1423 = shufflevector <4 x bfloat> %1422, <4 x bfloat> %1223, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1424 = bitcast <4 x bfloat> %1423 to <4 x i16>
- %1425 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1424, <4 x float> %1393, i32 0, i32 0, i32 0)
- %1426 = shufflevector <1 x bfloat> %1206, <1 x bfloat> %1210, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1427 = shufflevector <4 x bfloat> %1426, <4 x bfloat> %1217, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1428 = shufflevector <4 x bfloat> %1427, <4 x bfloat> %1225, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1429 = bitcast <4 x bfloat> %1428 to <4 x i16>
- %1430 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1429, <4 x float> %1398, i32 0, i32 0, i32 0)
- %1431 = shufflevector <1 x bfloat> %1207, <1 x bfloat> %1211, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1432 = shufflevector <4 x bfloat> %1431, <4 x bfloat> %1219, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1433 = shufflevector <4 x bfloat> %1432, <4 x bfloat> %1227, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1434 = bitcast <4 x bfloat> %1433 to <4 x i16>
- %1435 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1434, <4 x float> %1403, i32 0, i32 0, i32 0)
- %1436 = shufflevector <1 x bfloat> %1085, <1 x bfloat> %1087, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1437 = shufflevector <4 x bfloat> %1436, <4 x bfloat> %1091, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1438 = shufflevector <4 x bfloat> %1437, <4 x bfloat> %1095, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1439 = bitcast <4 x bfloat> %1438 to <4 x i16>
- %1440 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1419, <4 x float> %1408, i32 0, i32 0, i32 0)
- %1441 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1424, <4 x float> %1409, i32 0, i32 0, i32 0)
- %1442 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1429, <4 x float> %1410, i32 0, i32 0, i32 0)
- %1443 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1434, <4 x float> %1411, i32 0, i32 0, i32 0)
- %1444 = shufflevector <1 x bfloat> %1096, <1 x bfloat> %1098, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1445 = shufflevector <4 x bfloat> %1444, <4 x bfloat> %1101, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1446 = shufflevector <4 x bfloat> %1445, <4 x bfloat> %1105, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1447 = shufflevector <1 x bfloat> %1228, <1 x bfloat> %1232, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1448 = shufflevector <4 x bfloat> %1447, <4 x bfloat> %1237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1449 = shufflevector <4 x bfloat> %1448, <4 x bfloat> %1245, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1450 = bitcast <4 x bfloat> %1446 to <4 x i16>
- %1451 = bitcast <4 x bfloat> %1449 to <4 x i16>
- %1452 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1451, <4 x float> %1420, i32 0, i32 0, i32 0)
- %1453 = shufflevector <1 x bfloat> %1229, <1 x bfloat> %1233, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1454 = shufflevector <4 x bfloat> %1453, <4 x bfloat> %1239, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1455 = shufflevector <4 x bfloat> %1454, <4 x bfloat> %1247, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1456 = bitcast <4 x bfloat> %1455 to <4 x i16>
- %1457 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1456, <4 x float> %1425, i32 0, i32 0, i32 0)
- %1458 = shufflevector <1 x bfloat> %1230, <1 x bfloat> %1234, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1459 = shufflevector <4 x bfloat> %1458, <4 x bfloat> %1241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1460 = shufflevector <4 x bfloat> %1459, <4 x bfloat> %1249, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1461 = bitcast <4 x bfloat> %1460 to <4 x i16>
- %1462 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1461, <4 x float> %1430, i32 0, i32 0, i32 0)
- %1463 = shufflevector <1 x bfloat> %1231, <1 x bfloat> %1235, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1464 = shufflevector <4 x bfloat> %1463, <4 x bfloat> %1243, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1465 = shufflevector <4 x bfloat> %1464, <4 x bfloat> %1251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1466 = bitcast <4 x bfloat> %1465 to <4 x i16>
- %1467 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1466, <4 x float> %1435, i32 0, i32 0, i32 0)
- %1468 = shufflevector <1 x bfloat> %1097, <1 x bfloat> %1099, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1469 = shufflevector <4 x bfloat> %1468, <4 x bfloat> %1103, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1470 = shufflevector <4 x bfloat> %1469, <4 x bfloat> %1107, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1471 = bitcast <4 x bfloat> %1470 to <4 x i16>
- %1472 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1451, <4 x float> %1440, i32 0, i32 0, i32 0)
- %1473 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1456, <4 x float> %1441, i32 0, i32 0, i32 0)
- %1474 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1461, <4 x float> %1442, i32 0, i32 0, i32 0)
- %1475 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1466, <4 x float> %1443, i32 0, i32 0, i32 0)
- %1476 = shufflevector <1 x bfloat> %1108, <1 x bfloat> %1110, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1477 = shufflevector <4 x bfloat> %1476, <4 x bfloat> %1113, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1478 = shufflevector <4 x bfloat> %1477, <4 x bfloat> %1117, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1479 = shufflevector <1 x bfloat> %1252, <1 x bfloat> %1256, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1480 = shufflevector <4 x bfloat> %1479, <4 x bfloat> %1261, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1481 = shufflevector <4 x bfloat> %1480, <4 x bfloat> %1269, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1482 = bitcast <4 x bfloat> %1478 to <4 x i16>
- %1483 = bitcast <4 x bfloat> %1481 to <4 x i16>
- %1484 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1483, <4 x float> %1452, i32 0, i32 0, i32 0)
- %1485 = shufflevector <1 x bfloat> %1253, <1 x bfloat> %1257, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1486 = shufflevector <4 x bfloat> %1485, <4 x bfloat> %1263, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1487 = shufflevector <4 x bfloat> %1486, <4 x bfloat> %1271, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1488 = bitcast <4 x bfloat> %1487 to <4 x i16>
- %1489 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1488, <4 x float> %1457, i32 0, i32 0, i32 0)
- %1490 = shufflevector <1 x bfloat> %1254, <1 x bfloat> %1258, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1491 = shufflevector <4 x bfloat> %1490, <4 x bfloat> %1265, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1492 = shufflevector <4 x bfloat> %1491, <4 x bfloat> %1273, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1493 = bitcast <4 x bfloat> %1492 to <4 x i16>
- %1494 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1493, <4 x float> %1462, i32 0, i32 0, i32 0)
- %1495 = shufflevector <1 x bfloat> %1255, <1 x bfloat> %1259, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1496 = shufflevector <4 x bfloat> %1495, <4 x bfloat> %1267, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1497 = shufflevector <4 x bfloat> %1496, <4 x bfloat> %1275, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1498 = bitcast <4 x bfloat> %1497 to <4 x i16>
- %1499 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1498, <4 x float> %1467, i32 0, i32 0, i32 0)
- %1500 = shufflevector <1 x bfloat> %1109, <1 x bfloat> %1111, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1501 = shufflevector <4 x bfloat> %1500, <4 x bfloat> %1115, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1502 = shufflevector <4 x bfloat> %1501, <4 x bfloat> %1119, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1503 = bitcast <4 x bfloat> %1502 to <4 x i16>
- %1504 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1483, <4 x float> %1472, i32 0, i32 0, i32 0)
- %1505 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1488, <4 x float> %1473, i32 0, i32 0, i32 0)
- %1506 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1493, <4 x float> %1474, i32 0, i32 0, i32 0)
- %1507 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1498, <4 x float> %1475, i32 0, i32 0, i32 0)
- %1508 = shufflevector <1 x bfloat> %1120, <1 x bfloat> %1122, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1509 = shufflevector <4 x bfloat> %1508, <4 x bfloat> %1125, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1510 = shufflevector <4 x bfloat> %1509, <4 x bfloat> %1129, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1511 = shufflevector <1 x bfloat> %1276, <1 x bfloat> %1280, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1512 = shufflevector <4 x bfloat> %1511, <4 x bfloat> %1285, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1513 = shufflevector <4 x bfloat> %1512, <4 x bfloat> %1293, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1514 = bitcast <4 x bfloat> %1510 to <4 x i16>
- %1515 = bitcast <4 x bfloat> %1513 to <4 x i16>
- %1516 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1515, <4 x float> %1484, i32 0, i32 0, i32 0)
- %1517 = shufflevector <1 x bfloat> %1277, <1 x bfloat> %1281, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1518 = shufflevector <4 x bfloat> %1517, <4 x bfloat> %1287, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1519 = shufflevector <4 x bfloat> %1518, <4 x bfloat> %1295, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1520 = bitcast <4 x bfloat> %1519 to <4 x i16>
- %1521 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1520, <4 x float> %1489, i32 0, i32 0, i32 0)
- %1522 = shufflevector <1 x bfloat> %1278, <1 x bfloat> %1282, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1523 = shufflevector <4 x bfloat> %1522, <4 x bfloat> %1289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1524 = shufflevector <4 x bfloat> %1523, <4 x bfloat> %1297, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1525 = bitcast <4 x bfloat> %1524 to <4 x i16>
- %1526 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1525, <4 x float> %1494, i32 0, i32 0, i32 0)
- %1527 = shufflevector <1 x bfloat> %1279, <1 x bfloat> %1283, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1528 = shufflevector <4 x bfloat> %1527, <4 x bfloat> %1291, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1529 = shufflevector <4 x bfloat> %1528, <4 x bfloat> %1299, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1530 = bitcast <4 x bfloat> %1529 to <4 x i16>
- %1531 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1530, <4 x float> %1499, i32 0, i32 0, i32 0)
- %1532 = shufflevector <1 x bfloat> %1121, <1 x bfloat> %1123, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1533 = shufflevector <4 x bfloat> %1532, <4 x bfloat> %1127, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1534 = shufflevector <4 x bfloat> %1533, <4 x bfloat> %1131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1535 = bitcast <4 x bfloat> %1534 to <4 x i16>
- %1536 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1515, <4 x float> %1504, i32 0, i32 0, i32 0)
- %1537 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1520, <4 x float> %1505, i32 0, i32 0, i32 0)
- %1538 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1525, <4 x float> %1506, i32 0, i32 0, i32 0)
- %1539 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1530, <4 x float> %1507, i32 0, i32 0, i32 0)
- %1540 = shufflevector <1 x bfloat> %1132, <1 x bfloat> %1134, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1541 = shufflevector <4 x bfloat> %1540, <4 x bfloat> %1137, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1542 = shufflevector <4 x bfloat> %1541, <4 x bfloat> %1141, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1543 = shufflevector <1 x bfloat> %1300, <1 x bfloat> %1304, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1544 = shufflevector <4 x bfloat> %1543, <4 x bfloat> %1309, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1545 = shufflevector <4 x bfloat> %1544, <4 x bfloat> %1317, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1546 = bitcast <4 x bfloat> %1542 to <4 x i16>
- %1547 = bitcast <4 x bfloat> %1545 to <4 x i16>
- %1548 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1547, <4 x float> %1516, i32 0, i32 0, i32 0)
- %1549 = shufflevector <1 x bfloat> %1301, <1 x bfloat> %1305, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1550 = shufflevector <4 x bfloat> %1549, <4 x bfloat> %1311, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1551 = shufflevector <4 x bfloat> %1550, <4 x bfloat> %1319, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1552 = bitcast <4 x bfloat> %1551 to <4 x i16>
- %1553 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1552, <4 x float> %1521, i32 0, i32 0, i32 0)
- %1554 = shufflevector <1 x bfloat> %1302, <1 x bfloat> %1306, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1555 = shufflevector <4 x bfloat> %1554, <4 x bfloat> %1313, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1556 = shufflevector <4 x bfloat> %1555, <4 x bfloat> %1321, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1557 = bitcast <4 x bfloat> %1556 to <4 x i16>
- %1558 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1557, <4 x float> %1526, i32 0, i32 0, i32 0)
- %1559 = shufflevector <1 x bfloat> %1303, <1 x bfloat> %1307, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1560 = shufflevector <4 x bfloat> %1559, <4 x bfloat> %1315, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1561 = shufflevector <4 x bfloat> %1560, <4 x bfloat> %1323, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1562 = bitcast <4 x bfloat> %1561 to <4 x i16>
- %1563 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1562, <4 x float> %1531, i32 0, i32 0, i32 0)
- %1564 = shufflevector <1 x bfloat> %1133, <1 x bfloat> %1135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1565 = shufflevector <4 x bfloat> %1564, <4 x bfloat> %1139, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1566 = shufflevector <4 x bfloat> %1565, <4 x bfloat> %1143, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1567 = bitcast <4 x bfloat> %1566 to <4 x i16>
- %1568 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1547, <4 x float> %1536, i32 0, i32 0, i32 0)
- %1569 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1552, <4 x float> %1537, i32 0, i32 0, i32 0)
- %1570 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1557, <4 x float> %1538, i32 0, i32 0, i32 0)
- %1571 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1562, <4 x float> %1539, i32 0, i32 0, i32 0)
- %1572 = shufflevector <1 x bfloat> %1144, <1 x bfloat> %1146, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1573 = shufflevector <4 x bfloat> %1572, <4 x bfloat> %1149, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1574 = shufflevector <4 x bfloat> %1573, <4 x bfloat> %1153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1575 = shufflevector <1 x bfloat> %1324, <1 x bfloat> %1328, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1576 = shufflevector <4 x bfloat> %1575, <4 x bfloat> %1333, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1577 = shufflevector <4 x bfloat> %1576, <4 x bfloat> %1341, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1578 = bitcast <4 x bfloat> %1574 to <4 x i16>
- %1579 = bitcast <4 x bfloat> %1577 to <4 x i16>
- %1580 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1579, <4 x float> %1548, i32 0, i32 0, i32 0)
- %1581 = shufflevector <1 x bfloat> %1325, <1 x bfloat> %1329, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1582 = shufflevector <4 x bfloat> %1581, <4 x bfloat> %1335, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1583 = shufflevector <4 x bfloat> %1582, <4 x bfloat> %1343, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1584 = bitcast <4 x bfloat> %1583 to <4 x i16>
- %1585 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1584, <4 x float> %1553, i32 0, i32 0, i32 0)
- %1586 = shufflevector <4 x float> %1585, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1587 = shufflevector <1 x bfloat> %1326, <1 x bfloat> %1330, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1588 = shufflevector <4 x bfloat> %1587, <4 x bfloat> %1337, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1589 = shufflevector <4 x bfloat> %1588, <4 x bfloat> %1345, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1590 = bitcast <4 x bfloat> %1589 to <4 x i16>
- %1591 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1590, <4 x float> %1558, i32 0, i32 0, i32 0)
- %1592 = shufflevector <4 x float> %1591, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1593 = shufflevector <1 x bfloat> %1327, <1 x bfloat> %1331, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1594 = shufflevector <4 x bfloat> %1593, <4 x bfloat> %1339, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1595 = shufflevector <4 x bfloat> %1594, <4 x bfloat> %1347, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1596 = bitcast <4 x bfloat> %1595 to <4 x i16>
- %1597 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1596, <4 x float> %1563, i32 0, i32 0, i32 0)
- %1598 = shufflevector <4 x float> %1597, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1599 = shufflevector <1 x bfloat> %1145, <1 x bfloat> %1147, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1600 = shufflevector <4 x bfloat> %1599, <4 x bfloat> %1151, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1601 = shufflevector <4 x bfloat> %1600, <4 x bfloat> %1155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1602 = bitcast <4 x bfloat> %1601 to <4 x i16>
- %1603 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1579, <4 x float> %1568, i32 0, i32 0, i32 0)
- %1604 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1584, <4 x float> %1569, i32 0, i32 0, i32 0)
- %1605 = shufflevector <4 x float> %1604, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1606 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1590, <4 x float> %1570, i32 0, i32 0, i32 0)
- %1607 = shufflevector <4 x float> %1606, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1608 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1596, <4 x float> %1571, i32 0, i32 0, i32 0)
- %1609 = shufflevector <4 x float> %1608, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1610 = shufflevector <4 x float> %1580, <4 x float> %1603, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1611 = shufflevector <32 x float> %1610, <32 x float> %1586, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1612 = shufflevector <32 x float> %1611, <32 x float> %1605, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1613 = shufflevector <32 x float> %1612, <32 x float> %1592, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1614 = shufflevector <32 x float> %1613, <32 x float> %1607, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1615 = shufflevector <32 x float> %1614, <32 x float> %1598, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison>
- %1616 = shufflevector <32 x float> %1615, <32 x float> %1609, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
- %1617 = insertelement <4 x i32> poison, i32 %38, i64 0
- %1618 = insertelement <4 x i32> poison, i32 %32, i64 0
- %1619 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1620 = shufflevector <4 x i32> %1618, <4 x i32> poison, <8 x i32> zeroinitializer
- %1621 = or disjoint <8 x i32> %1619, %1620
- %1622 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1623 = shufflevector <4 x i32> %1617, <4 x i32> poison, <8 x i32> zeroinitializer
- %1624 = icmp samesign ult <8 x i32> %1622, %1623
- br label %1625
-
-1625: ; preds = %1053, %1625
- %1626 = phi i32 [ 0, %1053 ], [ %1676, %1625 ]
- %1627 = shl nuw nsw i32 %1626, 3
- %1628 = extractelement <32 x float> %1616, i32 %1627
- %1629 = insertelement <4 x float> poison, float %1628, i64 0
- %1630 = or disjoint i32 %1627, 1
- %1631 = extractelement <32 x float> %1616, i32 %1630
- %1632 = insertelement <4 x float> %1629, float %1631, i64 1
- %1633 = or disjoint i32 %1627, 2
- %1634 = extractelement <32 x float> %1616, i32 %1633
- %1635 = insertelement <4 x float> %1632, float %1634, i64 2
- %1636 = or disjoint i32 %1627, 3
- %1637 = extractelement <32 x float> %1616, i32 %1636
- %1638 = insertelement <4 x float> %1635, float %1637, i64 3
- %1639 = or disjoint i32 %1627, 4
- %1640 = extractelement <32 x float> %1616, i32 %1639
- %1641 = insertelement <4 x float> poison, float %1640, i64 0
- %1642 = or disjoint i32 %1627, 5
- %1643 = extractelement <32 x float> %1616, i32 %1642
- %1644 = insertelement <4 x float> %1641, float %1643, i64 1
- %1645 = or disjoint i32 %1627, 6
- %1646 = extractelement <32 x float> %1616, i32 %1645
- %1647 = insertelement <4 x float> %1644, float %1646, i64 2
- %1648 = or disjoint i32 %1627, 7
- %1649 = extractelement <32 x float> %1616, i32 %1648
- %1650 = insertelement <4 x float> %1647, float %1649, i64 3
- %1651 = bitcast <4 x float> %1638 to <4 x i32>
- %1652 = bitcast <4 x float> %1650 to <4 x i32>
- %1653 = shl nuw nsw i32 %1626, 4
- %.reass = add nuw nsw i32 %1653, %154
- %1654 = icmp samesign ult i32 %.reass, %36
- %1655 = insertelement <4 x i1> poison, i1 %1654, i64 0
- %1656 = add nuw nsw i32 %.reass, %34
- %1657 = mul nuw nsw i32 %1656, 2016
- %1658 = insertelement <4 x i32> poison, i32 %1657, i64 0
- %1659 = shufflevector <4 x i32> %1658, <4 x i32> poison, <8 x i32> zeroinitializer
- %1660 = add nuw nsw <8 x i32> %1621, %1659
- %1661 = shufflevector <4 x i1> %1655, <4 x i1> poison, <8 x i32> zeroinitializer
- %1662 = and <8 x i1> %1624, %1661
- %1663 = shufflevector <4 x float> %1638, <4 x float> %1650, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1664 = fcmp uno <8 x float> %1663, zeroinitializer
- %1665 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1666 = add <8 x i32> %1665, splat (i32 32767)
- %1667 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1668 = lshr <8 x i32> %1667, splat (i32 16)
- %1669 = and <8 x i32> %1668, splat (i32 1)
- %1670 = add <8 x i32> %1666, %1669
- %1671 = lshr <8 x i32> %1670, splat (i32 16)
- %1672 = trunc nuw <8 x i32> %1671 to <8 x i16>
- %1673 = bitcast <8 x i16> %1672 to <8 x bfloat>
- %1674 = select <8 x i1> %1664, <8 x bfloat> splat (bfloat 0xR7FC0), <8 x bfloat> %1673
- %1675 = getelementptr bfloat, ptr addrspace(7) %1054, <8 x i32> %1660
- tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %1674, <8 x ptr addrspace(7)> align 2 %1675, <8 x i1> %1662)
- %1676 = add nuw nsw i32 %1626, 1
- %exitcond.not = icmp eq i32 %1676, 4
- br i1 %exitcond.not, label %1677, label %1625
+ %307 = and i32 %78, 56
+ %308 = getelementptr bfloat, ptr addrspace(3) null, i32 %307
+ store <8 x bfloat> %306, ptr addrspace(3) %308, align 8
+ %309 = trunc i32 %85 to i1
+ %310 = select i1 %309, <8 x bfloat> %86, <8 x bfloat> zeroinitializer
+ %311 = getelementptr bfloat, ptr addrspace(3) %1, i32 %307
+ store <8 x bfloat> %310, ptr addrspace(3) %311, align 8
+ %312 = trunc i32 %91 to i1
+ %313 = select i1 %312, <8 x bfloat> %92, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %313, ptr addrspace(3) null, align 8
+ %314 = trunc i32 %97 to i1
+ %315 = select i1 %314, <8 x bfloat> %98, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %315, ptr addrspace(3) %1, align 8
+ %316 = trunc i32 %100 to i1
+ %317 = select i1 %316, <8 x bfloat> %101, <8 x bfloat> zeroinitializer
+ %318 = and i32 %78, 1
+ %319 = getelementptr bfloat, ptr addrspace(3) %3, i32 %318
+ store <8 x bfloat> %317, ptr addrspace(3) %319, align 8
+ %320 = trunc i32 %105 to i1
+ %321 = select i1 %320, <8 x bfloat> %106, <8 x bfloat> zeroinitializer
+ %322 = getelementptr bfloat, ptr addrspace(3) %4, i32 %318
+ store <8 x bfloat> %321, ptr addrspace(3) %322, align 8
+ %323 = trunc i32 %107 to i1
+ %324 = select i1 %323, <8 x bfloat> %108, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %324, ptr addrspace(3) null, align 8
+ %325 = trunc i32 %109 to i1
+ %326 = select i1 %325, <8 x bfloat> %110, <8 x bfloat> zeroinitializer
+ %327 = getelementptr bfloat, ptr addrspace(3) null, i32 %318
+ store <8 x bfloat> %326, ptr addrspace(3) %327, align 8
+ %328 = trunc i32 %112 to i1
+ %329 = select i1 %328, <8 x bfloat> %113, <8 x bfloat> zeroinitializer
+ %330 = getelementptr bfloat, ptr addrspace(3) %5, i32 %318
+ store <8 x bfloat> %329, ptr addrspace(3) %330, align 8
+ %331 = trunc i32 %115 to i1
+ %332 = select i1 %331, <8 x bfloat> %116, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %332, ptr addrspace(3) %1, align 8
+ %333 = trunc i32 %118 to i1
+ %334 = select i1 %333, <8 x bfloat> %119, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %334, ptr addrspace(3) null, align 8
+ %335 = trunc i32 %121 to i1
+ %336 = select i1 %335, <8 x bfloat> splat (bfloat 0xR3F80), <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %122, ptr addrspace(3) %1, align 8
+ %337 = bitcast <8 x bfloat> %336 to i128
+ %338 = trunc i128 %337 to i16
+ %339 = bitcast i16 %338 to <1 x bfloat>
+ %340 = icmp ult i32 %76, 1
+ br i1 %340, label %73, label %341
-1677: ; preds = %1625
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
- ret void
+341: ; preds = %341, %73
+ %342 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> zeroinitializer, <4 x float> splat (float 0x7FF8000000000000), i32 0, i32 0, i32 0)
+ %343 = shufflevector <4 x float> %342, <4 x float> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %344 = extractelement <32 x float> %343, i32 %0
+ %345 = insertelement <4 x float> zeroinitializer, float %344, i64 0
+ %346 = shufflevector <4 x float> %345, <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %347 = fcmp uno <8 x float> %346, zeroinitializer
+ %348 = select <8 x i1> %347, <8 x bfloat> splat (bfloat 0xR3F80), <8 x bfloat> zeroinitializer
+ tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %348, <8 x ptr addrspace(7)> zeroinitializer, <8 x i1> %66)
+ br label %341
}
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
-declare void @llvm.assume(i1 noundef) #2
-
-; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) readnone, i16, i64, i32) #3
-
-; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare i32 @llvm.umin.i32(i32, i32) #3
-
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
-
-; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare i32 @llvm.smin.i32(i32, i32) #3
-
-; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
-declare void @llvm.amdgcn.s.barrier() #4
-
-; Function Attrs: alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
-declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16>, <4 x i16>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #5
-
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write)
-declare void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat>, <8 x ptr addrspace(7)>, <8 x i1>) #6
-
-; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
-declare void @llvm.amdgcn.sched.barrier(i32 immarg) #4
-
-attributes #0 = { alwaysinline nofree norecurse nounwind "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="256,256" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
-attributes #1 = { alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #2 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
-attributes #3 = { alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #4 = { alwaysinline convergent mustprogress nocallback nofree nounwind willreturn }
-attributes #5 = { alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
-attributes #6 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write) }
-
-!llvm.module.flags = !{!0, !1}
-
-!0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !{i32 1, !"amdhsa_code_object_version", i32 500}
-!2 = !{i32 256, i32 1, i32 1}
-!3 = !{!"amdgpu-synchronize-as", !"local"}
+attributes #0 = { "amdgpu-flat-work-group-size"="256,256" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(write) }
+attributes #5 = { convergent nocallback nofree nounwind willreturn }
>From 645973f40f8a318ea6931f9bd144cac925ce6948 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 13:07:21 -0600
Subject: [PATCH 09/17] Better name for function and remove unneccesary
attributes
---
llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
index 8f431e2e9a46c..fbd9b60ccfe0b 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -1,6 +1,6 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s | FileCheck %s
-define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(i32 %0, ptr addrspace(3) %1, ptr addrspace(3) %2, ptr addrspace(3) %3, ptr addrspace(3) %4, ptr addrspace(3) %5, ptr addrspace(3) %6, ptr addrspace(3) %7, ptr addrspace(3) %8, ptr addrspace(3) %9, ptr addrspace(3) %10, ptr addrspace(3) %11, ptr addrspace(3) %12, ptr addrspace(7) %invariant.gep, ptr addrspace(7) %gep, ptr addrspace(7) %gep106, ptr addrspace(7) %gep108, ptr addrspace(7) %gep112, ptr addrspace(7) %gep114, ptr addrspace(7) %gep116, ptr addrspace(7) %gep118, ptr addrspace(7) %gep120, ptr addrspace(7) %gep122, ptr addrspace(7) %gep124, ptr addrspace(3) %13, ptr addrspace(3) %14, ptr addrspace(3) %15, ptr addrspace(3) %16, ptr addrspace(3) %17, ptr addrspace(3) %18, ptr addrspace(3) %19, ptr addrspace(3) %20, ptr addrspace(3) %21, ptr addrspace(3) %22, ptr addrspace(3) %23, ptr addrspace(3) %24, ptr addrspace(3) %25, ptr addrspace(3) %26, ptr addrspace(3) %27, ptr addrspace(3) %28, ptr addrspace(3) %29, ptr addrspace(3) %30, ptr addrspace(3) %31, ptr addrspace(3) %32, ptr addrspace(3) %33, ptr addrspace(3) %34, ptr addrspace(3) %35, ptr addrspace(3) %36, ptr addrspace(3) %37, ptr addrspace(3) %38, ptr addrspace(3) %39, ptr addrspace(3) %40, ptr addrspace(3) %41, ptr addrspace(3) %42, ptr addrspace(3) %43, ptr addrspace(3) %44, ptr addrspace(3) %45, ptr addrspace(3) %46, ptr addrspace(3) %47, ptr addrspace(3) %48, ptr addrspace(3) %49, ptr addrspace(3) %50, ptr addrspace(3) %51, ptr addrspace(3) %52, ptr addrspace(3) %53, ptr addrspace(3) %54, ptr addrspace(3) %55, ptr addrspace(3) %56, ptr addrspace(3) %57, ptr addrspace(3) %58, ptr addrspace(3) %59, <4 x bfloat> %60, <4 x i16> %61, <4 x i16> %62, <4 x i16> %63, <4 x i16> %64, <4 x i16> %65, <8 x i1> %66, <4 x i16> %67, <4 x i16> %68, <4 x i16> %69, <4 x i16> %70, <8 x i1> %71) #0 {
+define amdgpu_kernel void @gemm_rewrite_reject(i32 %0, ptr addrspace(3) %1, ptr addrspace(3) %2, ptr addrspace(3) %3, ptr addrspace(3) %4, ptr addrspace(3) %5, ptr addrspace(3) %6, ptr addrspace(3) %7, ptr addrspace(3) %8, ptr addrspace(3) %9, ptr addrspace(3) %10, ptr addrspace(3) %11, ptr addrspace(3) %12, ptr addrspace(7) %invariant.gep, ptr addrspace(7) %gep, ptr addrspace(7) %gep106, ptr addrspace(7) %gep108, ptr addrspace(7) %gep112, ptr addrspace(7) %gep114, ptr addrspace(7) %gep116, ptr addrspace(7) %gep118, ptr addrspace(7) %gep120, ptr addrspace(7) %gep122, ptr addrspace(7) %gep124, ptr addrspace(3) %13, ptr addrspace(3) %14, ptr addrspace(3) %15, ptr addrspace(3) %16, ptr addrspace(3) %17, ptr addrspace(3) %18, ptr addrspace(3) %19, ptr addrspace(3) %20, ptr addrspace(3) %21, ptr addrspace(3) %22, ptr addrspace(3) %23, ptr addrspace(3) %24, ptr addrspace(3) %25, ptr addrspace(3) %26, ptr addrspace(3) %27, ptr addrspace(3) %28, ptr addrspace(3) %29, ptr addrspace(3) %30, ptr addrspace(3) %31, ptr addrspace(3) %32, ptr addrspace(3) %33, ptr addrspace(3) %34, ptr addrspace(3) %35, ptr addrspace(3) %36, ptr addrspace(3) %37, ptr addrspace(3) %38, ptr addrspace(3) %39, ptr addrspace(3) %40, ptr addrspace(3) %41, ptr addrspace(3) %42, ptr addrspace(3) %43, ptr addrspace(3) %44, ptr addrspace(3) %45, ptr addrspace(3) %46, ptr addrspace(3) %47, ptr addrspace(3) %48, ptr addrspace(3) %49, ptr addrspace(3) %50, ptr addrspace(3) %51, ptr addrspace(3) %52, ptr addrspace(3) %53, ptr addrspace(3) %54, ptr addrspace(3) %55, ptr addrspace(3) %56, ptr addrspace(3) %57, ptr addrspace(3) %58, ptr addrspace(3) %59, <4 x bfloat> %60, <4 x i16> %61, <4 x i16> %62, <4 x i16> %63, <4 x i16> %64, <4 x i16> %65, <8 x i1> %66, <4 x i16> %67, <4 x i16> %68, <4 x i16> %69, <4 x i16> %70, <8 x i1> %71) #0 {
; This should be rejected for MFMA rewrite by the scheduler so we should see
; no AGPR operands.
; CHECK-NOT: v_mfma_f32_16x16x16_bf16 a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}]
@@ -302,8 +302,3 @@ define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324b
}
attributes #0 = { "amdgpu-flat-work-group-size"="256,256" }
-attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #2 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #3 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
-attributes #4 = { nocallback nofree nosync nounwind willreturn memory(write) }
-attributes #5 = { convergent nocallback nofree nounwind willreturn }
>From 398be9d139087b985984f689d5c44148b98a6077 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Tue, 27 Jan 2026 08:49:52 -0600
Subject: [PATCH 10/17] Convert rewrite cost analysis to use double rather than
int64 to clean up logic
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 48 ++++++++-------------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 2 +-
2 files changed, 18 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 277ac7d4ecf4c..e1ad56a951b2e 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1330,11 +1330,11 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
if (!initHeuristics(RewriteCands, CopyForUse, CopyForDef))
return false;
- int64_t Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
+ double Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
// If we haven't found the beneficial conditions, prefer the VGPR form which
// may result in less cross RC copies.
- if (Cost > 0)
+ if (Cost > 0.0)
return false;
return rewrite(RewriteCands);
@@ -2251,15 +2251,14 @@ bool RewriteMFMAFormStage::initHeuristics(
return Changed;
}
-int64_t RewriteMFMAFormStage::getRewriteCost(
+double RewriteMFMAFormStage::getRewriteCost(
const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
MachineBlockFrequencyInfo *MBFI = DAG.MBFI;
- int64_t BestSpillCost = 0;
- int64_t Cost = 0;
- uint64_t EntryFreq = MBFI->getEntryFreq().getFrequency();
+ double BestSpillCost = 0.0;
+ double Cost = 0.0;
std::pair<unsigned, unsigned> MaxVectorRegs =
ST.getMaxNumVectorRegs(MF.getFunction());
@@ -2283,28 +2282,18 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
unsigned SpillCostAfter = PressureAfter.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
- uint64_t BlockFreq =
- MBFI->getBlockFreq(DAG.Regions[Region].first->getParent())
- .getFrequency();
-
- bool RelativeFreqIsDenom = EntryFreq > BlockFreq;
- uint64_t RelativeFreq = EntryFreq && BlockFreq
- ? (RelativeFreqIsDenom ? EntryFreq / BlockFreq
- : BlockFreq / EntryFreq)
- : 1;
+ MachineBasicBlock *MBB = DAG.Regions[Region].first->getParent();
+ double BlockFreq = MBFI->getBlockFreqRelativeToEntryBlock(MBB);
// This assumes perfect spilling / splitting -- using one spill / copy
// instruction and one restoreFrom / copy for each excess register,
- int64_t SpillCost = ((int)SpillCostAfter - (int)SpillCostBefore) * 2;
+ double SpillCost = ((double)SpillCostAfter - (double)SpillCostBefore) * 2;
// Also account for the block frequency.
- if (RelativeFreqIsDenom)
- SpillCost /= (int64_t)RelativeFreq;
- else
- SpillCost *= (int64_t)RelativeFreq;
+ SpillCost *= BlockFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0)
+ if (SpillCost > 0.0)
return SpillCost;
if (SpillCost < BestSpillCost)
@@ -2314,31 +2303,28 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
// Set the cost to the largest decrease in spill cost in order to not double
// count spill reductions.
Cost = BestSpillCost;
- assert(Cost <= 0);
+ assert(Cost <= 0.0);
- unsigned CopyCost = 0;
+ double CopyCost = 0.0;
// For each CopyForDef, increase the cost by the register size while
// accounting for block frequency.
for (MachineInstr *DefMI : CopyForDef) {
Register DefReg = DefMI->getOperand(0).getReg();
- uint64_t DefFreq =
- EntryFreq
- ? MBFI->getBlockFreq(DefMI->getParent()).getFrequency() / EntryFreq
- : 1;
+ MachineBasicBlock *DefMBB = DefMI->getParent();
+ double DefFreq = MBFI->getBlockFreqRelativeToEntryBlock(DefMBB);
const TargetRegisterClass *RC = DAG.MRI.getRegClass(DefReg);
- CopyCost += RC->getCopyCost() * DefFreq;
+ CopyCost += (double) RC->getCopyCost() * DefFreq;
}
// Account for CopyForUse copies in each block that the register is used.
for (auto &[UseBlock, UseRegs] : CopyForUse) {
- uint64_t UseFreq =
- EntryFreq ? MBFI->getBlockFreq(UseBlock).getFrequency() / EntryFreq : 1;
+ uint64_t UseFreq = MBFI->getBlockFreqRelativeToEntryBlock(UseBlock);
for (Register UseReg : UseRegs) {
const TargetRegisterClass *RC = DAG.MRI.getRegClass(UseReg);
- CopyCost += RC->getCopyCost() * UseFreq;
+ CopyCost += (double) RC->getCopyCost() * UseFreq;
}
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 6b6a40365b52d..0f73cc4fda10f 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -442,7 +442,7 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Calculate the rewrite cost and undo the state change (e.g. rewriting) done
/// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
/// costs, and \p RewriteCands to undo rewriting.
- int64_t getRewriteCost(
+ double getRewriteCost(
const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef);
>From 85bca088211c26860a3966043119fea017759796 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Mon, 2 Feb 2026 08:58:03 -0600
Subject: [PATCH 11/17] Add some additional debug prints
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index e1ad56a951b2e..c3e24d9a02f07 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2273,6 +2273,8 @@ double RewriteMFMAFormStage::getRewriteCost(
GCNRegPressure &PressureBefore = DAG.Pressure[Region];
unsigned SpillCostBefore = PressureBefore.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Region " << Region
+ << " spill cost before: " << SpillCostBefore << "\n");
// For the cases we care about (i.e. ArchVGPR usage is greater than the
// addressable limit), rewriting alone should bring pressure to manageable
@@ -2281,6 +2283,8 @@ double RewriteMFMAFormStage::getRewriteCost(
GCNRegPressure PressureAfter = DAG.getRealRegPressure(Region);
unsigned SpillCostAfter = PressureAfter.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Region " << Region
+ << " spill cost after: " << SpillCostAfter << "\n");
MachineBasicBlock *MBB = DAG.Regions[Region].first->getParent();
double BlockFreq = MBFI->getBlockFreqRelativeToEntryBlock(MBB);
@@ -2302,32 +2306,37 @@ double RewriteMFMAFormStage::getRewriteCost(
// Set the cost to the largest decrease in spill cost in order to not double
// count spill reductions.
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: BestSpillCost: " << BestSpillCost << "\n");
Cost = BestSpillCost;
assert(Cost <= 0.0);
- double CopyCost = 0.0;
-
// For each CopyForDef, increase the cost by the register size while
// accounting for block frequency.
+ double DefCopyCost = 0.0;
for (MachineInstr *DefMI : CopyForDef) {
Register DefReg = DefMI->getOperand(0).getReg();
MachineBasicBlock *DefMBB = DefMI->getParent();
double DefFreq = MBFI->getBlockFreqRelativeToEntryBlock(DefMBB);
const TargetRegisterClass *RC = DAG.MRI.getRegClass(DefReg);
- CopyCost += (double) RC->getCopyCost() * DefFreq;
+ DefCopyCost += (double)RC->getCopyCost() * DefFreq;
}
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Def copy Costs: " << DefCopyCost << "\n");
// Account for CopyForUse copies in each block that the register is used.
+ double UseCopyCost = 0.0;
for (auto &[UseBlock, UseRegs] : CopyForUse) {
uint64_t UseFreq = MBFI->getBlockFreqRelativeToEntryBlock(UseBlock);
for (Register UseReg : UseRegs) {
const TargetRegisterClass *RC = DAG.MRI.getRegClass(UseReg);
- CopyCost += (double) RC->getCopyCost() * UseFreq;
+ UseCopyCost += (double)RC->getCopyCost() * UseFreq;
}
}
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Use copy Costs: " << UseCopyCost << "\n");
+ double CopyCost = UseCopyCost + DefCopyCost;
+
// Reset the classes that were changed to AGPR for better RB analysis.
// We must do rewriting after copy-insertion, as some defs of the register
// may require VGPR. Additionally, if we bail out and don't perform the
>From 8fc3df29392e931d8905c923de492aaacb086d2b Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Tue, 10 Feb 2026 08:50:46 -0600
Subject: [PATCH 12/17] Reenable AMDGPU MFMA rewrite scheduler stage.
After performance testing, it was determined that the large number of copies that
are inserted outside the loop are more than offset by better allocation within
the loop as a result of the rewrite. Additionally, there are two other fairly
minor changes in this commit:
- Fix for upstream issue #177696. The need for this fix will go away when we
remove the need to do an early conversion to MFMA form in order to get
proper spill cost analysis of the potential rewritten code.
- Convert cost logic to double from int64. This cleans up the logic a bit.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index c3e24d9a02f07..79603ccbb5ead 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,7 +2184,6 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
-
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
>From 35e269db5e3c4365c1e1bcfe8f71cd2fc5df7926 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:14:58 -0600
Subject: [PATCH 13/17] Reenable AMDGPU MFMA rewrite scheduler stage.
After performance testing, it was determined that the large number of
copies that are inserted outside the loop are more than offset by better
allocation within the loop as a result of the rewrite. Additionally,
there is a minor cleanup of the cost analysis code.
This reverts commit e47677c3bc38af91e237a390dc09545939be5cdb.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 79603ccbb5ead..688e1ddd54fa2 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,6 +2184,7 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
+
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
@@ -2350,8 +2351,6 @@ double RewriteMFMAFormStage::getRewriteCost(
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
- if (!Src2->isReg())
- continue;
// Have to get src types separately since subregs may cause C and D
// registers to be different types even though the actual operand is
>From e9316ba8f304506883aa137623d8df6579ce82c4 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 12:27:18 -0600
Subject: [PATCH 14/17] Fix rebase merge errors
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 688e1ddd54fa2..783a6f8440d95 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(true));
+ cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
const unsigned ScheduleMetrics::ScaleFactor = 100;
@@ -2351,6 +2351,8 @@ double RewriteMFMAFormStage::getRewriteCost(
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
+ if (!Src2->isReg())
+ continue;
// Have to get src types separately since subregs may cause C and D
// registers to be different types even though the actual operand is
>From afef95f174c5b28f6e963404cfc95c448059c939 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 19 Feb 2026 13:46:10 -0600
Subject: [PATCH 15/17] remove test
---
.../AMDGPU/sched_mfma_rewrite_reject.ll | 304 ------------------
1 file changed, 304 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
deleted file mode 100644
index fbd9b60ccfe0b..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
+++ /dev/null
@@ -1,304 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s | FileCheck %s
-
-define amdgpu_kernel void @gemm_rewrite_reject(i32 %0, ptr addrspace(3) %1, ptr addrspace(3) %2, ptr addrspace(3) %3, ptr addrspace(3) %4, ptr addrspace(3) %5, ptr addrspace(3) %6, ptr addrspace(3) %7, ptr addrspace(3) %8, ptr addrspace(3) %9, ptr addrspace(3) %10, ptr addrspace(3) %11, ptr addrspace(3) %12, ptr addrspace(7) %invariant.gep, ptr addrspace(7) %gep, ptr addrspace(7) %gep106, ptr addrspace(7) %gep108, ptr addrspace(7) %gep112, ptr addrspace(7) %gep114, ptr addrspace(7) %gep116, ptr addrspace(7) %gep118, ptr addrspace(7) %gep120, ptr addrspace(7) %gep122, ptr addrspace(7) %gep124, ptr addrspace(3) %13, ptr addrspace(3) %14, ptr addrspace(3) %15, ptr addrspace(3) %16, ptr addrspace(3) %17, ptr addrspace(3) %18, ptr addrspace(3) %19, ptr addrspace(3) %20, ptr addrspace(3) %21, ptr addrspace(3) %22, ptr addrspace(3) %23, ptr addrspace(3) %24, ptr addrspace(3) %25, ptr addrspace(3) %26, ptr addrspace(3) %27, ptr addrspace(3) %28, ptr addrspace(3) %29, ptr addrspace(3) %30, ptr addrspace(3) %31, ptr addrspace(3) %32, ptr addrspace(3) %33, ptr addrspace(3) %34, ptr addrspace(3) %35, ptr addrspace(3) %36, ptr addrspace(3) %37, ptr addrspace(3) %38, ptr addrspace(3) %39, ptr addrspace(3) %40, ptr addrspace(3) %41, ptr addrspace(3) %42, ptr addrspace(3) %43, ptr addrspace(3) %44, ptr addrspace(3) %45, ptr addrspace(3) %46, ptr addrspace(3) %47, ptr addrspace(3) %48, ptr addrspace(3) %49, ptr addrspace(3) %50, ptr addrspace(3) %51, ptr addrspace(3) %52, ptr addrspace(3) %53, ptr addrspace(3) %54, ptr addrspace(3) %55, ptr addrspace(3) %56, ptr addrspace(3) %57, ptr addrspace(3) %58, ptr addrspace(3) %59, <4 x bfloat> %60, <4 x i16> %61, <4 x i16> %62, <4 x i16> %63, <4 x i16> %64, <4 x i16> %65, <8 x i1> %66, <4 x i16> %67, <4 x i16> %68, <4 x i16> %69, <4 x i16> %70, <8 x i1> %71) #0 {
-; This should be rejected for MFMA rewrite by the scheduler so we should see
-; no AGPR operands.
-; CHECK-NOT: v_mfma_f32_16x16x16_bf16 a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}]
- br label %73
-73: ; preds = %73, %72
- %74 = phi <1 x bfloat> [ zeroinitializer, %72 ], [ %339, %73 ]
- %75 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %72 ], [ %304, %73 ]
- %76 = phi i32 [ 0, %72 ], [ %77, %73 ]
- %77 = add i32 %76, 1
- %78 = tail call i32 @llvm.amdgcn.workitem.id.x()
- %79 = tail call i32 @llvm.umin.i32(i32 %77, i32 %78)
- %80 = sub i32 0, %79
- %81 = tail call i32 @llvm.smin.i32(i32 %80, i32 0)
- %82 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
- %83 = sub i32 0, %76
- %84 = tail call i32 @llvm.umin.i32(i32 %83, i32 1)
- %85 = tail call i32 @llvm.umin.i32(i32 %84, i32 %0)
- %86 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
- %87 = or i32 %78, 512
- %88 = lshr i32 %87, 1
- %89 = tail call i32 @llvm.umin.i32(i32 %83, i32 %88)
- %90 = sub i32 0, %89
- %91 = tail call i32 @llvm.smin.i32(i32 %90, i32 0)
- %92 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
- %93 = or i32 %78, 768
- %94 = lshr i32 %93, 1
- %95 = tail call i32 @llvm.umin.i32(i32 %83, i32 %94)
- %96 = sub i32 0, %95
- %97 = tail call i32 @llvm.smin.i32(i32 %96, i32 0)
- %98 = load <8 x bfloat>, ptr addrspace(7) %invariant.gep, align 2
- %99 = lshr i32 %78, 1
- %100 = tail call i32 @llvm.umin.i32(i32 %77, i32 %99)
- %101 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
- %102 = or i32 %78, 256
- %103 = tail call i32 @llvm.umin.i32(i32 %83, i32 %102)
- %104 = sub i32 0, %103
- %105 = tail call i32 @llvm.smin.i32(i32 %104, i32 0)
- %106 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
- %107 = tail call i32 @llvm.umin.i32(i32 %83, i32 %87)
- %108 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
- %109 = tail call i32 @llvm.umin.i32(i32 %83, i32 %93)
- %110 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
- %111 = or i32 %78, 64
- %112 = tail call i32 @llvm.umin.i32(i32 %83, i32 %111)
- %113 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
- %114 = or i32 %78, 80
- %115 = tail call i32 @llvm.umin.i32(i32 %83, i32 %114)
- %116 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
- %117 = or i32 %78, 96
- %118 = tail call i32 @llvm.umin.i32(i32 %83, i32 %117)
- %119 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
- %120 = or i32 %78, 1
- %121 = tail call i32 @llvm.umin.i32(i32 %77, i32 %120)
- %122 = load <8 x bfloat>, ptr addrspace(7) null, align 2
- fence syncscope("workgroup") release
- %123 = load <1 x bfloat>, ptr addrspace(3) %13, align 2
- %124 = load <1 x bfloat>, ptr addrspace(3) %14, align 2
- %125 = load <1 x bfloat>, ptr addrspace(3) %15, align 2
- %126 = load <1 x bfloat>, ptr addrspace(3) %16, align 2
- %127 = load <1 x bfloat>, ptr addrspace(3) %17, align 2
- %128 = shufflevector <1 x bfloat> %127, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %129 = load <1 x bfloat>, ptr addrspace(3) %18, align 2
- %130 = load <1 x bfloat>, ptr addrspace(3) %19, align 2
- %131 = shufflevector <1 x bfloat> %130, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %132 = load <1 x bfloat>, ptr addrspace(3) %20, align 2
- %133 = load <1 x bfloat>, ptr addrspace(3) %21, align 2
- %134 = load <1 x bfloat>, ptr addrspace(3) %22, align 2
- %135 = load <1 x bfloat>, ptr addrspace(3) %23, align 2
- %136 = load <1 x bfloat>, ptr addrspace(3) %24, align 2
- %137 = load <1 x bfloat>, ptr addrspace(3) %40, align 2
- %138 = load <1 x bfloat>, ptr addrspace(3) %25, align 2
- %139 = shufflevector <1 x bfloat> %138, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %140 = load <1 x bfloat>, ptr addrspace(3) %26, align 2
- %141 = load <1 x bfloat>, ptr addrspace(3) %27, align 2
- %142 = load <1 x bfloat>, ptr addrspace(3) %28, align 2
- %143 = load <1 x bfloat>, ptr addrspace(3) %29, align 2
- %144 = load <1 x bfloat>, ptr addrspace(3) %31, align 2
- %145 = load <1 x bfloat>, ptr addrspace(3) %32, align 2
- %146 = load <1 x bfloat>, ptr addrspace(3) %33, align 2
- %147 = load <1 x bfloat>, ptr addrspace(3) %34, align 2
- %148 = load <1 x bfloat>, ptr addrspace(3) %35, align 2
- %149 = load <1 x bfloat>, ptr addrspace(3) %36, align 2
- %150 = load <1 x bfloat>, ptr addrspace(3) %37, align 2
- %151 = shufflevector <1 x bfloat> %150, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %152 = load <1 x bfloat>, ptr addrspace(3) %38, align 2
- %153 = shufflevector <1 x bfloat> %152, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %154 = load <1 x bfloat>, ptr addrspace(3) %39, align 2
- %155 = shufflevector <1 x bfloat> %154, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %156 = load <1 x bfloat>, ptr addrspace(3) %9, align 2
- %157 = load <1 x bfloat>, ptr addrspace(3) %43, align 2
- %158 = load <1 x bfloat>, ptr addrspace(3) %44, align 2
- %159 = load <1 x bfloat>, ptr addrspace(3) %45, align 2
- %160 = load <1 x bfloat>, ptr addrspace(3) %46, align 2
- %161 = load <1 x bfloat>, ptr addrspace(3) %47, align 2
- %162 = load <1 x bfloat>, ptr addrspace(3) %48, align 2
- %163 = load <1 x bfloat>, ptr addrspace(3) %49, align 2
- %164 = load <1 x bfloat>, ptr addrspace(3) %50, align 2
- %165 = shufflevector <1 x bfloat> %164, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %166 = load <1 x bfloat>, ptr addrspace(3) %51, align 2
- %167 = load <1 x bfloat>, ptr addrspace(3) %7, align 2
- %168 = shufflevector <1 x bfloat> %167, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %169 = load <1 x bfloat>, ptr addrspace(3) %53, align 2
- %170 = load <1 x bfloat>, ptr addrspace(3) %54, align 2
- %171 = load <1 x bfloat>, ptr addrspace(3) %55, align 2
- %172 = load <1 x bfloat>, ptr addrspace(3) %56, align 2
- %173 = load <1 x bfloat>, ptr addrspace(3) %3, align 2
- %174 = load <1 x bfloat>, ptr addrspace(3) %6, align 2
- %175 = load <1 x bfloat>, ptr addrspace(3) %5, align 2
- %176 = load <1 x bfloat>, ptr addrspace(3) %57, align 2
- %177 = load <1 x bfloat>, ptr addrspace(3) %8, align 2
- %178 = load <1 x bfloat>, ptr addrspace(3) %11, align 2
- %179 = load <1 x bfloat>, ptr addrspace(3) %30, align 2
- %180 = load <1 x bfloat>, ptr addrspace(3) %59, align 2
- %181 = load <1 x bfloat>, ptr addrspace(3) %58, align 2
- %182 = load <1 x bfloat>, ptr addrspace(3) %52, align 2
- %183 = load <1 x bfloat>, ptr addrspace(3) %42, align 2
- %184 = load <1 x bfloat>, ptr addrspace(3) %1, align 2
- %185 = load <1 x bfloat>, ptr addrspace(3) %4, align 2
- %186 = load <1 x bfloat>, ptr addrspace(3) %10, align 2
- %187 = load <1 x bfloat>, ptr addrspace(3) %41, align 2
- %188 = load <1 x bfloat>, ptr addrspace(3) %2, align 2
- %189 = load <1 x bfloat>, ptr addrspace(3) %12, align 2
- %190 = shufflevector <1 x bfloat> %74, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %191 = shufflevector <4 x bfloat> zeroinitializer, <4 x bfloat> %190, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %192 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 3, 1, 0
- %193 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %67, <4 x float> %192, i32 0, i32 0, i32 0)
- %194 = shufflevector <1 x bfloat> %123, <1 x bfloat> %125, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %195 = shufflevector <4 x bfloat> %194, <4 x bfloat> %128, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %196 = shufflevector <1 x bfloat> %156, <1 x bfloat> %160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %197 = shufflevector <4 x bfloat> %196, <4 x bfloat> %165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %198 = shufflevector <1 x bfloat> %170, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %199 = shufflevector <4 x bfloat> %195, <4 x bfloat> %131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %200 = bitcast <4 x bfloat> %199 to <4 x i16>
- %201 = shufflevector <4 x bfloat> %197, <4 x bfloat> %198, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %202 = bitcast <4 x bfloat> %201 to <4 x i16>
- %203 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 0, 0, 0
- %204 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %61, <4 x float> %203, i32 0, i32 0, i32 0)
- %205 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %200, <4 x i16> %202, <4 x float> %204, i32 0, i32 0, i32 0)
- %206 = shufflevector <1 x bfloat> %157, <1 x bfloat> %161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %207 = shufflevector <1 x bfloat> %166, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %208 = shufflevector <4 x bfloat> %206, <4 x bfloat> %207, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %209 = shufflevector <1 x bfloat> %171, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %210 = shufflevector <4 x bfloat> %208, <4 x bfloat> %209, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %211 = shufflevector <1 x bfloat> %158, <1 x bfloat> %162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %212 = shufflevector <4 x bfloat> %211, <4 x bfloat> %168, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %213 = shufflevector <1 x bfloat> %172, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %214 = shufflevector <1 x bfloat> %159, <1 x bfloat> %163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %215 = shufflevector <1 x bfloat> %169, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %216 = shufflevector <4 x bfloat> %214, <4 x bfloat> %215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %217 = shufflevector <1 x bfloat> %173, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %218 = shufflevector <4 x bfloat> %216, <4 x bfloat> %217, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %219 = bitcast <4 x bfloat> %218 to <4 x i16>
- %220 = shufflevector <1 x bfloat> %124, <1 x bfloat> %126, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %221 = shufflevector <1 x bfloat> %129, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %222 = bitcast <4 x bfloat> %210 to <4 x i16>
- %223 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 1, 1, 0
- %224 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %222, <4 x float> %223, i32 0, i32 0, i32 0)
- %225 = shufflevector <4 x bfloat> %220, <4 x bfloat> %221, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %226 = shufflevector <1 x bfloat> %132, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %227 = shufflevector <4 x bfloat> %225, <4 x bfloat> %226, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %228 = bitcast <4 x bfloat> %227 to <4 x i16>
- %229 = shufflevector <4 x bfloat> %212, <4 x bfloat> %213, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %230 = bitcast <4 x bfloat> %229 to <4 x i16>
- %231 = bitcast <4 x bfloat> %191 to <4 x i16>
- %232 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 2, 1, 0
- %233 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %62, <4 x i16> %231, <4 x float> %232, i32 0, i32 0, i32 0)
- %234 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %228, <4 x i16> %230, <4 x float> %233, i32 0, i32 0, i32 0)
- %235 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %219, <4 x float> %193, i32 0, i32 0, i32 0)
- %236 = shufflevector <1 x bfloat> %133, <1 x bfloat> %135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %237 = shufflevector <1 x bfloat> %137, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %238 = shufflevector <4 x bfloat> %236, <4 x bfloat> %237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %239 = shufflevector <4 x bfloat> %238, <4 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %240 = shufflevector <1 x bfloat> %174, <1 x bfloat> %178, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %241 = shufflevector <1 x bfloat> %182, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %242 = shufflevector <4 x bfloat> %240, <4 x bfloat> %241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %243 = shufflevector <1 x bfloat> %185, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %244 = shufflevector <4 x bfloat> %242, <4 x bfloat> %243, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %245 = bitcast <4 x bfloat> %239 to <4 x i16>
- %246 = bitcast <4 x bfloat> %244 to <4 x i16>
- %247 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %245, <4 x i16> %246, <4 x float> %205, i32 0, i32 0, i32 0)
- %248 = shufflevector <1 x bfloat> %175, <1 x bfloat> %179, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %249 = shufflevector <1 x bfloat> %183, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %250 = shufflevector <4 x bfloat> %248, <4 x bfloat> %249, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %251 = shufflevector <1 x bfloat> %186, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %252 = shufflevector <4 x bfloat> %250, <4 x bfloat> %251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %253 = bitcast <4 x bfloat> %252 to <4 x i16>
- %254 = shufflevector <1 x bfloat> %176, <1 x bfloat> %180, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %255 = shufflevector <4 x bfloat> %254, <4 x bfloat> %60, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %256 = shufflevector <1 x bfloat> %187, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %257 = shufflevector <4 x bfloat> %255, <4 x bfloat> %256, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %258 = bitcast <4 x bfloat> %257 to <4 x i16>
- %259 = shufflevector <1 x bfloat> %177, <1 x bfloat> %181, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %260 = shufflevector <1 x bfloat> %184, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %261 = shufflevector <4 x bfloat> %259, <4 x bfloat> %260, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %262 = shufflevector <1 x bfloat> %188, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %263 = shufflevector <4 x bfloat> %261, <4 x bfloat> %262, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %264 = bitcast <4 x bfloat> %263 to <4 x i16>
- %265 = shufflevector <1 x bfloat> %134, <1 x bfloat> %136, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %266 = shufflevector <4 x bfloat> %265, <4 x bfloat> %139, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %267 = bitcast <4 x bfloat> %266 to <4 x i16>
- %268 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %267, <4 x i16> %253, <4 x float> %224, i32 0, i32 0, i32 0)
- %269 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %258, <4 x float> %234, i32 0, i32 0, i32 0)
- %270 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %264, <4 x float> %235, i32 0, i32 0, i32 0)
- %271 = shufflevector <1 x bfloat> %140, <1 x bfloat> %141, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %272 = shufflevector <1 x bfloat> %142, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %273 = shufflevector <4 x bfloat> %271, <4 x bfloat> %272, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %274 = shufflevector <1 x bfloat> %143, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %275 = shufflevector <4 x bfloat> %273, <4 x bfloat> %274, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %276 = shufflevector <1 x bfloat> splat (bfloat 0xR3F80), <1 x bfloat> %189, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %277 = bitcast <4 x bfloat> %275 to <4 x i16>
- %278 = bitcast <4 x bfloat> %276 to <4 x i16>
- %279 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %277, <4 x i16> %278, <4 x float> %247, i32 0, i32 0, i32 0)
- %280 = shufflevector <1 x bfloat> %144, <1 x bfloat> %145, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %281 = shufflevector <1 x bfloat> %146, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %282 = shufflevector <4 x bfloat> %280, <4 x bfloat> %281, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %283 = shufflevector <1 x bfloat> %147, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %284 = shufflevector <4 x bfloat> %282, <4 x bfloat> %283, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %285 = bitcast <4 x bfloat> %284 to <4 x i16>
- %286 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %285, <4 x i16> %63, <4 x float> %279, i32 0, i32 0, i32 0)
- %287 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %69, <4 x float> %268, i32 0, i32 0, i32 0)
- %288 = shufflevector <1 x bfloat> %148, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %289 = shufflevector <1 x bfloat> %149, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %290 = shufflevector <4 x bfloat> %288, <4 x bfloat> %289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %291 = shufflevector <4 x bfloat> %290, <4 x bfloat> %153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %292 = bitcast <4 x bfloat> %291 to <4 x i16>
- %293 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %292, <4 x i16> splat (i16 1), <4 x float> %286, i32 0, i32 0, i32 0)
- %294 = shufflevector <4 x bfloat> %151, <4 x bfloat> %155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %295 = bitcast <4 x bfloat> %294 to <4 x i16>
- %296 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %70, <4 x i16> zeroinitializer, <4 x float> %293, i32 0, i32 0, i32 0)
- %297 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %65, <4 x i16> zeroinitializer, <4 x float> %296, i32 0, i32 0, i32 0)
- %298 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %297, 0, 0, 0
- %299 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %295, <4 x i16> zeroinitializer, <4 x float> %287, i32 0, i32 0, i32 0)
- %300 = insertvalue [4 x [2 x [1 x <4 x float>]]] %298, <4 x float> %299, 1, 1, 0
- %301 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %64, <4 x float> %269, i32 0, i32 0, i32 0)
- %302 = insertvalue [4 x [2 x [1 x <4 x float>]]] %300, <4 x float> %301, 2, 1, 0
- %303 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %68, <4 x i16> zeroinitializer, <4 x float> %270, i32 0, i32 0, i32 0)
- %304 = insertvalue [4 x [2 x [1 x <4 x float>]]] %302, <4 x float> %303, 3, 1, 0
- %305 = trunc i32 %81 to i1
- %306 = select i1 %305, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
- tail call void @llvm.amdgcn.sched.barrier(i32 0)
- %307 = and i32 %78, 56
- %308 = getelementptr bfloat, ptr addrspace(3) null, i32 %307
- store <8 x bfloat> %306, ptr addrspace(3) %308, align 8
- %309 = trunc i32 %85 to i1
- %310 = select i1 %309, <8 x bfloat> %86, <8 x bfloat> zeroinitializer
- %311 = getelementptr bfloat, ptr addrspace(3) %1, i32 %307
- store <8 x bfloat> %310, ptr addrspace(3) %311, align 8
- %312 = trunc i32 %91 to i1
- %313 = select i1 %312, <8 x bfloat> %92, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %313, ptr addrspace(3) null, align 8
- %314 = trunc i32 %97 to i1
- %315 = select i1 %314, <8 x bfloat> %98, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %315, ptr addrspace(3) %1, align 8
- %316 = trunc i32 %100 to i1
- %317 = select i1 %316, <8 x bfloat> %101, <8 x bfloat> zeroinitializer
- %318 = and i32 %78, 1
- %319 = getelementptr bfloat, ptr addrspace(3) %3, i32 %318
- store <8 x bfloat> %317, ptr addrspace(3) %319, align 8
- %320 = trunc i32 %105 to i1
- %321 = select i1 %320, <8 x bfloat> %106, <8 x bfloat> zeroinitializer
- %322 = getelementptr bfloat, ptr addrspace(3) %4, i32 %318
- store <8 x bfloat> %321, ptr addrspace(3) %322, align 8
- %323 = trunc i32 %107 to i1
- %324 = select i1 %323, <8 x bfloat> %108, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %324, ptr addrspace(3) null, align 8
- %325 = trunc i32 %109 to i1
- %326 = select i1 %325, <8 x bfloat> %110, <8 x bfloat> zeroinitializer
- %327 = getelementptr bfloat, ptr addrspace(3) null, i32 %318
- store <8 x bfloat> %326, ptr addrspace(3) %327, align 8
- %328 = trunc i32 %112 to i1
- %329 = select i1 %328, <8 x bfloat> %113, <8 x bfloat> zeroinitializer
- %330 = getelementptr bfloat, ptr addrspace(3) %5, i32 %318
- store <8 x bfloat> %329, ptr addrspace(3) %330, align 8
- %331 = trunc i32 %115 to i1
- %332 = select i1 %331, <8 x bfloat> %116, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %332, ptr addrspace(3) %1, align 8
- %333 = trunc i32 %118 to i1
- %334 = select i1 %333, <8 x bfloat> %119, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %334, ptr addrspace(3) null, align 8
- %335 = trunc i32 %121 to i1
- %336 = select i1 %335, <8 x bfloat> splat (bfloat 0xR3F80), <8 x bfloat> zeroinitializer
- store <8 x bfloat> %122, ptr addrspace(3) %1, align 8
- %337 = bitcast <8 x bfloat> %336 to i128
- %338 = trunc i128 %337 to i16
- %339 = bitcast i16 %338 to <1 x bfloat>
- %340 = icmp ult i32 %76, 1
- br i1 %340, label %73, label %341
-
-341: ; preds = %341, %73
- %342 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> zeroinitializer, <4 x float> splat (float 0x7FF8000000000000), i32 0, i32 0, i32 0)
- %343 = shufflevector <4 x float> %342, <4 x float> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %344 = extractelement <32 x float> %343, i32 %0
- %345 = insertelement <4 x float> zeroinitializer, float %344, i64 0
- %346 = shufflevector <4 x float> %345, <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %347 = fcmp uno <8 x float> %346, zeroinitializer
- %348 = select <8 x i1> %347, <8 x bfloat> splat (bfloat 0xR3F80), <8 x bfloat> zeroinitializer
- tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %348, <8 x ptr addrspace(7)> zeroinitializer, <8 x i1> %66)
- br label %341
-}
-
-attributes #0 = { "amdgpu-flat-work-group-size"="256,256" }
>From e431eb041415a7aeb7a42353cb821d8afde34eb8 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 19 Feb 2026 13:49:53 -0600
Subject: [PATCH 16/17] Flag flip only
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 93 ++++++++++-----------
1 file changed, 43 insertions(+), 50 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 783a6f8440d95..3dbace757add2 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1330,11 +1330,11 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
if (!initHeuristics(RewriteCands, CopyForUse, CopyForDef))
return false;
- double Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
+ int64_t Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
// If we haven't found the beneficial conditions, prefer the VGPR form which
// may result in less cross RC copies.
- if (Cost > 0.0)
+ if (Cost > 0)
return false;
return rewrite(RewriteCands);
@@ -2233,17 +2233,11 @@ bool RewriteMFMAFormStage::initHeuristics(
}
// Do the rewrite to allow for updated RP calculation.
- const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
- const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
- DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
- if (Src2->isReg()) {
- // Have to get src types separately since subregs may cause C and D
- // registers to be different types even though the actual operand is
- // the same size.
- const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
- DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
- }
+ const TargetRegisterClass *VGPRRC = DAG.MRI.getRegClass(Dst.getReg());
+ const TargetRegisterClass *AGPRRC = SRI->getEquivalentAGPRClass(VGPRRC);
+ DAG.MRI.setRegClass(Dst.getReg(), AGPRRC);
+ if (Src2->isReg())
+ DAG.MRI.setRegClass(Src2->getReg(), AGPRRC);
Changed = true;
}
}
@@ -2251,14 +2245,15 @@ bool RewriteMFMAFormStage::initHeuristics(
return Changed;
}
-double RewriteMFMAFormStage::getRewriteCost(
+int64_t RewriteMFMAFormStage::getRewriteCost(
const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
MachineBlockFrequencyInfo *MBFI = DAG.MBFI;
- double BestSpillCost = 0.0;
- double Cost = 0.0;
+ int64_t BestSpillCost = 0;
+ int64_t Cost = 0;
+ uint64_t EntryFreq = MBFI->getEntryFreq().getFrequency();
std::pair<unsigned, unsigned> MaxVectorRegs =
ST.getMaxNumVectorRegs(MF.getFunction());
@@ -2273,8 +2268,6 @@ double RewriteMFMAFormStage::getRewriteCost(
GCNRegPressure &PressureBefore = DAG.Pressure[Region];
unsigned SpillCostBefore = PressureBefore.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
- LLVM_DEBUG(dbgs() << "RewriteMFMA: Region " << Region
- << " spill cost before: " << SpillCostBefore << "\n");
// For the cases we care about (i.e. ArchVGPR usage is greater than the
// addressable limit), rewriting alone should bring pressure to manageable
@@ -2283,21 +2276,29 @@ double RewriteMFMAFormStage::getRewriteCost(
GCNRegPressure PressureAfter = DAG.getRealRegPressure(Region);
unsigned SpillCostAfter = PressureAfter.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
- LLVM_DEBUG(dbgs() << "RewriteMFMA: Region " << Region
- << " spill cost after: " << SpillCostAfter << "\n");
- MachineBasicBlock *MBB = DAG.Regions[Region].first->getParent();
- double BlockFreq = MBFI->getBlockFreqRelativeToEntryBlock(MBB);
+ uint64_t BlockFreq =
+ MBFI->getBlockFreq(DAG.Regions[Region].first->getParent())
+ .getFrequency();
+
+ bool RelativeFreqIsDenom = EntryFreq > BlockFreq;
+ uint64_t RelativeFreq = EntryFreq && BlockFreq
+ ? (RelativeFreqIsDenom ? EntryFreq / BlockFreq
+ : BlockFreq / EntryFreq)
+ : 1;
// This assumes perfect spilling / splitting -- using one spill / copy
// instruction and one restoreFrom / copy for each excess register,
- double SpillCost = ((double)SpillCostAfter - (double)SpillCostBefore) * 2;
+ int64_t SpillCost = ((int)SpillCostAfter - (int)SpillCostBefore) * 2;
// Also account for the block frequency.
- SpillCost *= BlockFreq;
+ if (RelativeFreqIsDenom)
+ SpillCost /= (int64_t)RelativeFreq;
+ else
+ SpillCost *= (int64_t)RelativeFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0.0)
+ if (SpillCost > 0)
return SpillCost;
if (SpillCost < BestSpillCost)
@@ -2306,60 +2307,52 @@ double RewriteMFMAFormStage::getRewriteCost(
// Set the cost to the largest decrease in spill cost in order to not double
// count spill reductions.
- LLVM_DEBUG(dbgs() << "RewriteMFMA: BestSpillCost: " << BestSpillCost << "\n");
Cost = BestSpillCost;
- assert(Cost <= 0.0);
+ assert(Cost <= 0);
+
+ unsigned CopyCost = 0;
// For each CopyForDef, increase the cost by the register size while
// accounting for block frequency.
- double DefCopyCost = 0.0;
for (MachineInstr *DefMI : CopyForDef) {
Register DefReg = DefMI->getOperand(0).getReg();
- MachineBasicBlock *DefMBB = DefMI->getParent();
- double DefFreq = MBFI->getBlockFreqRelativeToEntryBlock(DefMBB);
+ uint64_t DefFreq =
+ EntryFreq
+ ? MBFI->getBlockFreq(DefMI->getParent()).getFrequency() / EntryFreq
+ : 1;
const TargetRegisterClass *RC = DAG.MRI.getRegClass(DefReg);
- DefCopyCost += (double)RC->getCopyCost() * DefFreq;
+ CopyCost += RC->getCopyCost() * DefFreq;
}
- LLVM_DEBUG(dbgs() << "RewriteMFMA: Def copy Costs: " << DefCopyCost << "\n");
// Account for CopyForUse copies in each block that the register is used.
- double UseCopyCost = 0.0;
for (auto &[UseBlock, UseRegs] : CopyForUse) {
- uint64_t UseFreq = MBFI->getBlockFreqRelativeToEntryBlock(UseBlock);
+ uint64_t UseFreq =
+ EntryFreq ? MBFI->getBlockFreq(UseBlock).getFrequency() / EntryFreq : 1;
for (Register UseReg : UseRegs) {
const TargetRegisterClass *RC = DAG.MRI.getRegClass(UseReg);
- UseCopyCost += (double)RC->getCopyCost() * UseFreq;
+ CopyCost += RC->getCopyCost() * UseFreq;
}
}
- LLVM_DEBUG(dbgs() << "RewriteMFMA: Use copy Costs: " << UseCopyCost << "\n");
- double CopyCost = UseCopyCost + DefCopyCost;
-
// Reset the classes that were changed to AGPR for better RB analysis.
// We must do rewriting after copy-insertion, as some defs of the register
// may require VGPR. Additionally, if we bail out and don't perform the
// rewrite then these need to be restored anyway.
for (auto &[MI, OriginalOpcode] : RewriteCands) {
assert(TII->isMAI(*MI));
- const TargetRegisterClass *ADefRC =
+ const TargetRegisterClass *AGPRRC =
DAG.MRI.getRegClass(MI->getOperand(0).getReg());
- const TargetRegisterClass *VDefRC = SRI->getEquivalentVGPRClass(ADefRC);
- DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VDefRC);
- MI->setDesc(TII->get(OriginalOpcode));
+ const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(AGPRRC);
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
- if (!Src2->isReg())
- continue;
- // Have to get src types separately since subregs may cause C and D
- // registers to be different types even though the actual operand is
- // the same size.
- const TargetRegisterClass *AUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *VUseRC = SRI->getEquivalentVGPRClass(AUseRC);
- DAG.MRI.setRegClass(Src2->getReg(), VUseRC);
+ if (Src2->isReg())
+ DAG.MRI.setRegClass(Src2->getReg(), VGPRRC);
+ DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VGPRRC);
+ MI->setDesc(TII->get(OriginalOpcode));
}
return Cost + CopyCost;
>From 118b8959760614a7b090197ea6021a48e7370449 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 19 Feb 2026 13:51:19 -0600
Subject: [PATCH 17/17] flag flip only
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 3dbace757add2..ddecc66134a61 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
+ cl::desc("Disable rewrite mfma rewrite scheduling stage"), cl::init(false));
const unsigned ScheduleMetrics::ScaleFactor = 100;
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 0f73cc4fda10f..6b6a40365b52d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -442,7 +442,7 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Calculate the rewrite cost and undo the state change (e.g. rewriting) done
/// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
/// costs, and \p RewriteCands to undo rewriting.
- double getRewriteCost(
+ int64_t getRewriteCost(
const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef);
More information about the llvm-commits
mailing list