[llvm] Reenable sched mfma rewrite (PR #180751)
Tony Linthicum via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 12 10:32:28 PST 2026
https://github.com/tlinthic updated https://github.com/llvm/llvm-project/pull/180751
>From 59c6d77cb907287e73ae509f5364f85eb40637a3 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Mon, 26 Jan 2026 09:35:17 -0600
Subject: [PATCH 01/13] Turn mfma rewrite in scheduler back on by default
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b0441955e70b9..3dbace757add2 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(true));
+ cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
const unsigned ScheduleMetrics::ScaleFactor = 100;
>From bb6bfacfd1679bc77b93d25a92bbee425a94440a Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Mon, 9 Feb 2026 08:55:48 -0600
Subject: [PATCH 02/13] Revert "Turn mfma rewrite in scheduler back on by
default"
This reverts commit fe5cc69dfdc58dc941ab3d2e9cd6bcf6a358cb90.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 3dbace757add2..b0441955e70b9 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
+ cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(true));
const unsigned ScheduleMetrics::ScaleFactor = 100;
>From 1a9469bd5911f2083d8bd50426f17487b366761f Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:12:31 -0600
Subject: [PATCH 03/13] Fix for issue #177696
Fixes problem with setting/resetting register types in the MFMA rewrite
sched stage.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 40 +-
.../AMDGPU/sched_mfma_rewrite_reject.ll | 1867 +++++++++++++++++
2 files changed, 1894 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b0441955e70b9..12526862927ed 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,7 +2184,6 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
-
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
@@ -2233,11 +2232,17 @@ bool RewriteMFMAFormStage::initHeuristics(
}
// Do the rewrite to allow for updated RP calculation.
- const TargetRegisterClass *VGPRRC = DAG.MRI.getRegClass(Dst.getReg());
- const TargetRegisterClass *AGPRRC = SRI->getEquivalentAGPRClass(VGPRRC);
- DAG.MRI.setRegClass(Dst.getReg(), AGPRRC);
- if (Src2->isReg())
- DAG.MRI.setRegClass(Src2->getReg(), AGPRRC);
+ const TargetRegisterClass *VDefRC = DAG.MRI.getRegClass(Dst.getReg());
+ const TargetRegisterClass *ADefRC = SRI->getEquivalentAGPRClass(VDefRC);
+ DAG.MRI.setRegClass(Dst.getReg(), ADefRC);
+ if (Src2->isReg()) {
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *AUseRC = SRI->getEquivalentVGPRClass(VUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
+ }
Changed = true;
}
}
@@ -2298,8 +2303,11 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
SpillCost *= (int64_t)RelativeFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0)
+ if (SpillCost > 0) {
+ LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite rejected.\n");
return SpillCost;
+ }
+ LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite successful.\n");
if (SpillCost < BestSpillCost)
BestSpillCost = SpillCost;
@@ -2342,17 +2350,23 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
// rewrite then these need to be restored anyway.
for (auto &[MI, OriginalOpcode] : RewriteCands) {
assert(TII->isMAI(*MI));
- const TargetRegisterClass *AGPRRC =
+ const TargetRegisterClass *ADefRC =
DAG.MRI.getRegClass(MI->getOperand(0).getReg());
- const TargetRegisterClass *VGPRRC = SRI->getEquivalentVGPRClass(AGPRRC);
+ const TargetRegisterClass *VDefRC = SRI->getEquivalentVGPRClass(ADefRC);
+ DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VDefRC);
+ MI->setDesc(TII->get(OriginalOpcode));
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
+ if (!Src2->isReg())
+ continue;
- if (Src2->isReg())
- DAG.MRI.setRegClass(Src2->getReg(), VGPRRC);
- DAG.MRI.setRegClass(MI->getOperand(0).getReg(), VGPRRC);
- MI->setDesc(TII->get(OriginalOpcode));
+ // Have to get src types separately since subregs may cause C and D
+ // registers to be different types even though the actual operand is
+ // the same size.
+ const TargetRegisterClass *AUseRC = DAG.MRI.getRegClass(Src2->getReg());
+ const TargetRegisterClass *VUseRC = SRI->getEquivalentVGPRClass(AUseRC);
+ DAG.MRI.setRegClass(Src2->getReg(), VUseRC);
}
return Cost + CopyCost;
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
new file mode 100644
index 0000000000000..c8f444c175be8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -0,0 +1,1867 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+
+ at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] undef, align 16
+ at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] undef, align 16
+
+; Function Attrs: alwaysinline nofree norecurse nounwind
+define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %0, ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %1, ptr addrspace(1) inreg noalias noundef nonnull align 16 %2) local_unnamed_addr #0 !reqd_work_group_size !2 {
+; This should be rejected for MFMA rewrite by the scheduler so we should see
+; no AGPR operands.
+; CHECK-NOT: v_mfma_f32_16x16x16_bf16 a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}]
+ %4 = tail call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
+ call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %0, i64 64) ]
+ %5 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %0, i16 0, i64 5564160, i32 159744)
+ call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %1, i64 64) ]
+ %6 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %1, i16 0, i64 5564160, i32 159744)
+ call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %2, i64 64) ]
+ %7 = lshr i32 %4, 6
+ %8 = and i32 %7, 1
+ %9 = and i32 %4, 15
+ %10 = lshr i32 %4, 2
+ %11 = and i32 %10, 12
+ %12 = lshr i32 %4, 3
+ %13 = or disjoint i32 %4, 256
+ %14 = lshr i32 %13, 3
+ %15 = or disjoint i32 %4, 512
+ %16 = lshr i32 %15, 3
+ %17 = or disjoint i32 %4, 768
+ %18 = lshr i32 %17, 3
+ %19 = lshr i32 %4, 4
+ %20 = lshr i32 %13, 4
+ %21 = lshr i32 %15, 4
+ %22 = lshr i32 %17, 4
+ %23 = or disjoint i32 %19, 64
+ %24 = or disjoint i32 %19, 80
+ %25 = or disjoint i32 %19, 96
+ %26 = or disjoint i32 %19, 112
+ %27 = shl nuw nsw i32 %4, 3
+ %28 = and i32 %27, 56
+ %29 = and i32 %27, 120
+ %30 = tail call range(i32 0, 512) i32 @llvm.amdgcn.workgroup.id.x()
+ %31 = shl nuw nsw i32 %30, 6
+ %32 = and i32 %31, 1984
+ %33 = shl nuw nsw i32 %30, 2
+ %34 = and i32 %33, 1920
+ %35 = sub nuw nsw i32 2016, %34
+ %36 = tail call i32 @llvm.umin.i32(i32 %35, i32 128)
+ %37 = sub nuw nsw i32 2016, %32
+ %38 = tail call i32 @llvm.umin.i32(i32 %37, i32 64)
+ %39 = tail call i32 @llvm.umin.i32(i32 %38, i32 %28)
+ %40 = sub nsw i32 %38, %39
+ %41 = icmp sgt i32 %40, 7
+ %42 = tail call i32 @llvm.umin.i32(i32 %36, i32 %29)
+ %43 = sub nsw i32 %36, %42
+ %44 = icmp sgt i32 %43, 7
+ %45 = or disjoint i32 %39, %32
+ %.idx = mul nuw nsw i32 %12, 4032
+ %46 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx
+ %47 = getelementptr bfloat, ptr addrspace(7) %46, i32 %45
+ %48 = load <8 x bfloat>, ptr addrspace(7) %47, align 2
+ %.idx4 = mul nuw nsw i32 %14, 4032
+ %49 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx4
+ %50 = getelementptr bfloat, ptr addrspace(7) %49, i32 %45
+ %51 = load <8 x bfloat>, ptr addrspace(7) %50, align 2
+ %.idx6 = mul nuw nsw i32 %16, 4032
+ %52 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx6
+ %53 = getelementptr bfloat, ptr addrspace(7) %52, i32 %45
+ %54 = load <8 x bfloat>, ptr addrspace(7) %53, align 2
+ %.idx8 = mul nuw nsw i32 %18, 4032
+ %55 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx8
+ %56 = getelementptr bfloat, ptr addrspace(7) %55, i32 %45
+ %57 = load <8 x bfloat>, ptr addrspace(7) %56, align 2
+ %58 = or disjoint i32 %42, %34
+ %.idx10 = mul nuw nsw i32 %19, 4032
+ %59 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx10
+ %60 = getelementptr bfloat, ptr addrspace(7) %59, i32 %58
+ %61 = load <8 x bfloat>, ptr addrspace(7) %60, align 2
+ %.idx12 = mul nuw nsw i32 %20, 4032
+ %62 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx12
+ %63 = getelementptr bfloat, ptr addrspace(7) %62, i32 %58
+ %64 = load <8 x bfloat>, ptr addrspace(7) %63, align 2
+ %.idx14 = mul nuw nsw i32 %21, 4032
+ %65 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx14
+ %66 = getelementptr bfloat, ptr addrspace(7) %65, i32 %58
+ %67 = load <8 x bfloat>, ptr addrspace(7) %66, align 2
+ %.idx16 = mul nuw nsw i32 %22, 4032
+ %68 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx16
+ %69 = getelementptr bfloat, ptr addrspace(7) %68, i32 %58
+ %70 = load <8 x bfloat>, ptr addrspace(7) %69, align 2
+ %.idx18 = mul nuw nsw i32 %23, 4032
+ %71 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx18
+ %72 = getelementptr bfloat, ptr addrspace(7) %71, i32 %58
+ %73 = load <8 x bfloat>, ptr addrspace(7) %72, align 2
+ %.idx20 = mul nuw nsw i32 %24, 4032
+ %74 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx20
+ %75 = getelementptr bfloat, ptr addrspace(7) %74, i32 %58
+ %76 = load <8 x bfloat>, ptr addrspace(7) %75, align 2
+ %.idx22 = mul nuw nsw i32 %25, 4032
+ %77 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx22
+ %78 = getelementptr bfloat, ptr addrspace(7) %77, i32 %58
+ %79 = load <8 x bfloat>, ptr addrspace(7) %78, align 2
+ %.idx24 = mul nuw nsw i32 %26, 4032
+ %80 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx24
+ %81 = getelementptr bfloat, ptr addrspace(7) %80, i32 %58
+ %82 = load <8 x bfloat>, ptr addrspace(7) %81, align 2
+ %83 = select i1 %41, <8 x bfloat> %48, <8 x bfloat> zeroinitializer
+ %.idx26 = mul nuw nsw i32 %12, 136
+ %84 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx26
+ %85 = getelementptr bfloat, ptr addrspace(3) %84, i32 %28
+ store <8 x bfloat> %83, ptr addrspace(3) %85, align 8
+ %86 = select i1 %41, <8 x bfloat> %51, <8 x bfloat> zeroinitializer
+ %.idx28 = mul nuw nsw i32 %14, 136
+ %87 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx28
+ %88 = getelementptr bfloat, ptr addrspace(3) %87, i32 %28
+ store <8 x bfloat> %86, ptr addrspace(3) %88, align 8
+ %89 = select i1 %41, <8 x bfloat> %54, <8 x bfloat> zeroinitializer
+ %.idx30 = mul nuw nsw i32 %16, 136
+ %90 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx30
+ %91 = getelementptr bfloat, ptr addrspace(3) %90, i32 %28
+ store <8 x bfloat> %89, ptr addrspace(3) %91, align 8
+ %92 = select i1 %41, <8 x bfloat> %57, <8 x bfloat> zeroinitializer
+ %.idx32 = mul nuw nsw i32 %18, 136
+ %93 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx32
+ %94 = getelementptr bfloat, ptr addrspace(3) %93, i32 %28
+ store <8 x bfloat> %92, ptr addrspace(3) %94, align 8
+ %95 = select i1 %44, <8 x bfloat> %61, <8 x bfloat> zeroinitializer
+ %.idx34 = mul nuw nsw i32 %19, 264
+ %96 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx34
+ %97 = getelementptr bfloat, ptr addrspace(3) %96, i32 %29
+ store <8 x bfloat> %95, ptr addrspace(3) %97, align 8
+ %98 = select i1 %44, <8 x bfloat> %64, <8 x bfloat> zeroinitializer
+ %.idx36 = mul nuw nsw i32 %20, 264
+ %99 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx36
+ %100 = getelementptr bfloat, ptr addrspace(3) %99, i32 %29
+ store <8 x bfloat> %98, ptr addrspace(3) %100, align 8
+ %101 = select i1 %44, <8 x bfloat> %67, <8 x bfloat> zeroinitializer
+ %.idx38 = mul nuw nsw i32 %21, 264
+ %102 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx38
+ %103 = getelementptr bfloat, ptr addrspace(3) %102, i32 %29
+ store <8 x bfloat> %101, ptr addrspace(3) %103, align 8
+ %104 = select i1 %44, <8 x bfloat> %70, <8 x bfloat> zeroinitializer
+ %.idx40 = mul nuw nsw i32 %22, 264
+ %105 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx40
+ %106 = getelementptr bfloat, ptr addrspace(3) %105, i32 %29
+ store <8 x bfloat> %104, ptr addrspace(3) %106, align 8
+ %107 = select i1 %44, <8 x bfloat> %73, <8 x bfloat> zeroinitializer
+ %.idx42 = mul nuw nsw i32 %23, 264
+ %108 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx42
+ %109 = getelementptr bfloat, ptr addrspace(3) %108, i32 %29
+ store <8 x bfloat> %107, ptr addrspace(3) %109, align 8
+ %110 = select i1 %44, <8 x bfloat> %76, <8 x bfloat> zeroinitializer
+ %.idx44 = mul nuw nsw i32 %24, 264
+ %111 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx44
+ %112 = getelementptr bfloat, ptr addrspace(3) %111, i32 %29
+ store <8 x bfloat> %110, ptr addrspace(3) %112, align 8
+ %113 = select i1 %44, <8 x bfloat> %79, <8 x bfloat> zeroinitializer
+ %.idx46 = mul nuw nsw i32 %25, 264
+ %114 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx46
+ %115 = getelementptr bfloat, ptr addrspace(3) %114, i32 %29
+ store <8 x bfloat> %113, ptr addrspace(3) %115, align 8
+ %116 = select i1 %44, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
+ %.idx48 = mul nuw nsw i32 %26, 264
+ %117 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx48
+ %118 = getelementptr bfloat, ptr addrspace(3) %117, i32 %29
+ store <8 x bfloat> %116, ptr addrspace(3) %118, align 8
+ %119 = shl nuw nsw i32 %8, 5
+ %120 = or disjoint i32 %119, %9
+ %121 = or disjoint i32 %11, 1
+ %122 = or disjoint i32 %11, 2
+ %123 = or disjoint i32 %11, 3
+ %124 = or disjoint i32 %11, 16
+ %125 = or disjoint i32 %11, 17
+ %126 = or disjoint i32 %11, 18
+ %127 = or disjoint i32 %11, 19
+ %128 = or disjoint i32 %11, 32
+ %129 = or disjoint i32 %11, 33
+ %130 = or disjoint i32 %11, 34
+ %131 = or disjoint i32 %11, 35
+ %132 = or disjoint i32 %11, 48
+ %133 = or disjoint i32 %11, 49
+ %134 = or disjoint i32 %11, 50
+ %135 = or i32 %10, 51
+ %136 = or disjoint i32 %11, 64
+ %137 = or disjoint i32 %11, 65
+ %138 = or disjoint i32 %11, 66
+ %139 = or disjoint i32 %11, 67
+ %140 = or disjoint i32 %11, 80
+ %141 = or disjoint i32 %11, 81
+ %142 = or disjoint i32 %11, 82
+ %143 = or disjoint i32 %11, 83
+ %144 = or disjoint i32 %11, 96
+ %145 = or disjoint i32 %11, 97
+ %146 = or disjoint i32 %11, 98
+ %147 = or disjoint i32 %11, 99
+ %148 = or disjoint i32 %11, 112
+ %149 = or disjoint i32 %11, 113
+ %150 = or disjoint i32 %11, 114
+ %151 = or i32 %10, 115
+ %152 = lshr i32 %4, 1
+ %153 = and i32 %152, 64
+ %154 = or disjoint i32 %153, %9
+ %invariant.gep = getelementptr bfloat, ptr addrspace(7) %5, i32 %45
+ %invariant.gep111 = getelementptr bfloat, ptr addrspace(7) %6, i32 %58
+ %155 = mul nuw nsw i32 %11, 68
+ %156 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
+ %157 = getelementptr bfloat, ptr addrspace(3) %156, i32 %155
+ %158 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
+ %159 = getelementptr i8, ptr addrspace(3) %158, i32 32
+ %160 = getelementptr bfloat, ptr addrspace(3) %159, i32 %155
+ %161 = mul nuw nsw i32 %121, 68
+ %162 = getelementptr bfloat, ptr addrspace(3) %156, i32 %161
+ %163 = getelementptr bfloat, ptr addrspace(3) %159, i32 %161
+ %164 = mul nuw nsw i32 %122, 68
+ %165 = getelementptr bfloat, ptr addrspace(3) %156, i32 %164
+ %166 = getelementptr bfloat, ptr addrspace(3) %159, i32 %164
+ %167 = mul nuw nsw i32 %123, 68
+ %168 = getelementptr bfloat, ptr addrspace(3) %156, i32 %167
+ %169 = getelementptr bfloat, ptr addrspace(3) %159, i32 %167
+ %170 = mul nuw nsw i32 %124, 68
+ %171 = getelementptr bfloat, ptr addrspace(3) %156, i32 %170
+ %172 = getelementptr bfloat, ptr addrspace(3) %159, i32 %170
+ %173 = mul nuw nsw i32 %125, 68
+ %174 = getelementptr bfloat, ptr addrspace(3) %156, i32 %173
+ %175 = getelementptr bfloat, ptr addrspace(3) %159, i32 %173
+ %176 = mul nuw nsw i32 %126, 68
+ %177 = getelementptr bfloat, ptr addrspace(3) %156, i32 %176
+ %178 = getelementptr bfloat, ptr addrspace(3) %159, i32 %176
+ %179 = mul nuw nsw i32 %127, 68
+ %180 = getelementptr bfloat, ptr addrspace(3) %156, i32 %179
+ %181 = getelementptr bfloat, ptr addrspace(3) %159, i32 %179
+ %182 = mul nuw nsw i32 %128, 68
+ %183 = getelementptr bfloat, ptr addrspace(3) %156, i32 %182
+ %184 = getelementptr bfloat, ptr addrspace(3) %159, i32 %182
+ %185 = mul nuw nsw i32 %129, 68
+ %186 = getelementptr bfloat, ptr addrspace(3) %156, i32 %185
+ %187 = getelementptr bfloat, ptr addrspace(3) %159, i32 %185
+ %188 = mul nuw nsw i32 %130, 68
+ %189 = getelementptr bfloat, ptr addrspace(3) %156, i32 %188
+ %190 = getelementptr bfloat, ptr addrspace(3) %159, i32 %188
+ %191 = mul nuw nsw i32 %131, 68
+ %192 = getelementptr bfloat, ptr addrspace(3) %156, i32 %191
+ %193 = getelementptr bfloat, ptr addrspace(3) %159, i32 %191
+ %194 = mul nuw nsw i32 %132, 68
+ %195 = getelementptr bfloat, ptr addrspace(3) %156, i32 %194
+ %196 = getelementptr bfloat, ptr addrspace(3) %159, i32 %194
+ %197 = mul nuw nsw i32 %133, 68
+ %198 = getelementptr bfloat, ptr addrspace(3) %156, i32 %197
+ %199 = getelementptr bfloat, ptr addrspace(3) %159, i32 %197
+ %200 = mul nuw nsw i32 %134, 68
+ %201 = getelementptr bfloat, ptr addrspace(3) %156, i32 %200
+ %202 = getelementptr bfloat, ptr addrspace(3) %159, i32 %200
+ %203 = mul nuw nsw i32 %135, 68
+ %204 = getelementptr bfloat, ptr addrspace(3) %156, i32 %203
+ %205 = getelementptr bfloat, ptr addrspace(3) %159, i32 %203
+ %206 = mul nuw nsw i32 %136, 68
+ %207 = getelementptr bfloat, ptr addrspace(3) %156, i32 %206
+ %208 = getelementptr bfloat, ptr addrspace(3) %159, i32 %206
+ %209 = mul nuw nsw i32 %137, 68
+ %210 = getelementptr bfloat, ptr addrspace(3) %156, i32 %209
+ %211 = getelementptr bfloat, ptr addrspace(3) %159, i32 %209
+ %212 = mul nuw nsw i32 %138, 68
+ %213 = getelementptr bfloat, ptr addrspace(3) %156, i32 %212
+ %214 = getelementptr bfloat, ptr addrspace(3) %159, i32 %212
+ %215 = mul nuw nsw i32 %139, 68
+ %216 = getelementptr bfloat, ptr addrspace(3) %156, i32 %215
+ %217 = getelementptr bfloat, ptr addrspace(3) %159, i32 %215
+ %218 = mul nuw nsw i32 %140, 68
+ %219 = getelementptr bfloat, ptr addrspace(3) %156, i32 %218
+ %220 = getelementptr bfloat, ptr addrspace(3) %159, i32 %218
+ %221 = mul nuw nsw i32 %141, 68
+ %222 = getelementptr bfloat, ptr addrspace(3) %156, i32 %221
+ %223 = getelementptr bfloat, ptr addrspace(3) %159, i32 %221
+ %224 = mul nuw nsw i32 %142, 68
+ %225 = getelementptr bfloat, ptr addrspace(3) %156, i32 %224
+ %226 = getelementptr bfloat, ptr addrspace(3) %159, i32 %224
+ %227 = mul nuw nsw i32 %143, 68
+ %228 = getelementptr bfloat, ptr addrspace(3) %156, i32 %227
+ %229 = getelementptr bfloat, ptr addrspace(3) %159, i32 %227
+ %230 = mul nuw nsw i32 %144, 68
+ %231 = getelementptr bfloat, ptr addrspace(3) %156, i32 %230
+ %232 = getelementptr bfloat, ptr addrspace(3) %159, i32 %230
+ %233 = mul nuw nsw i32 %145, 68
+ %234 = getelementptr bfloat, ptr addrspace(3) %156, i32 %233
+ %235 = getelementptr bfloat, ptr addrspace(3) %159, i32 %233
+ %236 = mul nuw nsw i32 %146, 68
+ %237 = getelementptr bfloat, ptr addrspace(3) %156, i32 %236
+ %238 = getelementptr bfloat, ptr addrspace(3) %159, i32 %236
+ %239 = mul nuw nsw i32 %147, 68
+ %240 = getelementptr bfloat, ptr addrspace(3) %156, i32 %239
+ %241 = getelementptr bfloat, ptr addrspace(3) %159, i32 %239
+ %242 = mul nuw nsw i32 %148, 68
+ %243 = getelementptr bfloat, ptr addrspace(3) %156, i32 %242
+ %244 = getelementptr bfloat, ptr addrspace(3) %159, i32 %242
+ %245 = mul nuw nsw i32 %149, 68
+ %246 = getelementptr bfloat, ptr addrspace(3) %156, i32 %245
+ %247 = getelementptr bfloat, ptr addrspace(3) %159, i32 %245
+ %248 = mul nuw nsw i32 %150, 68
+ %249 = getelementptr bfloat, ptr addrspace(3) %156, i32 %248
+ %250 = getelementptr bfloat, ptr addrspace(3) %159, i32 %248
+ %251 = mul nuw nsw i32 %151, 68
+ %252 = getelementptr bfloat, ptr addrspace(3) %156, i32 %251
+ %253 = getelementptr bfloat, ptr addrspace(3) %159, i32 %251
+ %narrow = mul nuw nsw i32 %11, 132
+ %254 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %255 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow
+ %256 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %257 = getelementptr i8, ptr addrspace(3) %256, i32 32
+ %258 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow
+ %259 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %260 = getelementptr i8, ptr addrspace(3) %259, i32 64
+ %261 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow
+ %262 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
+ %263 = getelementptr i8, ptr addrspace(3) %262, i32 96
+ %264 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow
+ %narrow74 = mul nuw nsw i32 %121, 132
+ %265 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow74
+ %266 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow74
+ %267 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow74
+ %268 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow74
+ %narrow75 = mul nuw nsw i32 %122, 132
+ %269 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow75
+ %270 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow75
+ %271 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow75
+ %272 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow75
+ %narrow76 = mul nuw nsw i32 %123, 132
+ %273 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow76
+ %274 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow76
+ %275 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow76
+ %276 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow76
+ %narrow77 = mul nuw nsw i32 %124, 132
+ %277 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow77
+ %278 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow77
+ %279 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow77
+ %280 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow77
+ %narrow78 = mul nuw nsw i32 %125, 132
+ %281 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow78
+ %282 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow78
+ %283 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow78
+ %284 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow78
+ %narrow79 = mul nuw nsw i32 %126, 132
+ %285 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow79
+ %286 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow79
+ %287 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow79
+ %288 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow79
+ %narrow80 = mul nuw nsw i32 %127, 132
+ %289 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow80
+ %290 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow80
+ %291 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow80
+ %292 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow80
+ %narrow81 = mul nuw nsw i32 %128, 132
+ %293 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow81
+ %294 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow81
+ %295 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow81
+ %296 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow81
+ %narrow82 = mul nuw nsw i32 %129, 132
+ %297 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow82
+ %298 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow82
+ %299 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow82
+ %300 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow82
+ %narrow83 = mul nuw nsw i32 %130, 132
+ %301 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow83
+ %302 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow83
+ %303 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow83
+ %304 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow83
+ %narrow84 = mul nuw nsw i32 %131, 132
+ %305 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow84
+ %306 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow84
+ %307 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow84
+ %308 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow84
+ %narrow85 = mul nuw nsw i32 %132, 132
+ %309 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow85
+ %310 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow85
+ %311 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow85
+ %312 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow85
+ %narrow86 = mul nuw nsw i32 %133, 132
+ %313 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow86
+ %314 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow86
+ %315 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow86
+ %316 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow86
+ %narrow87 = mul nuw nsw i32 %134, 132
+ %317 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow87
+ %318 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow87
+ %319 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow87
+ %320 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow87
+ %narrow88 = mul nuw nsw i32 %135, 132
+ %321 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow88
+ %322 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow88
+ %323 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow88
+ %324 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow88
+ %narrow89 = mul nuw nsw i32 %136, 132
+ %325 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow89
+ %326 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow89
+ %327 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow89
+ %328 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow89
+ %narrow90 = mul nuw nsw i32 %137, 132
+ %329 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow90
+ %330 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow90
+ %331 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow90
+ %332 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow90
+ %narrow91 = mul nuw nsw i32 %138, 132
+ %333 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow91
+ %334 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow91
+ %335 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow91
+ %336 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow91
+ %narrow92 = mul nuw nsw i32 %139, 132
+ %337 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow92
+ %338 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow92
+ %339 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow92
+ %340 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow92
+ %narrow93 = mul nuw nsw i32 %140, 132
+ %341 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow93
+ %342 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow93
+ %343 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow93
+ %344 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow93
+ %narrow94 = mul nuw nsw i32 %141, 132
+ %345 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow94
+ %346 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow94
+ %347 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow94
+ %348 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow94
+ %narrow95 = mul nuw nsw i32 %142, 132
+ %349 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow95
+ %350 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow95
+ %351 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow95
+ %352 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow95
+ %narrow96 = mul nuw nsw i32 %143, 132
+ %353 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow96
+ %354 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow96
+ %355 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow96
+ %356 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow96
+ %narrow97 = mul nuw nsw i32 %144, 132
+ %357 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow97
+ %358 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow97
+ %359 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow97
+ %360 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow97
+ %narrow98 = mul nuw nsw i32 %145, 132
+ %361 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow98
+ %362 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow98
+ %363 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow98
+ %364 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow98
+ %narrow99 = mul nuw nsw i32 %146, 132
+ %365 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow99
+ %366 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow99
+ %367 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow99
+ %368 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow99
+ %narrow100 = mul nuw nsw i32 %147, 132
+ %369 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow100
+ %370 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow100
+ %371 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow100
+ %372 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow100
+ %narrow101 = mul nuw nsw i32 %148, 132
+ %373 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow101
+ %374 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow101
+ %375 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow101
+ %376 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow101
+ %narrow102 = mul nuw nsw i32 %149, 132
+ %377 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow102
+ %378 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow102
+ %379 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow102
+ %380 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow102
+ %narrow103 = mul nuw nsw i32 %150, 132
+ %381 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow103
+ %382 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow103
+ %383 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow103
+ %384 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow103
+ %narrow104 = mul nuw nsw i32 %151, 132
+ %385 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow104
+ %386 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow104
+ %387 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow104
+ %388 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow104
+ br label %389
+
+389: ; preds = %3, %389
+ %390 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %3 ], [ %1015, %389 ]
+ %391 = phi i32 [ 0, %3 ], [ %392, %389 ]
+ %392 = add nuw nsw i32 %391, 8
+ %393 = shl nuw nsw i32 %392, 4
+ %394 = sub nuw nsw i32 1380, %393
+ %395 = tail call i32 @llvm.umin.i32(i32 %394, i32 128)
+ %396 = tail call i32 @llvm.umin.i32(i32 %395, i32 %12)
+ %397 = sub nsw i32 %395, %396
+ %398 = tail call i32 @llvm.smin.i32(i32 %397, i32 1)
+ %399 = or disjoint i32 %396, %393
+ %.idx50 = mul nuw nsw i32 %399, 4032
+ %gep = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx50
+ %400 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
+ %401 = tail call i32 @llvm.umin.i32(i32 %395, i32 %14)
+ %402 = sub nsw i32 %395, %401
+ %403 = tail call i32 @llvm.smin.i32(i32 %402, i32 1)
+ %404 = or disjoint i32 %401, %393
+ %.idx52 = mul nuw nsw i32 %404, 4032
+ %gep106 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx52
+ %405 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
+ %406 = tail call i32 @llvm.umin.i32(i32 %395, i32 %16)
+ %407 = sub nsw i32 %395, %406
+ %408 = tail call i32 @llvm.smin.i32(i32 %407, i32 1)
+ %409 = or disjoint i32 %406, %393
+ %.idx54 = mul nuw nsw i32 %409, 4032
+ %gep108 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx54
+ %410 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
+ %411 = tail call i32 @llvm.umin.i32(i32 %395, i32 %18)
+ %412 = sub nsw i32 %395, %411
+ %413 = tail call i32 @llvm.smin.i32(i32 %412, i32 1)
+ %414 = or disjoint i32 %411, %393
+ %.idx56 = mul nuw nsw i32 %414, 4032
+ %gep110 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx56
+ %415 = load <8 x bfloat>, ptr addrspace(7) %gep110, align 2
+ %416 = tail call i32 @llvm.umin.i32(i32 %395, i32 %19)
+ %417 = sub nsw i32 %395, %416
+ %418 = tail call i32 @llvm.smin.i32(i32 %417, i32 1)
+ %419 = or disjoint i32 %416, %393
+ %.idx58 = mul nuw nsw i32 %419, 4032
+ %gep112 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx58
+ %420 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
+ %421 = tail call i32 @llvm.umin.i32(i32 %395, i32 %20)
+ %422 = sub nsw i32 %395, %421
+ %423 = tail call i32 @llvm.smin.i32(i32 %422, i32 1)
+ %424 = or disjoint i32 %421, %393
+ %.idx60 = mul nuw nsw i32 %424, 4032
+ %gep114 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx60
+ %425 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
+ %426 = tail call i32 @llvm.umin.i32(i32 %395, i32 %21)
+ %427 = sub nsw i32 %395, %426
+ %428 = tail call i32 @llvm.smin.i32(i32 %427, i32 1)
+ %429 = or disjoint i32 %426, %393
+ %.idx62 = mul nuw nsw i32 %429, 4032
+ %gep116 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx62
+ %430 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
+ %431 = tail call i32 @llvm.umin.i32(i32 %395, i32 %22)
+ %432 = sub nsw i32 %395, %431
+ %433 = tail call i32 @llvm.smin.i32(i32 %432, i32 1)
+ %434 = or disjoint i32 %431, %393
+ %.idx64 = mul nuw nsw i32 %434, 4032
+ %gep118 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx64
+ %435 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
+ %436 = tail call i32 @llvm.umin.i32(i32 %395, i32 %23)
+ %437 = sub nsw i32 %395, %436
+ %438 = tail call i32 @llvm.smin.i32(i32 %437, i32 1)
+ %439 = or disjoint i32 %436, %393
+ %.idx66 = mul nuw nsw i32 %439, 4032
+ %gep120 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx66
+ %440 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
+ %441 = tail call i32 @llvm.umin.i32(i32 %395, i32 %24)
+ %442 = sub nsw i32 %395, %441
+ %443 = tail call i32 @llvm.smin.i32(i32 %442, i32 1)
+ %444 = or disjoint i32 %441, %393
+ %.idx68 = mul nuw nsw i32 %444, 4032
+ %gep122 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx68
+ %445 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
+ %446 = tail call i32 @llvm.umin.i32(i32 %395, i32 %25)
+ %447 = sub nsw i32 %395, %446
+ %448 = tail call i32 @llvm.smin.i32(i32 %447, i32 1)
+ %449 = or disjoint i32 %446, %393
+ %.idx70 = mul nuw nsw i32 %449, 4032
+ %gep124 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx70
+ %450 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
+ %451 = tail call i32 @llvm.umin.i32(i32 %395, i32 %26)
+ %452 = sub nsw i32 %395, %451
+ %453 = tail call i32 @llvm.smin.i32(i32 %452, i32 1)
+ %454 = or disjoint i32 %451, %393
+ %.idx72 = mul nuw nsw i32 %454, 4032
+ %gep126 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx72
+ %455 = load <8 x bfloat>, ptr addrspace(7) %gep126, align 2
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ %456 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
+ %457 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
+ %458 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
+ %459 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
+ %460 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
+ %461 = shufflevector <1 x bfloat> %460, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %462 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
+ %463 = shufflevector <1 x bfloat> %462, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %464 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
+ %465 = shufflevector <1 x bfloat> %464, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %466 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
+ %467 = shufflevector <1 x bfloat> %466, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %468 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
+ %469 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
+ %470 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
+ %471 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
+ %472 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
+ %473 = shufflevector <1 x bfloat> %472, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %474 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
+ %475 = shufflevector <1 x bfloat> %474, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %476 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
+ %477 = shufflevector <1 x bfloat> %476, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %478 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
+ %479 = shufflevector <1 x bfloat> %478, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %480 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
+ %481 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
+ %482 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
+ %483 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
+ %484 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
+ %485 = shufflevector <1 x bfloat> %484, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %486 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
+ %487 = shufflevector <1 x bfloat> %486, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %488 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
+ %489 = shufflevector <1 x bfloat> %488, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %490 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
+ %491 = shufflevector <1 x bfloat> %490, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %492 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
+ %493 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
+ %494 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
+ %495 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
+ %496 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
+ %497 = shufflevector <1 x bfloat> %496, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %498 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
+ %499 = shufflevector <1 x bfloat> %498, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %500 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
+ %501 = shufflevector <1 x bfloat> %500, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %502 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
+ %503 = shufflevector <1 x bfloat> %502, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %504 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
+ %505 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
+ %506 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
+ %507 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
+ %508 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
+ %509 = shufflevector <1 x bfloat> %508, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %510 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
+ %511 = shufflevector <1 x bfloat> %510, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %512 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
+ %513 = shufflevector <1 x bfloat> %512, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %514 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
+ %515 = shufflevector <1 x bfloat> %514, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %516 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
+ %517 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
+ %518 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
+ %519 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
+ %520 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
+ %521 = shufflevector <1 x bfloat> %520, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %522 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
+ %523 = shufflevector <1 x bfloat> %522, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %524 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
+ %525 = shufflevector <1 x bfloat> %524, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %526 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
+ %527 = shufflevector <1 x bfloat> %526, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %528 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
+ %529 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
+ %530 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
+ %531 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
+ %532 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
+ %533 = shufflevector <1 x bfloat> %532, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %534 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
+ %535 = shufflevector <1 x bfloat> %534, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %536 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
+ %537 = shufflevector <1 x bfloat> %536, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %538 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
+ %539 = shufflevector <1 x bfloat> %538, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %540 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
+ %541 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
+ %542 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
+ %543 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
+ %544 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
+ %545 = shufflevector <1 x bfloat> %544, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %546 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
+ %547 = shufflevector <1 x bfloat> %546, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %548 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
+ %549 = shufflevector <1 x bfloat> %548, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %550 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
+ %551 = shufflevector <1 x bfloat> %550, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %552 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
+ %553 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
+ %554 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
+ %555 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
+ %556 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
+ %557 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
+ %558 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
+ %559 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
+ %560 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
+ %561 = shufflevector <1 x bfloat> %560, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %562 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
+ %563 = shufflevector <1 x bfloat> %562, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %564 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
+ %565 = shufflevector <1 x bfloat> %564, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %566 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
+ %567 = shufflevector <1 x bfloat> %566, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %568 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
+ %569 = shufflevector <1 x bfloat> %568, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %570 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
+ %571 = shufflevector <1 x bfloat> %570, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %572 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
+ %573 = shufflevector <1 x bfloat> %572, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %574 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
+ %575 = shufflevector <1 x bfloat> %574, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %576 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
+ %577 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
+ %578 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
+ %579 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
+ %580 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
+ %581 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
+ %582 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
+ %583 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
+ %584 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
+ %585 = shufflevector <1 x bfloat> %584, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %586 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
+ %587 = shufflevector <1 x bfloat> %586, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %588 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
+ %589 = shufflevector <1 x bfloat> %588, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %590 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
+ %591 = shufflevector <1 x bfloat> %590, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %592 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
+ %593 = shufflevector <1 x bfloat> %592, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %594 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
+ %595 = shufflevector <1 x bfloat> %594, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %596 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
+ %597 = shufflevector <1 x bfloat> %596, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %598 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
+ %599 = shufflevector <1 x bfloat> %598, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %600 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
+ %601 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
+ %602 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
+ %603 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
+ %604 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
+ %605 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
+ %606 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
+ %607 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
+ %608 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
+ %609 = shufflevector <1 x bfloat> %608, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %610 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
+ %611 = shufflevector <1 x bfloat> %610, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %612 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
+ %613 = shufflevector <1 x bfloat> %612, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %614 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
+ %615 = shufflevector <1 x bfloat> %614, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %616 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
+ %617 = shufflevector <1 x bfloat> %616, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %618 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
+ %619 = shufflevector <1 x bfloat> %618, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %620 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
+ %621 = shufflevector <1 x bfloat> %620, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %622 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
+ %623 = shufflevector <1 x bfloat> %622, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %624 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
+ %625 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
+ %626 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
+ %627 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
+ %628 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
+ %629 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
+ %630 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
+ %631 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
+ %632 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
+ %633 = shufflevector <1 x bfloat> %632, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %634 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
+ %635 = shufflevector <1 x bfloat> %634, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %636 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
+ %637 = shufflevector <1 x bfloat> %636, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %638 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
+ %639 = shufflevector <1 x bfloat> %638, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %640 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
+ %641 = shufflevector <1 x bfloat> %640, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %642 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
+ %643 = shufflevector <1 x bfloat> %642, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %644 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
+ %645 = shufflevector <1 x bfloat> %644, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %646 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
+ %647 = shufflevector <1 x bfloat> %646, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %648 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
+ %649 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
+ %650 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
+ %651 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
+ %652 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
+ %653 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
+ %654 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
+ %655 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
+ %656 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
+ %657 = shufflevector <1 x bfloat> %656, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %658 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
+ %659 = shufflevector <1 x bfloat> %658, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %660 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
+ %661 = shufflevector <1 x bfloat> %660, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %662 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
+ %663 = shufflevector <1 x bfloat> %662, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %664 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
+ %665 = shufflevector <1 x bfloat> %664, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %666 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
+ %667 = shufflevector <1 x bfloat> %666, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %668 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
+ %669 = shufflevector <1 x bfloat> %668, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %670 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
+ %671 = shufflevector <1 x bfloat> %670, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %672 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
+ %673 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
+ %674 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
+ %675 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
+ %676 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
+ %677 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
+ %678 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
+ %679 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
+ %680 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
+ %681 = shufflevector <1 x bfloat> %680, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %682 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
+ %683 = shufflevector <1 x bfloat> %682, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %684 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
+ %685 = shufflevector <1 x bfloat> %684, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %686 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
+ %687 = shufflevector <1 x bfloat> %686, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %688 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
+ %689 = shufflevector <1 x bfloat> %688, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %690 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
+ %691 = shufflevector <1 x bfloat> %690, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %692 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
+ %693 = shufflevector <1 x bfloat> %692, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %694 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
+ %695 = shufflevector <1 x bfloat> %694, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %696 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
+ %697 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
+ %698 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
+ %699 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
+ %700 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
+ %701 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
+ %702 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
+ %703 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
+ %704 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
+ %705 = shufflevector <1 x bfloat> %704, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %706 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
+ %707 = shufflevector <1 x bfloat> %706, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %708 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
+ %709 = shufflevector <1 x bfloat> %708, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %710 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
+ %711 = shufflevector <1 x bfloat> %710, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %712 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
+ %713 = shufflevector <1 x bfloat> %712, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %714 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
+ %715 = shufflevector <1 x bfloat> %714, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %716 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
+ %717 = shufflevector <1 x bfloat> %716, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %718 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
+ %719 = shufflevector <1 x bfloat> %718, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %720 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
+ %721 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
+ %722 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
+ %723 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
+ %724 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
+ %725 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
+ %726 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
+ %727 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
+ %728 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
+ %729 = shufflevector <1 x bfloat> %728, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %730 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
+ %731 = shufflevector <1 x bfloat> %730, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %732 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
+ %733 = shufflevector <1 x bfloat> %732, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %734 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
+ %735 = shufflevector <1 x bfloat> %734, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %736 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
+ %737 = shufflevector <1 x bfloat> %736, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %738 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
+ %739 = shufflevector <1 x bfloat> %738, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %740 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
+ %741 = shufflevector <1 x bfloat> %740, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %742 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
+ %743 = shufflevector <1 x bfloat> %742, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %744 = shufflevector <1 x bfloat> %456, <1 x bfloat> %458, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %745 = shufflevector <4 x bfloat> %744, <4 x bfloat> %461, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %746 = shufflevector <4 x bfloat> %745, <4 x bfloat> %465, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %747 = shufflevector <1 x bfloat> %552, <1 x bfloat> %556, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %748 = shufflevector <4 x bfloat> %747, <4 x bfloat> %561, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %749 = shufflevector <4 x bfloat> %748, <4 x bfloat> %569, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %750 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 0, 0
+ %751 = bitcast <4 x bfloat> %746 to <4 x i16>
+ %752 = bitcast <4 x bfloat> %749 to <4 x i16>
+ %753 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %752, <4 x float> %750, i32 0, i32 0, i32 0)
+ %754 = shufflevector <1 x bfloat> %553, <1 x bfloat> %557, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %755 = shufflevector <4 x bfloat> %754, <4 x bfloat> %563, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %756 = shufflevector <4 x bfloat> %755, <4 x bfloat> %571, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %757 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 0, 0
+ %758 = bitcast <4 x bfloat> %756 to <4 x i16>
+ %759 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %758, <4 x float> %757, i32 0, i32 0, i32 0)
+ %760 = shufflevector <1 x bfloat> %554, <1 x bfloat> %558, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %761 = shufflevector <4 x bfloat> %760, <4 x bfloat> %565, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %762 = shufflevector <4 x bfloat> %761, <4 x bfloat> %573, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %763 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 0, 0
+ %764 = bitcast <4 x bfloat> %762 to <4 x i16>
+ %765 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %764, <4 x float> %763, i32 0, i32 0, i32 0)
+ %766 = shufflevector <1 x bfloat> %555, <1 x bfloat> %559, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %767 = shufflevector <4 x bfloat> %766, <4 x bfloat> %567, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %768 = shufflevector <4 x bfloat> %767, <4 x bfloat> %575, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %769 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 0, 0
+ %770 = bitcast <4 x bfloat> %768 to <4 x i16>
+ %771 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %770, <4 x float> %769, i32 0, i32 0, i32 0)
+ %772 = shufflevector <1 x bfloat> %457, <1 x bfloat> %459, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %773 = shufflevector <4 x bfloat> %772, <4 x bfloat> %463, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %774 = shufflevector <4 x bfloat> %773, <4 x bfloat> %467, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %775 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 1, 0
+ %776 = bitcast <4 x bfloat> %774 to <4 x i16>
+ %777 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %752, <4 x float> %775, i32 0, i32 0, i32 0)
+ %778 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 1, 0
+ %779 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %758, <4 x float> %778, i32 0, i32 0, i32 0)
+ %780 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 1, 0
+ %781 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %764, <4 x float> %780, i32 0, i32 0, i32 0)
+ %782 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 1, 0
+ %783 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %770, <4 x float> %782, i32 0, i32 0, i32 0)
+ %784 = shufflevector <1 x bfloat> %468, <1 x bfloat> %470, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %785 = shufflevector <4 x bfloat> %784, <4 x bfloat> %473, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %786 = shufflevector <4 x bfloat> %785, <4 x bfloat> %477, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %787 = shufflevector <1 x bfloat> %576, <1 x bfloat> %580, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %788 = shufflevector <4 x bfloat> %787, <4 x bfloat> %585, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %789 = shufflevector <4 x bfloat> %788, <4 x bfloat> %593, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %790 = bitcast <4 x bfloat> %786 to <4 x i16>
+ %791 = bitcast <4 x bfloat> %789 to <4 x i16>
+ %792 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %791, <4 x float> %753, i32 0, i32 0, i32 0)
+ %793 = shufflevector <1 x bfloat> %577, <1 x bfloat> %581, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %794 = shufflevector <4 x bfloat> %793, <4 x bfloat> %587, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %795 = shufflevector <4 x bfloat> %794, <4 x bfloat> %595, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %796 = bitcast <4 x bfloat> %795 to <4 x i16>
+ %797 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %796, <4 x float> %759, i32 0, i32 0, i32 0)
+ %798 = shufflevector <1 x bfloat> %578, <1 x bfloat> %582, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %799 = shufflevector <4 x bfloat> %798, <4 x bfloat> %589, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %800 = shufflevector <4 x bfloat> %799, <4 x bfloat> %597, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %801 = bitcast <4 x bfloat> %800 to <4 x i16>
+ %802 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %801, <4 x float> %765, i32 0, i32 0, i32 0)
+ %803 = shufflevector <1 x bfloat> %579, <1 x bfloat> %583, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %804 = shufflevector <4 x bfloat> %803, <4 x bfloat> %591, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %805 = shufflevector <4 x bfloat> %804, <4 x bfloat> %599, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %806 = bitcast <4 x bfloat> %805 to <4 x i16>
+ %807 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %806, <4 x float> %771, i32 0, i32 0, i32 0)
+ %808 = shufflevector <1 x bfloat> %469, <1 x bfloat> %471, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %809 = shufflevector <4 x bfloat> %808, <4 x bfloat> %475, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %810 = shufflevector <4 x bfloat> %809, <4 x bfloat> %479, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %811 = bitcast <4 x bfloat> %810 to <4 x i16>
+ %812 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %791, <4 x float> %777, i32 0, i32 0, i32 0)
+ %813 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %796, <4 x float> %779, i32 0, i32 0, i32 0)
+ %814 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %801, <4 x float> %781, i32 0, i32 0, i32 0)
+ %815 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %806, <4 x float> %783, i32 0, i32 0, i32 0)
+ %816 = shufflevector <1 x bfloat> %480, <1 x bfloat> %482, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %817 = shufflevector <4 x bfloat> %816, <4 x bfloat> %485, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %818 = shufflevector <4 x bfloat> %817, <4 x bfloat> %489, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %819 = shufflevector <1 x bfloat> %600, <1 x bfloat> %604, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %820 = shufflevector <4 x bfloat> %819, <4 x bfloat> %609, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %821 = shufflevector <4 x bfloat> %820, <4 x bfloat> %617, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %822 = bitcast <4 x bfloat> %818 to <4 x i16>
+ %823 = bitcast <4 x bfloat> %821 to <4 x i16>
+ %824 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %823, <4 x float> %792, i32 0, i32 0, i32 0)
+ %825 = shufflevector <1 x bfloat> %601, <1 x bfloat> %605, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %826 = shufflevector <4 x bfloat> %825, <4 x bfloat> %611, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %827 = shufflevector <4 x bfloat> %826, <4 x bfloat> %619, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %828 = bitcast <4 x bfloat> %827 to <4 x i16>
+ %829 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %828, <4 x float> %797, i32 0, i32 0, i32 0)
+ %830 = shufflevector <1 x bfloat> %602, <1 x bfloat> %606, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %831 = shufflevector <4 x bfloat> %830, <4 x bfloat> %613, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %832 = shufflevector <4 x bfloat> %831, <4 x bfloat> %621, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %833 = bitcast <4 x bfloat> %832 to <4 x i16>
+ %834 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %833, <4 x float> %802, i32 0, i32 0, i32 0)
+ %835 = shufflevector <1 x bfloat> %603, <1 x bfloat> %607, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %836 = shufflevector <4 x bfloat> %835, <4 x bfloat> %615, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %837 = shufflevector <4 x bfloat> %836, <4 x bfloat> %623, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %838 = bitcast <4 x bfloat> %837 to <4 x i16>
+ %839 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %838, <4 x float> %807, i32 0, i32 0, i32 0)
+ %840 = shufflevector <1 x bfloat> %481, <1 x bfloat> %483, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %841 = shufflevector <4 x bfloat> %840, <4 x bfloat> %487, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %842 = shufflevector <4 x bfloat> %841, <4 x bfloat> %491, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %843 = bitcast <4 x bfloat> %842 to <4 x i16>
+ %844 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %823, <4 x float> %812, i32 0, i32 0, i32 0)
+ %845 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %828, <4 x float> %813, i32 0, i32 0, i32 0)
+ %846 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %833, <4 x float> %814, i32 0, i32 0, i32 0)
+ %847 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %838, <4 x float> %815, i32 0, i32 0, i32 0)
+ %848 = shufflevector <1 x bfloat> %492, <1 x bfloat> %494, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %849 = shufflevector <4 x bfloat> %848, <4 x bfloat> %497, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %850 = shufflevector <4 x bfloat> %849, <4 x bfloat> %501, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %851 = shufflevector <1 x bfloat> %624, <1 x bfloat> %628, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %852 = shufflevector <4 x bfloat> %851, <4 x bfloat> %633, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %853 = shufflevector <4 x bfloat> %852, <4 x bfloat> %641, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %854 = bitcast <4 x bfloat> %850 to <4 x i16>
+ %855 = bitcast <4 x bfloat> %853 to <4 x i16>
+ %856 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %855, <4 x float> %824, i32 0, i32 0, i32 0)
+ %857 = shufflevector <1 x bfloat> %625, <1 x bfloat> %629, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %858 = shufflevector <4 x bfloat> %857, <4 x bfloat> %635, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %859 = shufflevector <4 x bfloat> %858, <4 x bfloat> %643, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %860 = bitcast <4 x bfloat> %859 to <4 x i16>
+ %861 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %860, <4 x float> %829, i32 0, i32 0, i32 0)
+ %862 = shufflevector <1 x bfloat> %626, <1 x bfloat> %630, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %863 = shufflevector <4 x bfloat> %862, <4 x bfloat> %637, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %864 = shufflevector <4 x bfloat> %863, <4 x bfloat> %645, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %865 = bitcast <4 x bfloat> %864 to <4 x i16>
+ %866 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %865, <4 x float> %834, i32 0, i32 0, i32 0)
+ %867 = shufflevector <1 x bfloat> %627, <1 x bfloat> %631, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %868 = shufflevector <4 x bfloat> %867, <4 x bfloat> %639, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %869 = shufflevector <4 x bfloat> %868, <4 x bfloat> %647, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %870 = bitcast <4 x bfloat> %869 to <4 x i16>
+ %871 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %870, <4 x float> %839, i32 0, i32 0, i32 0)
+ %872 = shufflevector <1 x bfloat> %493, <1 x bfloat> %495, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %873 = shufflevector <4 x bfloat> %872, <4 x bfloat> %499, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %874 = shufflevector <4 x bfloat> %873, <4 x bfloat> %503, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %875 = bitcast <4 x bfloat> %874 to <4 x i16>
+ %876 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %855, <4 x float> %844, i32 0, i32 0, i32 0)
+ %877 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %860, <4 x float> %845, i32 0, i32 0, i32 0)
+ %878 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %865, <4 x float> %846, i32 0, i32 0, i32 0)
+ %879 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %870, <4 x float> %847, i32 0, i32 0, i32 0)
+ %880 = shufflevector <1 x bfloat> %504, <1 x bfloat> %506, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %881 = shufflevector <4 x bfloat> %880, <4 x bfloat> %509, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %882 = shufflevector <4 x bfloat> %881, <4 x bfloat> %513, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %883 = shufflevector <1 x bfloat> %648, <1 x bfloat> %652, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %884 = shufflevector <4 x bfloat> %883, <4 x bfloat> %657, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %885 = shufflevector <4 x bfloat> %884, <4 x bfloat> %665, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %886 = bitcast <4 x bfloat> %882 to <4 x i16>
+ %887 = bitcast <4 x bfloat> %885 to <4 x i16>
+ %888 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %887, <4 x float> %856, i32 0, i32 0, i32 0)
+ %889 = shufflevector <1 x bfloat> %649, <1 x bfloat> %653, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %890 = shufflevector <4 x bfloat> %889, <4 x bfloat> %659, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %891 = shufflevector <4 x bfloat> %890, <4 x bfloat> %667, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %892 = bitcast <4 x bfloat> %891 to <4 x i16>
+ %893 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %892, <4 x float> %861, i32 0, i32 0, i32 0)
+ %894 = shufflevector <1 x bfloat> %650, <1 x bfloat> %654, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %895 = shufflevector <4 x bfloat> %894, <4 x bfloat> %661, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %896 = shufflevector <4 x bfloat> %895, <4 x bfloat> %669, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %897 = bitcast <4 x bfloat> %896 to <4 x i16>
+ %898 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %897, <4 x float> %866, i32 0, i32 0, i32 0)
+ %899 = shufflevector <1 x bfloat> %651, <1 x bfloat> %655, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %900 = shufflevector <4 x bfloat> %899, <4 x bfloat> %663, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %901 = shufflevector <4 x bfloat> %900, <4 x bfloat> %671, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %902 = bitcast <4 x bfloat> %901 to <4 x i16>
+ %903 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %902, <4 x float> %871, i32 0, i32 0, i32 0)
+ %904 = shufflevector <1 x bfloat> %505, <1 x bfloat> %507, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %905 = shufflevector <4 x bfloat> %904, <4 x bfloat> %511, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %906 = shufflevector <4 x bfloat> %905, <4 x bfloat> %515, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %907 = bitcast <4 x bfloat> %906 to <4 x i16>
+ %908 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %887, <4 x float> %876, i32 0, i32 0, i32 0)
+ %909 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %892, <4 x float> %877, i32 0, i32 0, i32 0)
+ %910 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %897, <4 x float> %878, i32 0, i32 0, i32 0)
+ %911 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %902, <4 x float> %879, i32 0, i32 0, i32 0)
+ %912 = shufflevector <1 x bfloat> %516, <1 x bfloat> %518, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %913 = shufflevector <4 x bfloat> %912, <4 x bfloat> %521, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %914 = shufflevector <4 x bfloat> %913, <4 x bfloat> %525, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %915 = shufflevector <1 x bfloat> %672, <1 x bfloat> %676, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %916 = shufflevector <4 x bfloat> %915, <4 x bfloat> %681, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %917 = shufflevector <4 x bfloat> %916, <4 x bfloat> %689, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %918 = bitcast <4 x bfloat> %914 to <4 x i16>
+ %919 = bitcast <4 x bfloat> %917 to <4 x i16>
+ %920 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %919, <4 x float> %888, i32 0, i32 0, i32 0)
+ %921 = shufflevector <1 x bfloat> %673, <1 x bfloat> %677, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %922 = shufflevector <4 x bfloat> %921, <4 x bfloat> %683, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %923 = shufflevector <4 x bfloat> %922, <4 x bfloat> %691, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %924 = bitcast <4 x bfloat> %923 to <4 x i16>
+ %925 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %924, <4 x float> %893, i32 0, i32 0, i32 0)
+ %926 = shufflevector <1 x bfloat> %674, <1 x bfloat> %678, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %927 = shufflevector <4 x bfloat> %926, <4 x bfloat> %685, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %928 = shufflevector <4 x bfloat> %927, <4 x bfloat> %693, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %929 = bitcast <4 x bfloat> %928 to <4 x i16>
+ %930 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %929, <4 x float> %898, i32 0, i32 0, i32 0)
+ %931 = shufflevector <1 x bfloat> %675, <1 x bfloat> %679, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %932 = shufflevector <4 x bfloat> %931, <4 x bfloat> %687, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %933 = shufflevector <4 x bfloat> %932, <4 x bfloat> %695, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %934 = bitcast <4 x bfloat> %933 to <4 x i16>
+ %935 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %934, <4 x float> %903, i32 0, i32 0, i32 0)
+ %936 = shufflevector <1 x bfloat> %517, <1 x bfloat> %519, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %937 = shufflevector <4 x bfloat> %936, <4 x bfloat> %523, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %938 = shufflevector <4 x bfloat> %937, <4 x bfloat> %527, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %939 = bitcast <4 x bfloat> %938 to <4 x i16>
+ %940 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %919, <4 x float> %908, i32 0, i32 0, i32 0)
+ %941 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %924, <4 x float> %909, i32 0, i32 0, i32 0)
+ %942 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %929, <4 x float> %910, i32 0, i32 0, i32 0)
+ %943 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %934, <4 x float> %911, i32 0, i32 0, i32 0)
+ %944 = shufflevector <1 x bfloat> %528, <1 x bfloat> %530, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %945 = shufflevector <4 x bfloat> %944, <4 x bfloat> %533, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %946 = shufflevector <4 x bfloat> %945, <4 x bfloat> %537, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %947 = shufflevector <1 x bfloat> %696, <1 x bfloat> %700, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %948 = shufflevector <4 x bfloat> %947, <4 x bfloat> %705, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %949 = shufflevector <4 x bfloat> %948, <4 x bfloat> %713, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %950 = bitcast <4 x bfloat> %946 to <4 x i16>
+ %951 = bitcast <4 x bfloat> %949 to <4 x i16>
+ %952 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %951, <4 x float> %920, i32 0, i32 0, i32 0)
+ %953 = shufflevector <1 x bfloat> %697, <1 x bfloat> %701, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %954 = shufflevector <4 x bfloat> %953, <4 x bfloat> %707, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %955 = shufflevector <4 x bfloat> %954, <4 x bfloat> %715, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %956 = bitcast <4 x bfloat> %955 to <4 x i16>
+ %957 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %956, <4 x float> %925, i32 0, i32 0, i32 0)
+ %958 = shufflevector <1 x bfloat> %698, <1 x bfloat> %702, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %959 = shufflevector <4 x bfloat> %958, <4 x bfloat> %709, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %960 = shufflevector <4 x bfloat> %959, <4 x bfloat> %717, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %961 = bitcast <4 x bfloat> %960 to <4 x i16>
+ %962 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %961, <4 x float> %930, i32 0, i32 0, i32 0)
+ %963 = shufflevector <1 x bfloat> %699, <1 x bfloat> %703, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %964 = shufflevector <4 x bfloat> %963, <4 x bfloat> %711, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %965 = shufflevector <4 x bfloat> %964, <4 x bfloat> %719, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %966 = bitcast <4 x bfloat> %965 to <4 x i16>
+ %967 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %966, <4 x float> %935, i32 0, i32 0, i32 0)
+ %968 = shufflevector <1 x bfloat> %529, <1 x bfloat> %531, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %969 = shufflevector <4 x bfloat> %968, <4 x bfloat> %535, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %970 = shufflevector <4 x bfloat> %969, <4 x bfloat> %539, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %971 = bitcast <4 x bfloat> %970 to <4 x i16>
+ %972 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %951, <4 x float> %940, i32 0, i32 0, i32 0)
+ %973 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %956, <4 x float> %941, i32 0, i32 0, i32 0)
+ %974 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %961, <4 x float> %942, i32 0, i32 0, i32 0)
+ %975 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %966, <4 x float> %943, i32 0, i32 0, i32 0)
+ %976 = shufflevector <1 x bfloat> %540, <1 x bfloat> %542, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %977 = shufflevector <4 x bfloat> %976, <4 x bfloat> %545, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %978 = shufflevector <4 x bfloat> %977, <4 x bfloat> %549, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %979 = shufflevector <1 x bfloat> %720, <1 x bfloat> %724, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %980 = shufflevector <4 x bfloat> %979, <4 x bfloat> %729, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %981 = shufflevector <4 x bfloat> %980, <4 x bfloat> %737, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %982 = bitcast <4 x bfloat> %978 to <4 x i16>
+ %983 = bitcast <4 x bfloat> %981 to <4 x i16>
+ %984 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %983, <4 x float> %952, i32 0, i32 0, i32 0)
+ %985 = shufflevector <1 x bfloat> %721, <1 x bfloat> %725, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %986 = shufflevector <4 x bfloat> %985, <4 x bfloat> %731, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %987 = shufflevector <4 x bfloat> %986, <4 x bfloat> %739, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %988 = bitcast <4 x bfloat> %987 to <4 x i16>
+ %989 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %988, <4 x float> %957, i32 0, i32 0, i32 0)
+ %990 = shufflevector <1 x bfloat> %722, <1 x bfloat> %726, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %991 = shufflevector <4 x bfloat> %990, <4 x bfloat> %733, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %992 = shufflevector <4 x bfloat> %991, <4 x bfloat> %741, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %993 = bitcast <4 x bfloat> %992 to <4 x i16>
+ %994 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %993, <4 x float> %962, i32 0, i32 0, i32 0)
+ %995 = shufflevector <1 x bfloat> %723, <1 x bfloat> %727, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %996 = shufflevector <4 x bfloat> %995, <4 x bfloat> %735, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %997 = shufflevector <4 x bfloat> %996, <4 x bfloat> %743, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %998 = bitcast <4 x bfloat> %997 to <4 x i16>
+ %999 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %998, <4 x float> %967, i32 0, i32 0, i32 0)
+ %1000 = shufflevector <1 x bfloat> %541, <1 x bfloat> %543, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1001 = shufflevector <4 x bfloat> %1000, <4 x bfloat> %547, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1002 = shufflevector <4 x bfloat> %1001, <4 x bfloat> %551, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1003 = bitcast <4 x bfloat> %1002 to <4 x i16>
+ %1004 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %983, <4 x float> %972, i32 0, i32 0, i32 0)
+ %1005 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %988, <4 x float> %973, i32 0, i32 0, i32 0)
+ %1006 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %993, <4 x float> %974, i32 0, i32 0, i32 0)
+ %1007 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %998, <4 x float> %975, i32 0, i32 0, i32 0)
+ %1008 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %984, 0, 0, 0
+ %1009 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1008, <4 x float> %989, 1, 0, 0
+ %1010 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1009, <4 x float> %994, 2, 0, 0
+ %1011 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1010, <4 x float> %999, 3, 0, 0
+ %1012 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1011, <4 x float> %1004, 0, 1, 0
+ %1013 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1012, <4 x float> %1005, 1, 1, 0
+ %1014 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1013, <4 x float> %1006, 2, 1, 0
+ %1015 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1014, <4 x float> %1007, 3, 1, 0
+ %1016 = trunc i32 %398 to i1
+ %1017 = and i1 %41, %1016
+ %1018 = select i1 %1017, <8 x bfloat> %400, <8 x bfloat> zeroinitializer
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ tail call void @llvm.amdgcn.sched.barrier(i32 0)
+ store <8 x bfloat> %1018, ptr addrspace(3) %85, align 8
+ %1019 = trunc i32 %403 to i1
+ %1020 = and i1 %41, %1019
+ %1021 = select i1 %1020, <8 x bfloat> %405, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1021, ptr addrspace(3) %88, align 8
+ %1022 = trunc i32 %408 to i1
+ %1023 = and i1 %41, %1022
+ %1024 = select i1 %1023, <8 x bfloat> %410, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1024, ptr addrspace(3) %91, align 8
+ %1025 = trunc i32 %413 to i1
+ %1026 = and i1 %41, %1025
+ %1027 = select i1 %1026, <8 x bfloat> %415, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1027, ptr addrspace(3) %94, align 8
+ %1028 = trunc i32 %418 to i1
+ %1029 = and i1 %44, %1028
+ %1030 = select i1 %1029, <8 x bfloat> %420, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1030, ptr addrspace(3) %97, align 8
+ %1031 = trunc i32 %423 to i1
+ %1032 = and i1 %44, %1031
+ %1033 = select i1 %1032, <8 x bfloat> %425, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1033, ptr addrspace(3) %100, align 8
+ %1034 = trunc i32 %428 to i1
+ %1035 = and i1 %44, %1034
+ %1036 = select i1 %1035, <8 x bfloat> %430, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1036, ptr addrspace(3) %103, align 8
+ %1037 = trunc i32 %433 to i1
+ %1038 = and i1 %44, %1037
+ %1039 = select i1 %1038, <8 x bfloat> %435, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1039, ptr addrspace(3) %106, align 8
+ %1040 = trunc i32 %438 to i1
+ %1041 = and i1 %44, %1040
+ %1042 = select i1 %1041, <8 x bfloat> %440, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1042, ptr addrspace(3) %109, align 8
+ %1043 = trunc i32 %443 to i1
+ %1044 = and i1 %44, %1043
+ %1045 = select i1 %1044, <8 x bfloat> %445, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1045, ptr addrspace(3) %112, align 8
+ %1046 = trunc i32 %448 to i1
+ %1047 = and i1 %44, %1046
+ %1048 = select i1 %1047, <8 x bfloat> %450, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1048, ptr addrspace(3) %115, align 8
+ %1049 = trunc i32 %453 to i1
+ %1050 = and i1 %44, %1049
+ %1051 = select i1 %1050, <8 x bfloat> %455, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %1051, ptr addrspace(3) %118, align 8
+ %1052 = icmp samesign ult i32 %391, 72
+ br i1 %1052, label %389, label %1053
+
+1053: ; preds = %389
+ %1054 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %2, i16 0, i64 8128512, i32 159744)
+ %1055 = shl nuw nsw i32 %8, 5
+ %.scalar = or disjoint i32 %1055, %11
+ %1056 = insertelement <4 x i32> poison, i32 %.scalar, i64 0
+ %1057 = shufflevector <4 x i32> %1056, <4 x i32> poison, <4 x i32> zeroinitializer
+ %1058 = or disjoint <4 x i32> %1057, <i32 0, i32 1, i32 2, i32 3>
+ %1059 = or disjoint <4 x i32> %1057, <i32 16, i32 17, i32 18, i32 19>
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ %1060 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
+ %1061 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
+ %1062 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
+ %1063 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
+ %1064 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
+ %1065 = shufflevector <1 x bfloat> %1064, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1066 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
+ %1067 = shufflevector <1 x bfloat> %1066, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1068 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
+ %1069 = shufflevector <1 x bfloat> %1068, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1070 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
+ %1071 = shufflevector <1 x bfloat> %1070, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1072 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
+ %1073 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
+ %1074 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
+ %1075 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
+ %1076 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
+ %1077 = shufflevector <1 x bfloat> %1076, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1078 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
+ %1079 = shufflevector <1 x bfloat> %1078, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1080 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
+ %1081 = shufflevector <1 x bfloat> %1080, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1082 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
+ %1083 = shufflevector <1 x bfloat> %1082, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1084 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
+ %1085 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
+ %1086 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
+ %1087 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
+ %1088 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
+ %1089 = shufflevector <1 x bfloat> %1088, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1090 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
+ %1091 = shufflevector <1 x bfloat> %1090, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1092 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
+ %1093 = shufflevector <1 x bfloat> %1092, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1094 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
+ %1095 = shufflevector <1 x bfloat> %1094, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1096 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
+ %1097 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
+ %1098 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
+ %1099 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
+ %1100 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
+ %1101 = shufflevector <1 x bfloat> %1100, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1102 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
+ %1103 = shufflevector <1 x bfloat> %1102, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1104 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
+ %1105 = shufflevector <1 x bfloat> %1104, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1106 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
+ %1107 = shufflevector <1 x bfloat> %1106, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1108 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
+ %1109 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
+ %1110 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
+ %1111 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
+ %1112 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
+ %1113 = shufflevector <1 x bfloat> %1112, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1114 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
+ %1115 = shufflevector <1 x bfloat> %1114, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1116 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
+ %1117 = shufflevector <1 x bfloat> %1116, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1118 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
+ %1119 = shufflevector <1 x bfloat> %1118, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1120 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
+ %1121 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
+ %1122 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
+ %1123 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
+ %1124 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
+ %1125 = shufflevector <1 x bfloat> %1124, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1126 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
+ %1127 = shufflevector <1 x bfloat> %1126, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1128 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
+ %1129 = shufflevector <1 x bfloat> %1128, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1130 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
+ %1131 = shufflevector <1 x bfloat> %1130, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1132 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
+ %1133 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
+ %1134 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
+ %1135 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
+ %1136 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
+ %1137 = shufflevector <1 x bfloat> %1136, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1138 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
+ %1139 = shufflevector <1 x bfloat> %1138, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1140 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
+ %1141 = shufflevector <1 x bfloat> %1140, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1142 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
+ %1143 = shufflevector <1 x bfloat> %1142, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1144 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
+ %1145 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
+ %1146 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
+ %1147 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
+ %1148 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
+ %1149 = shufflevector <1 x bfloat> %1148, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1150 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
+ %1151 = shufflevector <1 x bfloat> %1150, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1152 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
+ %1153 = shufflevector <1 x bfloat> %1152, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1154 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
+ %1155 = shufflevector <1 x bfloat> %1154, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1156 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
+ %1157 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
+ %1158 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
+ %1159 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
+ %1160 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
+ %1161 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
+ %1162 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
+ %1163 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
+ %1164 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
+ %1165 = shufflevector <1 x bfloat> %1164, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1166 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
+ %1167 = shufflevector <1 x bfloat> %1166, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1168 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
+ %1169 = shufflevector <1 x bfloat> %1168, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1170 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
+ %1171 = shufflevector <1 x bfloat> %1170, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1172 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
+ %1173 = shufflevector <1 x bfloat> %1172, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1174 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
+ %1175 = shufflevector <1 x bfloat> %1174, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1176 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
+ %1177 = shufflevector <1 x bfloat> %1176, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1178 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
+ %1179 = shufflevector <1 x bfloat> %1178, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1180 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
+ %1181 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
+ %1182 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
+ %1183 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
+ %1184 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
+ %1185 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
+ %1186 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
+ %1187 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
+ %1188 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
+ %1189 = shufflevector <1 x bfloat> %1188, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1190 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
+ %1191 = shufflevector <1 x bfloat> %1190, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1192 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
+ %1193 = shufflevector <1 x bfloat> %1192, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1194 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
+ %1195 = shufflevector <1 x bfloat> %1194, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1196 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
+ %1197 = shufflevector <1 x bfloat> %1196, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1198 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
+ %1199 = shufflevector <1 x bfloat> %1198, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1200 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
+ %1201 = shufflevector <1 x bfloat> %1200, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1202 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
+ %1203 = shufflevector <1 x bfloat> %1202, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1204 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
+ %1205 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
+ %1206 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
+ %1207 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
+ %1208 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
+ %1209 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
+ %1210 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
+ %1211 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
+ %1212 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
+ %1213 = shufflevector <1 x bfloat> %1212, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1214 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
+ %1215 = shufflevector <1 x bfloat> %1214, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1216 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
+ %1217 = shufflevector <1 x bfloat> %1216, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1218 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
+ %1219 = shufflevector <1 x bfloat> %1218, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1220 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
+ %1221 = shufflevector <1 x bfloat> %1220, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1222 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
+ %1223 = shufflevector <1 x bfloat> %1222, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1224 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
+ %1225 = shufflevector <1 x bfloat> %1224, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1226 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
+ %1227 = shufflevector <1 x bfloat> %1226, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1228 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
+ %1229 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
+ %1230 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
+ %1231 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
+ %1232 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
+ %1233 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
+ %1234 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
+ %1235 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
+ %1236 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
+ %1237 = shufflevector <1 x bfloat> %1236, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1238 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
+ %1239 = shufflevector <1 x bfloat> %1238, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1240 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
+ %1241 = shufflevector <1 x bfloat> %1240, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1242 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
+ %1243 = shufflevector <1 x bfloat> %1242, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1244 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
+ %1245 = shufflevector <1 x bfloat> %1244, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1246 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
+ %1247 = shufflevector <1 x bfloat> %1246, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1248 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
+ %1249 = shufflevector <1 x bfloat> %1248, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1250 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
+ %1251 = shufflevector <1 x bfloat> %1250, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1252 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
+ %1253 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
+ %1254 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
+ %1255 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
+ %1256 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
+ %1257 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
+ %1258 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
+ %1259 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
+ %1260 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
+ %1261 = shufflevector <1 x bfloat> %1260, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1262 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
+ %1263 = shufflevector <1 x bfloat> %1262, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1264 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
+ %1265 = shufflevector <1 x bfloat> %1264, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1266 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
+ %1267 = shufflevector <1 x bfloat> %1266, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1268 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
+ %1269 = shufflevector <1 x bfloat> %1268, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1270 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
+ %1271 = shufflevector <1 x bfloat> %1270, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1272 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
+ %1273 = shufflevector <1 x bfloat> %1272, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1274 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
+ %1275 = shufflevector <1 x bfloat> %1274, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1276 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
+ %1277 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
+ %1278 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
+ %1279 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
+ %1280 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
+ %1281 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
+ %1282 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
+ %1283 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
+ %1284 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
+ %1285 = shufflevector <1 x bfloat> %1284, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1286 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
+ %1287 = shufflevector <1 x bfloat> %1286, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1288 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
+ %1289 = shufflevector <1 x bfloat> %1288, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1290 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
+ %1291 = shufflevector <1 x bfloat> %1290, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1292 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
+ %1293 = shufflevector <1 x bfloat> %1292, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1294 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
+ %1295 = shufflevector <1 x bfloat> %1294, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1296 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
+ %1297 = shufflevector <1 x bfloat> %1296, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1298 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
+ %1299 = shufflevector <1 x bfloat> %1298, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1300 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
+ %1301 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
+ %1302 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
+ %1303 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
+ %1304 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
+ %1305 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
+ %1306 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
+ %1307 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
+ %1308 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
+ %1309 = shufflevector <1 x bfloat> %1308, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1310 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
+ %1311 = shufflevector <1 x bfloat> %1310, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1312 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
+ %1313 = shufflevector <1 x bfloat> %1312, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1314 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
+ %1315 = shufflevector <1 x bfloat> %1314, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1316 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
+ %1317 = shufflevector <1 x bfloat> %1316, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1318 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
+ %1319 = shufflevector <1 x bfloat> %1318, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1320 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
+ %1321 = shufflevector <1 x bfloat> %1320, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1322 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
+ %1323 = shufflevector <1 x bfloat> %1322, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1324 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
+ %1325 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
+ %1326 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
+ %1327 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
+ %1328 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
+ %1329 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
+ %1330 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
+ %1331 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
+ %1332 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
+ %1333 = shufflevector <1 x bfloat> %1332, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1334 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
+ %1335 = shufflevector <1 x bfloat> %1334, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1336 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
+ %1337 = shufflevector <1 x bfloat> %1336, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1338 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
+ %1339 = shufflevector <1 x bfloat> %1338, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1340 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
+ %1341 = shufflevector <1 x bfloat> %1340, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1342 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
+ %1343 = shufflevector <1 x bfloat> %1342, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1344 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
+ %1345 = shufflevector <1 x bfloat> %1344, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1346 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
+ %1347 = shufflevector <1 x bfloat> %1346, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %1348 = shufflevector <1 x bfloat> %1060, <1 x bfloat> %1062, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1349 = shufflevector <4 x bfloat> %1348, <4 x bfloat> %1065, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1350 = shufflevector <4 x bfloat> %1349, <4 x bfloat> %1069, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1351 = shufflevector <1 x bfloat> %1156, <1 x bfloat> %1160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1352 = shufflevector <4 x bfloat> %1351, <4 x bfloat> %1165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1353 = shufflevector <4 x bfloat> %1352, <4 x bfloat> %1173, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1354 = bitcast <4 x bfloat> %1350 to <4 x i16>
+ %1355 = bitcast <4 x bfloat> %1353 to <4 x i16>
+ %1356 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1355, <4 x float> %984, i32 0, i32 0, i32 0)
+ %1357 = shufflevector <1 x bfloat> %1157, <1 x bfloat> %1161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1358 = shufflevector <4 x bfloat> %1357, <4 x bfloat> %1167, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1359 = shufflevector <4 x bfloat> %1358, <4 x bfloat> %1175, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1360 = bitcast <4 x bfloat> %1359 to <4 x i16>
+ %1361 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1360, <4 x float> %989, i32 0, i32 0, i32 0)
+ %1362 = shufflevector <1 x bfloat> %1158, <1 x bfloat> %1162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1363 = shufflevector <4 x bfloat> %1362, <4 x bfloat> %1169, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1364 = shufflevector <4 x bfloat> %1363, <4 x bfloat> %1177, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1365 = bitcast <4 x bfloat> %1364 to <4 x i16>
+ %1366 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1365, <4 x float> %994, i32 0, i32 0, i32 0)
+ %1367 = shufflevector <1 x bfloat> %1159, <1 x bfloat> %1163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1368 = shufflevector <4 x bfloat> %1367, <4 x bfloat> %1171, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1369 = shufflevector <4 x bfloat> %1368, <4 x bfloat> %1179, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1370 = bitcast <4 x bfloat> %1369 to <4 x i16>
+ %1371 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1370, <4 x float> %999, i32 0, i32 0, i32 0)
+ %1372 = shufflevector <1 x bfloat> %1061, <1 x bfloat> %1063, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1373 = shufflevector <4 x bfloat> %1372, <4 x bfloat> %1067, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1374 = shufflevector <4 x bfloat> %1373, <4 x bfloat> %1071, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1375 = bitcast <4 x bfloat> %1374 to <4 x i16>
+ %1376 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1355, <4 x float> %1004, i32 0, i32 0, i32 0)
+ %1377 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1360, <4 x float> %1005, i32 0, i32 0, i32 0)
+ %1378 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1365, <4 x float> %1006, i32 0, i32 0, i32 0)
+ %1379 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1370, <4 x float> %1007, i32 0, i32 0, i32 0)
+ %1380 = shufflevector <1 x bfloat> %1072, <1 x bfloat> %1074, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1381 = shufflevector <4 x bfloat> %1380, <4 x bfloat> %1077, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1382 = shufflevector <4 x bfloat> %1381, <4 x bfloat> %1081, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1383 = shufflevector <1 x bfloat> %1180, <1 x bfloat> %1184, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1384 = shufflevector <4 x bfloat> %1383, <4 x bfloat> %1189, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1385 = shufflevector <4 x bfloat> %1384, <4 x bfloat> %1197, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1386 = bitcast <4 x bfloat> %1382 to <4 x i16>
+ %1387 = bitcast <4 x bfloat> %1385 to <4 x i16>
+ %1388 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1387, <4 x float> %1356, i32 0, i32 0, i32 0)
+ %1389 = shufflevector <1 x bfloat> %1181, <1 x bfloat> %1185, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1390 = shufflevector <4 x bfloat> %1389, <4 x bfloat> %1191, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1391 = shufflevector <4 x bfloat> %1390, <4 x bfloat> %1199, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1392 = bitcast <4 x bfloat> %1391 to <4 x i16>
+ %1393 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1392, <4 x float> %1361, i32 0, i32 0, i32 0)
+ %1394 = shufflevector <1 x bfloat> %1182, <1 x bfloat> %1186, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1395 = shufflevector <4 x bfloat> %1394, <4 x bfloat> %1193, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1396 = shufflevector <4 x bfloat> %1395, <4 x bfloat> %1201, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1397 = bitcast <4 x bfloat> %1396 to <4 x i16>
+ %1398 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1397, <4 x float> %1366, i32 0, i32 0, i32 0)
+ %1399 = shufflevector <1 x bfloat> %1183, <1 x bfloat> %1187, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1400 = shufflevector <4 x bfloat> %1399, <4 x bfloat> %1195, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1401 = shufflevector <4 x bfloat> %1400, <4 x bfloat> %1203, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1402 = bitcast <4 x bfloat> %1401 to <4 x i16>
+ %1403 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1402, <4 x float> %1371, i32 0, i32 0, i32 0)
+ %1404 = shufflevector <1 x bfloat> %1073, <1 x bfloat> %1075, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1405 = shufflevector <4 x bfloat> %1404, <4 x bfloat> %1079, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1406 = shufflevector <4 x bfloat> %1405, <4 x bfloat> %1083, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1407 = bitcast <4 x bfloat> %1406 to <4 x i16>
+ %1408 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1387, <4 x float> %1376, i32 0, i32 0, i32 0)
+ %1409 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1392, <4 x float> %1377, i32 0, i32 0, i32 0)
+ %1410 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1397, <4 x float> %1378, i32 0, i32 0, i32 0)
+ %1411 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1402, <4 x float> %1379, i32 0, i32 0, i32 0)
+ %1412 = shufflevector <1 x bfloat> %1084, <1 x bfloat> %1086, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1413 = shufflevector <4 x bfloat> %1412, <4 x bfloat> %1089, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1414 = shufflevector <4 x bfloat> %1413, <4 x bfloat> %1093, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1415 = shufflevector <1 x bfloat> %1204, <1 x bfloat> %1208, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1416 = shufflevector <4 x bfloat> %1415, <4 x bfloat> %1213, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1417 = shufflevector <4 x bfloat> %1416, <4 x bfloat> %1221, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1418 = bitcast <4 x bfloat> %1414 to <4 x i16>
+ %1419 = bitcast <4 x bfloat> %1417 to <4 x i16>
+ %1420 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1419, <4 x float> %1388, i32 0, i32 0, i32 0)
+ %1421 = shufflevector <1 x bfloat> %1205, <1 x bfloat> %1209, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1422 = shufflevector <4 x bfloat> %1421, <4 x bfloat> %1215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1423 = shufflevector <4 x bfloat> %1422, <4 x bfloat> %1223, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1424 = bitcast <4 x bfloat> %1423 to <4 x i16>
+ %1425 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1424, <4 x float> %1393, i32 0, i32 0, i32 0)
+ %1426 = shufflevector <1 x bfloat> %1206, <1 x bfloat> %1210, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1427 = shufflevector <4 x bfloat> %1426, <4 x bfloat> %1217, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1428 = shufflevector <4 x bfloat> %1427, <4 x bfloat> %1225, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1429 = bitcast <4 x bfloat> %1428 to <4 x i16>
+ %1430 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1429, <4 x float> %1398, i32 0, i32 0, i32 0)
+ %1431 = shufflevector <1 x bfloat> %1207, <1 x bfloat> %1211, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1432 = shufflevector <4 x bfloat> %1431, <4 x bfloat> %1219, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1433 = shufflevector <4 x bfloat> %1432, <4 x bfloat> %1227, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1434 = bitcast <4 x bfloat> %1433 to <4 x i16>
+ %1435 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1434, <4 x float> %1403, i32 0, i32 0, i32 0)
+ %1436 = shufflevector <1 x bfloat> %1085, <1 x bfloat> %1087, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1437 = shufflevector <4 x bfloat> %1436, <4 x bfloat> %1091, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1438 = shufflevector <4 x bfloat> %1437, <4 x bfloat> %1095, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1439 = bitcast <4 x bfloat> %1438 to <4 x i16>
+ %1440 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1419, <4 x float> %1408, i32 0, i32 0, i32 0)
+ %1441 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1424, <4 x float> %1409, i32 0, i32 0, i32 0)
+ %1442 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1429, <4 x float> %1410, i32 0, i32 0, i32 0)
+ %1443 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1434, <4 x float> %1411, i32 0, i32 0, i32 0)
+ %1444 = shufflevector <1 x bfloat> %1096, <1 x bfloat> %1098, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1445 = shufflevector <4 x bfloat> %1444, <4 x bfloat> %1101, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1446 = shufflevector <4 x bfloat> %1445, <4 x bfloat> %1105, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1447 = shufflevector <1 x bfloat> %1228, <1 x bfloat> %1232, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1448 = shufflevector <4 x bfloat> %1447, <4 x bfloat> %1237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1449 = shufflevector <4 x bfloat> %1448, <4 x bfloat> %1245, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1450 = bitcast <4 x bfloat> %1446 to <4 x i16>
+ %1451 = bitcast <4 x bfloat> %1449 to <4 x i16>
+ %1452 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1451, <4 x float> %1420, i32 0, i32 0, i32 0)
+ %1453 = shufflevector <1 x bfloat> %1229, <1 x bfloat> %1233, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1454 = shufflevector <4 x bfloat> %1453, <4 x bfloat> %1239, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1455 = shufflevector <4 x bfloat> %1454, <4 x bfloat> %1247, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1456 = bitcast <4 x bfloat> %1455 to <4 x i16>
+ %1457 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1456, <4 x float> %1425, i32 0, i32 0, i32 0)
+ %1458 = shufflevector <1 x bfloat> %1230, <1 x bfloat> %1234, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1459 = shufflevector <4 x bfloat> %1458, <4 x bfloat> %1241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1460 = shufflevector <4 x bfloat> %1459, <4 x bfloat> %1249, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1461 = bitcast <4 x bfloat> %1460 to <4 x i16>
+ %1462 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1461, <4 x float> %1430, i32 0, i32 0, i32 0)
+ %1463 = shufflevector <1 x bfloat> %1231, <1 x bfloat> %1235, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1464 = shufflevector <4 x bfloat> %1463, <4 x bfloat> %1243, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1465 = shufflevector <4 x bfloat> %1464, <4 x bfloat> %1251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1466 = bitcast <4 x bfloat> %1465 to <4 x i16>
+ %1467 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1466, <4 x float> %1435, i32 0, i32 0, i32 0)
+ %1468 = shufflevector <1 x bfloat> %1097, <1 x bfloat> %1099, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1469 = shufflevector <4 x bfloat> %1468, <4 x bfloat> %1103, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1470 = shufflevector <4 x bfloat> %1469, <4 x bfloat> %1107, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1471 = bitcast <4 x bfloat> %1470 to <4 x i16>
+ %1472 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1451, <4 x float> %1440, i32 0, i32 0, i32 0)
+ %1473 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1456, <4 x float> %1441, i32 0, i32 0, i32 0)
+ %1474 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1461, <4 x float> %1442, i32 0, i32 0, i32 0)
+ %1475 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1466, <4 x float> %1443, i32 0, i32 0, i32 0)
+ %1476 = shufflevector <1 x bfloat> %1108, <1 x bfloat> %1110, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1477 = shufflevector <4 x bfloat> %1476, <4 x bfloat> %1113, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1478 = shufflevector <4 x bfloat> %1477, <4 x bfloat> %1117, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1479 = shufflevector <1 x bfloat> %1252, <1 x bfloat> %1256, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1480 = shufflevector <4 x bfloat> %1479, <4 x bfloat> %1261, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1481 = shufflevector <4 x bfloat> %1480, <4 x bfloat> %1269, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1482 = bitcast <4 x bfloat> %1478 to <4 x i16>
+ %1483 = bitcast <4 x bfloat> %1481 to <4 x i16>
+ %1484 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1483, <4 x float> %1452, i32 0, i32 0, i32 0)
+ %1485 = shufflevector <1 x bfloat> %1253, <1 x bfloat> %1257, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1486 = shufflevector <4 x bfloat> %1485, <4 x bfloat> %1263, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1487 = shufflevector <4 x bfloat> %1486, <4 x bfloat> %1271, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1488 = bitcast <4 x bfloat> %1487 to <4 x i16>
+ %1489 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1488, <4 x float> %1457, i32 0, i32 0, i32 0)
+ %1490 = shufflevector <1 x bfloat> %1254, <1 x bfloat> %1258, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1491 = shufflevector <4 x bfloat> %1490, <4 x bfloat> %1265, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1492 = shufflevector <4 x bfloat> %1491, <4 x bfloat> %1273, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1493 = bitcast <4 x bfloat> %1492 to <4 x i16>
+ %1494 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1493, <4 x float> %1462, i32 0, i32 0, i32 0)
+ %1495 = shufflevector <1 x bfloat> %1255, <1 x bfloat> %1259, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1496 = shufflevector <4 x bfloat> %1495, <4 x bfloat> %1267, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1497 = shufflevector <4 x bfloat> %1496, <4 x bfloat> %1275, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1498 = bitcast <4 x bfloat> %1497 to <4 x i16>
+ %1499 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1498, <4 x float> %1467, i32 0, i32 0, i32 0)
+ %1500 = shufflevector <1 x bfloat> %1109, <1 x bfloat> %1111, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1501 = shufflevector <4 x bfloat> %1500, <4 x bfloat> %1115, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1502 = shufflevector <4 x bfloat> %1501, <4 x bfloat> %1119, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1503 = bitcast <4 x bfloat> %1502 to <4 x i16>
+ %1504 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1483, <4 x float> %1472, i32 0, i32 0, i32 0)
+ %1505 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1488, <4 x float> %1473, i32 0, i32 0, i32 0)
+ %1506 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1493, <4 x float> %1474, i32 0, i32 0, i32 0)
+ %1507 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1498, <4 x float> %1475, i32 0, i32 0, i32 0)
+ %1508 = shufflevector <1 x bfloat> %1120, <1 x bfloat> %1122, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1509 = shufflevector <4 x bfloat> %1508, <4 x bfloat> %1125, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1510 = shufflevector <4 x bfloat> %1509, <4 x bfloat> %1129, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1511 = shufflevector <1 x bfloat> %1276, <1 x bfloat> %1280, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1512 = shufflevector <4 x bfloat> %1511, <4 x bfloat> %1285, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1513 = shufflevector <4 x bfloat> %1512, <4 x bfloat> %1293, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1514 = bitcast <4 x bfloat> %1510 to <4 x i16>
+ %1515 = bitcast <4 x bfloat> %1513 to <4 x i16>
+ %1516 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1515, <4 x float> %1484, i32 0, i32 0, i32 0)
+ %1517 = shufflevector <1 x bfloat> %1277, <1 x bfloat> %1281, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1518 = shufflevector <4 x bfloat> %1517, <4 x bfloat> %1287, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1519 = shufflevector <4 x bfloat> %1518, <4 x bfloat> %1295, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1520 = bitcast <4 x bfloat> %1519 to <4 x i16>
+ %1521 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1520, <4 x float> %1489, i32 0, i32 0, i32 0)
+ %1522 = shufflevector <1 x bfloat> %1278, <1 x bfloat> %1282, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1523 = shufflevector <4 x bfloat> %1522, <4 x bfloat> %1289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1524 = shufflevector <4 x bfloat> %1523, <4 x bfloat> %1297, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1525 = bitcast <4 x bfloat> %1524 to <4 x i16>
+ %1526 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1525, <4 x float> %1494, i32 0, i32 0, i32 0)
+ %1527 = shufflevector <1 x bfloat> %1279, <1 x bfloat> %1283, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1528 = shufflevector <4 x bfloat> %1527, <4 x bfloat> %1291, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1529 = shufflevector <4 x bfloat> %1528, <4 x bfloat> %1299, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1530 = bitcast <4 x bfloat> %1529 to <4 x i16>
+ %1531 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1530, <4 x float> %1499, i32 0, i32 0, i32 0)
+ %1532 = shufflevector <1 x bfloat> %1121, <1 x bfloat> %1123, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1533 = shufflevector <4 x bfloat> %1532, <4 x bfloat> %1127, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1534 = shufflevector <4 x bfloat> %1533, <4 x bfloat> %1131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1535 = bitcast <4 x bfloat> %1534 to <4 x i16>
+ %1536 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1515, <4 x float> %1504, i32 0, i32 0, i32 0)
+ %1537 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1520, <4 x float> %1505, i32 0, i32 0, i32 0)
+ %1538 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1525, <4 x float> %1506, i32 0, i32 0, i32 0)
+ %1539 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1530, <4 x float> %1507, i32 0, i32 0, i32 0)
+ %1540 = shufflevector <1 x bfloat> %1132, <1 x bfloat> %1134, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1541 = shufflevector <4 x bfloat> %1540, <4 x bfloat> %1137, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1542 = shufflevector <4 x bfloat> %1541, <4 x bfloat> %1141, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1543 = shufflevector <1 x bfloat> %1300, <1 x bfloat> %1304, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1544 = shufflevector <4 x bfloat> %1543, <4 x bfloat> %1309, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1545 = shufflevector <4 x bfloat> %1544, <4 x bfloat> %1317, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1546 = bitcast <4 x bfloat> %1542 to <4 x i16>
+ %1547 = bitcast <4 x bfloat> %1545 to <4 x i16>
+ %1548 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1547, <4 x float> %1516, i32 0, i32 0, i32 0)
+ %1549 = shufflevector <1 x bfloat> %1301, <1 x bfloat> %1305, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1550 = shufflevector <4 x bfloat> %1549, <4 x bfloat> %1311, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1551 = shufflevector <4 x bfloat> %1550, <4 x bfloat> %1319, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1552 = bitcast <4 x bfloat> %1551 to <4 x i16>
+ %1553 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1552, <4 x float> %1521, i32 0, i32 0, i32 0)
+ %1554 = shufflevector <1 x bfloat> %1302, <1 x bfloat> %1306, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1555 = shufflevector <4 x bfloat> %1554, <4 x bfloat> %1313, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1556 = shufflevector <4 x bfloat> %1555, <4 x bfloat> %1321, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1557 = bitcast <4 x bfloat> %1556 to <4 x i16>
+ %1558 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1557, <4 x float> %1526, i32 0, i32 0, i32 0)
+ %1559 = shufflevector <1 x bfloat> %1303, <1 x bfloat> %1307, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1560 = shufflevector <4 x bfloat> %1559, <4 x bfloat> %1315, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1561 = shufflevector <4 x bfloat> %1560, <4 x bfloat> %1323, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1562 = bitcast <4 x bfloat> %1561 to <4 x i16>
+ %1563 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1562, <4 x float> %1531, i32 0, i32 0, i32 0)
+ %1564 = shufflevector <1 x bfloat> %1133, <1 x bfloat> %1135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1565 = shufflevector <4 x bfloat> %1564, <4 x bfloat> %1139, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1566 = shufflevector <4 x bfloat> %1565, <4 x bfloat> %1143, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1567 = bitcast <4 x bfloat> %1566 to <4 x i16>
+ %1568 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1547, <4 x float> %1536, i32 0, i32 0, i32 0)
+ %1569 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1552, <4 x float> %1537, i32 0, i32 0, i32 0)
+ %1570 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1557, <4 x float> %1538, i32 0, i32 0, i32 0)
+ %1571 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1562, <4 x float> %1539, i32 0, i32 0, i32 0)
+ %1572 = shufflevector <1 x bfloat> %1144, <1 x bfloat> %1146, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1573 = shufflevector <4 x bfloat> %1572, <4 x bfloat> %1149, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1574 = shufflevector <4 x bfloat> %1573, <4 x bfloat> %1153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1575 = shufflevector <1 x bfloat> %1324, <1 x bfloat> %1328, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1576 = shufflevector <4 x bfloat> %1575, <4 x bfloat> %1333, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1577 = shufflevector <4 x bfloat> %1576, <4 x bfloat> %1341, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1578 = bitcast <4 x bfloat> %1574 to <4 x i16>
+ %1579 = bitcast <4 x bfloat> %1577 to <4 x i16>
+ %1580 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1579, <4 x float> %1548, i32 0, i32 0, i32 0)
+ %1581 = shufflevector <1 x bfloat> %1325, <1 x bfloat> %1329, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1582 = shufflevector <4 x bfloat> %1581, <4 x bfloat> %1335, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1583 = shufflevector <4 x bfloat> %1582, <4 x bfloat> %1343, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1584 = bitcast <4 x bfloat> %1583 to <4 x i16>
+ %1585 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1584, <4 x float> %1553, i32 0, i32 0, i32 0)
+ %1586 = shufflevector <4 x float> %1585, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1587 = shufflevector <1 x bfloat> %1326, <1 x bfloat> %1330, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1588 = shufflevector <4 x bfloat> %1587, <4 x bfloat> %1337, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1589 = shufflevector <4 x bfloat> %1588, <4 x bfloat> %1345, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1590 = bitcast <4 x bfloat> %1589 to <4 x i16>
+ %1591 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1590, <4 x float> %1558, i32 0, i32 0, i32 0)
+ %1592 = shufflevector <4 x float> %1591, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1593 = shufflevector <1 x bfloat> %1327, <1 x bfloat> %1331, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1594 = shufflevector <4 x bfloat> %1593, <4 x bfloat> %1339, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1595 = shufflevector <4 x bfloat> %1594, <4 x bfloat> %1347, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1596 = bitcast <4 x bfloat> %1595 to <4 x i16>
+ %1597 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1596, <4 x float> %1563, i32 0, i32 0, i32 0)
+ %1598 = shufflevector <4 x float> %1597, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1599 = shufflevector <1 x bfloat> %1145, <1 x bfloat> %1147, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %1600 = shufflevector <4 x bfloat> %1599, <4 x bfloat> %1151, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %1601 = shufflevector <4 x bfloat> %1600, <4 x bfloat> %1155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %1602 = bitcast <4 x bfloat> %1601 to <4 x i16>
+ %1603 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1579, <4 x float> %1568, i32 0, i32 0, i32 0)
+ %1604 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1584, <4 x float> %1569, i32 0, i32 0, i32 0)
+ %1605 = shufflevector <4 x float> %1604, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1606 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1590, <4 x float> %1570, i32 0, i32 0, i32 0)
+ %1607 = shufflevector <4 x float> %1606, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1608 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1596, <4 x float> %1571, i32 0, i32 0, i32 0)
+ %1609 = shufflevector <4 x float> %1608, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1610 = shufflevector <4 x float> %1580, <4 x float> %1603, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1611 = shufflevector <32 x float> %1610, <32 x float> %1586, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1612 = shufflevector <32 x float> %1611, <32 x float> %1605, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1613 = shufflevector <32 x float> %1612, <32 x float> %1592, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1614 = shufflevector <32 x float> %1613, <32 x float> %1607, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1615 = shufflevector <32 x float> %1614, <32 x float> %1598, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison>
+ %1616 = shufflevector <32 x float> %1615, <32 x float> %1609, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
+ %1617 = insertelement <4 x i32> poison, i32 %38, i64 0
+ %1618 = insertelement <4 x i32> poison, i32 %32, i64 0
+ %1619 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1620 = shufflevector <4 x i32> %1618, <4 x i32> poison, <8 x i32> zeroinitializer
+ %1621 = or disjoint <8 x i32> %1619, %1620
+ %1622 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1623 = shufflevector <4 x i32> %1617, <4 x i32> poison, <8 x i32> zeroinitializer
+ %1624 = icmp samesign ult <8 x i32> %1622, %1623
+ br label %1625
+
+1625: ; preds = %1053, %1625
+ %1626 = phi i32 [ 0, %1053 ], [ %1676, %1625 ]
+ %1627 = shl nuw nsw i32 %1626, 3
+ %1628 = extractelement <32 x float> %1616, i32 %1627
+ %1629 = insertelement <4 x float> poison, float %1628, i64 0
+ %1630 = or disjoint i32 %1627, 1
+ %1631 = extractelement <32 x float> %1616, i32 %1630
+ %1632 = insertelement <4 x float> %1629, float %1631, i64 1
+ %1633 = or disjoint i32 %1627, 2
+ %1634 = extractelement <32 x float> %1616, i32 %1633
+ %1635 = insertelement <4 x float> %1632, float %1634, i64 2
+ %1636 = or disjoint i32 %1627, 3
+ %1637 = extractelement <32 x float> %1616, i32 %1636
+ %1638 = insertelement <4 x float> %1635, float %1637, i64 3
+ %1639 = or disjoint i32 %1627, 4
+ %1640 = extractelement <32 x float> %1616, i32 %1639
+ %1641 = insertelement <4 x float> poison, float %1640, i64 0
+ %1642 = or disjoint i32 %1627, 5
+ %1643 = extractelement <32 x float> %1616, i32 %1642
+ %1644 = insertelement <4 x float> %1641, float %1643, i64 1
+ %1645 = or disjoint i32 %1627, 6
+ %1646 = extractelement <32 x float> %1616, i32 %1645
+ %1647 = insertelement <4 x float> %1644, float %1646, i64 2
+ %1648 = or disjoint i32 %1627, 7
+ %1649 = extractelement <32 x float> %1616, i32 %1648
+ %1650 = insertelement <4 x float> %1647, float %1649, i64 3
+ %1651 = bitcast <4 x float> %1638 to <4 x i32>
+ %1652 = bitcast <4 x float> %1650 to <4 x i32>
+ %1653 = shl nuw nsw i32 %1626, 4
+ %.reass = add nuw nsw i32 %1653, %154
+ %1654 = icmp samesign ult i32 %.reass, %36
+ %1655 = insertelement <4 x i1> poison, i1 %1654, i64 0
+ %1656 = add nuw nsw i32 %.reass, %34
+ %1657 = mul nuw nsw i32 %1656, 2016
+ %1658 = insertelement <4 x i32> poison, i32 %1657, i64 0
+ %1659 = shufflevector <4 x i32> %1658, <4 x i32> poison, <8 x i32> zeroinitializer
+ %1660 = add nuw nsw <8 x i32> %1621, %1659
+ %1661 = shufflevector <4 x i1> %1655, <4 x i1> poison, <8 x i32> zeroinitializer
+ %1662 = and <8 x i1> %1624, %1661
+ %1663 = shufflevector <4 x float> %1638, <4 x float> %1650, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1664 = fcmp uno <8 x float> %1663, zeroinitializer
+ %1665 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1666 = add <8 x i32> %1665, splat (i32 32767)
+ %1667 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %1668 = lshr <8 x i32> %1667, splat (i32 16)
+ %1669 = and <8 x i32> %1668, splat (i32 1)
+ %1670 = add <8 x i32> %1666, %1669
+ %1671 = lshr <8 x i32> %1670, splat (i32 16)
+ %1672 = trunc nuw <8 x i32> %1671 to <8 x i16>
+ %1673 = bitcast <8 x i16> %1672 to <8 x bfloat>
+ %1674 = select <8 x i1> %1664, <8 x bfloat> splat (bfloat 0xR7FC0), <8 x bfloat> %1673
+ %1675 = getelementptr bfloat, ptr addrspace(7) %1054, <8 x i32> %1660
+ tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %1674, <8 x ptr addrspace(7)> align 2 %1675, <8 x i1> %1662)
+ %1676 = add nuw nsw i32 %1626, 1
+ %exitcond.not = icmp eq i32 %1676, 4
+ br i1 %exitcond.not, label %1677, label %1625
+
+1677: ; preds = %1625
+ fence syncscope("workgroup") release, !mmra !3
+ tail call void @llvm.amdgcn.s.barrier()
+ fence syncscope("workgroup") acquire, !mmra !3
+ ret void
+}
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef) #2
+
+; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) readnone, i16, i64, i32) #3
+
+; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.umin.i32(i32, i32) #3
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
+
+; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare i32 @llvm.smin.i32(i32, i32) #3
+
+; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.s.barrier() #4
+
+; Function Attrs: alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16>, <4 x i16>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #5
+
+; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write)
+declare void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat>, <8 x ptr addrspace(7)>, <8 x i1>) #6
+
+; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.sched.barrier(i32 immarg) #4
+
+attributes #0 = { alwaysinline nofree norecurse nounwind "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="256,256" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
+attributes #1 = { alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #3 = { alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #4 = { alwaysinline convergent mustprogress nocallback nofree nounwind willreturn }
+attributes #5 = { alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
+attributes #6 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write) }
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !{i32 1, !"amdhsa_code_object_version", i32 500}
+!2 = !{i32 256, i32 1, i32 1}
+!3 = !{!"amdgpu-synchronize-as", !"local"}
>From b190a35cc1f1339369c3f08522c12de390a412a9 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:20:29 -0600
Subject: [PATCH 04/13] Remove unnecessary debug code.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 12526862927ed..0479fb04be8bd 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2303,11 +2303,8 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
SpillCost *= (int64_t)RelativeFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0) {
- LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite rejected.\n");
+ if (SpillCost > 0)
return SpillCost;
- }
- LLVM_DEBUG(dbgs() << "RewriteMFMAFormStage: Rewrite successful.\n");
if (SpillCost < BestSpillCost)
BestSpillCost = SpillCost;
>From b91b7346365e8f4a20d74144255305b98ad09a7e Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:39:36 -0600
Subject: [PATCH 05/13] Fix typo
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 0479fb04be8bd..b6a6707f6dc83 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2240,7 +2240,7 @@ bool RewriteMFMAFormStage::initHeuristics(
// registers to be different types even though the actual operand is
// the same size.
const TargetRegisterClass *VUseRC = DAG.MRI.getRegClass(Src2->getReg());
- const TargetRegisterClass *AUseRC = SRI->getEquivalentVGPRClass(VUseRC);
+ const TargetRegisterClass *AUseRC = SRI->getEquivalentAGPRClass(VUseRC);
DAG.MRI.setRegClass(Src2->getReg(), AUseRC);
}
Changed = true;
>From 084444d13533e169bd94895f2fb813c728864991 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 18:09:47 -0600
Subject: [PATCH 06/13] Get rid of use of undef in test.
---
llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
index c8f444c175be8..38d3f566bf4de 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -2,8 +2,8 @@
target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
- at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] undef, align 16
- at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] undef, align 16
+ at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] poison, align 16
+ at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] poison, align 16
; Function Attrs: alwaysinline nofree norecurse nounwind
define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %0, ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %1, ptr addrspace(1) inreg noalias noundef nonnull align 16 %2) local_unnamed_addr #0 !reqd_work_group_size !2 {
>From b7db0b05b73ddc7e5071fe82febd09b467909248 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 11:01:23 -0600
Subject: [PATCH 07/13] Restore blank line.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index b6a6707f6dc83..277ac7d4ecf4c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,6 +2184,7 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
+
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
>From a4c78439026971fa0e36d42d1d14657574e706b9 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 11:45:20 -0600
Subject: [PATCH 08/13] Reduce test case size
---
.../AMDGPU/sched_mfma_rewrite_reject.ll | 2160 +++--------------
1 file changed, 301 insertions(+), 1859 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
index 38d3f566bf4de..8f431e2e9a46c 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_reject.ll
@@ -1,1867 +1,309 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O1 -amdgpu-disable-rewrite-mfma-form-sched-stage=false -verify-machineinstrs < %s | FileCheck %s
-target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
-
- at __shared_memory___0 = private unnamed_addr addrspace(3) global [128 x [132 x bfloat]] poison, align 16
- at __shared_memory__ = private unnamed_addr addrspace(3) global [128 x [68 x bfloat]] poison, align 16
-
-; Function Attrs: alwaysinline nofree norecurse nounwind
-define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %0, ptr addrspace(1) inreg noalias noundef nonnull readonly align 16 captures(none) %1, ptr addrspace(1) inreg noalias noundef nonnull align 16 %2) local_unnamed_addr #0 !reqd_work_group_size !2 {
+define amdgpu_kernel void @"fused_op_layout_cus_534d8e85a33330539c81dbeb009d324be60530f2_1x30x46x2016xbfloat16_1x30x46x2016xbfloat16_2016x1x1x2016xbfloat16$async_dispatch_0_matmul_like_2016x2016x1380_bf16xbf16xf32"(i32 %0, ptr addrspace(3) %1, ptr addrspace(3) %2, ptr addrspace(3) %3, ptr addrspace(3) %4, ptr addrspace(3) %5, ptr addrspace(3) %6, ptr addrspace(3) %7, ptr addrspace(3) %8, ptr addrspace(3) %9, ptr addrspace(3) %10, ptr addrspace(3) %11, ptr addrspace(3) %12, ptr addrspace(7) %invariant.gep, ptr addrspace(7) %gep, ptr addrspace(7) %gep106, ptr addrspace(7) %gep108, ptr addrspace(7) %gep112, ptr addrspace(7) %gep114, ptr addrspace(7) %gep116, ptr addrspace(7) %gep118, ptr addrspace(7) %gep120, ptr addrspace(7) %gep122, ptr addrspace(7) %gep124, ptr addrspace(3) %13, ptr addrspace(3) %14, ptr addrspace(3) %15, ptr addrspace(3) %16, ptr addrspace(3) %17, ptr addrspace(3) %18, ptr addrspace(3) %19, ptr addrspace(3) %20, ptr addrspace(3) %21, ptr addrspace(3) %22, ptr addrspace(3) %23, ptr addrspace(3) %24, ptr addrspace(3) %25, ptr addrspace(3) %26, ptr addrspace(3) %27, ptr addrspace(3) %28, ptr addrspace(3) %29, ptr addrspace(3) %30, ptr addrspace(3) %31, ptr addrspace(3) %32, ptr addrspace(3) %33, ptr addrspace(3) %34, ptr addrspace(3) %35, ptr addrspace(3) %36, ptr addrspace(3) %37, ptr addrspace(3) %38, ptr addrspace(3) %39, ptr addrspace(3) %40, ptr addrspace(3) %41, ptr addrspace(3) %42, ptr addrspace(3) %43, ptr addrspace(3) %44, ptr addrspace(3) %45, ptr addrspace(3) %46, ptr addrspace(3) %47, ptr addrspace(3) %48, ptr addrspace(3) %49, ptr addrspace(3) %50, ptr addrspace(3) %51, ptr addrspace(3) %52, ptr addrspace(3) %53, ptr addrspace(3) %54, ptr addrspace(3) %55, ptr addrspace(3) %56, ptr addrspace(3) %57, ptr addrspace(3) %58, ptr addrspace(3) %59, <4 x bfloat> %60, <4 x i16> %61, <4 x i16> %62, <4 x i16> %63, <4 x i16> %64, <4 x i16> %65, <8 x i1> %66, <4 x i16> %67, <4 x i16> %68, <4 x i16> %69, <4 x i16> %70, <8 x i1> %71) #0 {
; This should be rejected for MFMA rewrite by the scheduler so we should see
; no AGPR operands.
; CHECK-NOT: v_mfma_f32_16x16x16_bf16 a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}], a[{{[0-9:]+}}]
- %4 = tail call range(i32 0, 256) i32 @llvm.amdgcn.workitem.id.x()
- call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %0, i64 64) ]
- %5 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %0, i16 0, i64 5564160, i32 159744)
- call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %1, i64 64) ]
- %6 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %1, i16 0, i64 5564160, i32 159744)
- call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %2, i64 64) ]
- %7 = lshr i32 %4, 6
- %8 = and i32 %7, 1
- %9 = and i32 %4, 15
- %10 = lshr i32 %4, 2
- %11 = and i32 %10, 12
- %12 = lshr i32 %4, 3
- %13 = or disjoint i32 %4, 256
- %14 = lshr i32 %13, 3
- %15 = or disjoint i32 %4, 512
- %16 = lshr i32 %15, 3
- %17 = or disjoint i32 %4, 768
- %18 = lshr i32 %17, 3
- %19 = lshr i32 %4, 4
- %20 = lshr i32 %13, 4
- %21 = lshr i32 %15, 4
- %22 = lshr i32 %17, 4
- %23 = or disjoint i32 %19, 64
- %24 = or disjoint i32 %19, 80
- %25 = or disjoint i32 %19, 96
- %26 = or disjoint i32 %19, 112
- %27 = shl nuw nsw i32 %4, 3
- %28 = and i32 %27, 56
- %29 = and i32 %27, 120
- %30 = tail call range(i32 0, 512) i32 @llvm.amdgcn.workgroup.id.x()
- %31 = shl nuw nsw i32 %30, 6
- %32 = and i32 %31, 1984
- %33 = shl nuw nsw i32 %30, 2
- %34 = and i32 %33, 1920
- %35 = sub nuw nsw i32 2016, %34
- %36 = tail call i32 @llvm.umin.i32(i32 %35, i32 128)
- %37 = sub nuw nsw i32 2016, %32
- %38 = tail call i32 @llvm.umin.i32(i32 %37, i32 64)
- %39 = tail call i32 @llvm.umin.i32(i32 %38, i32 %28)
- %40 = sub nsw i32 %38, %39
- %41 = icmp sgt i32 %40, 7
- %42 = tail call i32 @llvm.umin.i32(i32 %36, i32 %29)
- %43 = sub nsw i32 %36, %42
- %44 = icmp sgt i32 %43, 7
- %45 = or disjoint i32 %39, %32
- %.idx = mul nuw nsw i32 %12, 4032
- %46 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx
- %47 = getelementptr bfloat, ptr addrspace(7) %46, i32 %45
- %48 = load <8 x bfloat>, ptr addrspace(7) %47, align 2
- %.idx4 = mul nuw nsw i32 %14, 4032
- %49 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx4
- %50 = getelementptr bfloat, ptr addrspace(7) %49, i32 %45
- %51 = load <8 x bfloat>, ptr addrspace(7) %50, align 2
- %.idx6 = mul nuw nsw i32 %16, 4032
- %52 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx6
- %53 = getelementptr bfloat, ptr addrspace(7) %52, i32 %45
- %54 = load <8 x bfloat>, ptr addrspace(7) %53, align 2
- %.idx8 = mul nuw nsw i32 %18, 4032
- %55 = getelementptr i8, ptr addrspace(7) %5, i32 %.idx8
- %56 = getelementptr bfloat, ptr addrspace(7) %55, i32 %45
- %57 = load <8 x bfloat>, ptr addrspace(7) %56, align 2
- %58 = or disjoint i32 %42, %34
- %.idx10 = mul nuw nsw i32 %19, 4032
- %59 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx10
- %60 = getelementptr bfloat, ptr addrspace(7) %59, i32 %58
- %61 = load <8 x bfloat>, ptr addrspace(7) %60, align 2
- %.idx12 = mul nuw nsw i32 %20, 4032
- %62 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx12
- %63 = getelementptr bfloat, ptr addrspace(7) %62, i32 %58
- %64 = load <8 x bfloat>, ptr addrspace(7) %63, align 2
- %.idx14 = mul nuw nsw i32 %21, 4032
- %65 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx14
- %66 = getelementptr bfloat, ptr addrspace(7) %65, i32 %58
- %67 = load <8 x bfloat>, ptr addrspace(7) %66, align 2
- %.idx16 = mul nuw nsw i32 %22, 4032
- %68 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx16
- %69 = getelementptr bfloat, ptr addrspace(7) %68, i32 %58
- %70 = load <8 x bfloat>, ptr addrspace(7) %69, align 2
- %.idx18 = mul nuw nsw i32 %23, 4032
- %71 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx18
- %72 = getelementptr bfloat, ptr addrspace(7) %71, i32 %58
- %73 = load <8 x bfloat>, ptr addrspace(7) %72, align 2
- %.idx20 = mul nuw nsw i32 %24, 4032
- %74 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx20
- %75 = getelementptr bfloat, ptr addrspace(7) %74, i32 %58
- %76 = load <8 x bfloat>, ptr addrspace(7) %75, align 2
- %.idx22 = mul nuw nsw i32 %25, 4032
- %77 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx22
- %78 = getelementptr bfloat, ptr addrspace(7) %77, i32 %58
- %79 = load <8 x bfloat>, ptr addrspace(7) %78, align 2
- %.idx24 = mul nuw nsw i32 %26, 4032
- %80 = getelementptr i8, ptr addrspace(7) %6, i32 %.idx24
- %81 = getelementptr bfloat, ptr addrspace(7) %80, i32 %58
- %82 = load <8 x bfloat>, ptr addrspace(7) %81, align 2
- %83 = select i1 %41, <8 x bfloat> %48, <8 x bfloat> zeroinitializer
- %.idx26 = mul nuw nsw i32 %12, 136
- %84 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx26
- %85 = getelementptr bfloat, ptr addrspace(3) %84, i32 %28
- store <8 x bfloat> %83, ptr addrspace(3) %85, align 8
- %86 = select i1 %41, <8 x bfloat> %51, <8 x bfloat> zeroinitializer
- %.idx28 = mul nuw nsw i32 %14, 136
- %87 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx28
- %88 = getelementptr bfloat, ptr addrspace(3) %87, i32 %28
- store <8 x bfloat> %86, ptr addrspace(3) %88, align 8
- %89 = select i1 %41, <8 x bfloat> %54, <8 x bfloat> zeroinitializer
- %.idx30 = mul nuw nsw i32 %16, 136
- %90 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx30
- %91 = getelementptr bfloat, ptr addrspace(3) %90, i32 %28
- store <8 x bfloat> %89, ptr addrspace(3) %91, align 8
- %92 = select i1 %41, <8 x bfloat> %57, <8 x bfloat> zeroinitializer
- %.idx32 = mul nuw nsw i32 %18, 136
- %93 = getelementptr i8, ptr addrspace(3) @__shared_memory__, i32 %.idx32
- %94 = getelementptr bfloat, ptr addrspace(3) %93, i32 %28
- store <8 x bfloat> %92, ptr addrspace(3) %94, align 8
- %95 = select i1 %44, <8 x bfloat> %61, <8 x bfloat> zeroinitializer
- %.idx34 = mul nuw nsw i32 %19, 264
- %96 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx34
- %97 = getelementptr bfloat, ptr addrspace(3) %96, i32 %29
- store <8 x bfloat> %95, ptr addrspace(3) %97, align 8
- %98 = select i1 %44, <8 x bfloat> %64, <8 x bfloat> zeroinitializer
- %.idx36 = mul nuw nsw i32 %20, 264
- %99 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx36
- %100 = getelementptr bfloat, ptr addrspace(3) %99, i32 %29
- store <8 x bfloat> %98, ptr addrspace(3) %100, align 8
- %101 = select i1 %44, <8 x bfloat> %67, <8 x bfloat> zeroinitializer
- %.idx38 = mul nuw nsw i32 %21, 264
- %102 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx38
- %103 = getelementptr bfloat, ptr addrspace(3) %102, i32 %29
- store <8 x bfloat> %101, ptr addrspace(3) %103, align 8
- %104 = select i1 %44, <8 x bfloat> %70, <8 x bfloat> zeroinitializer
- %.idx40 = mul nuw nsw i32 %22, 264
- %105 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx40
- %106 = getelementptr bfloat, ptr addrspace(3) %105, i32 %29
- store <8 x bfloat> %104, ptr addrspace(3) %106, align 8
- %107 = select i1 %44, <8 x bfloat> %73, <8 x bfloat> zeroinitializer
- %.idx42 = mul nuw nsw i32 %23, 264
- %108 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx42
- %109 = getelementptr bfloat, ptr addrspace(3) %108, i32 %29
- store <8 x bfloat> %107, ptr addrspace(3) %109, align 8
- %110 = select i1 %44, <8 x bfloat> %76, <8 x bfloat> zeroinitializer
- %.idx44 = mul nuw nsw i32 %24, 264
- %111 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx44
- %112 = getelementptr bfloat, ptr addrspace(3) %111, i32 %29
- store <8 x bfloat> %110, ptr addrspace(3) %112, align 8
- %113 = select i1 %44, <8 x bfloat> %79, <8 x bfloat> zeroinitializer
- %.idx46 = mul nuw nsw i32 %25, 264
- %114 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx46
- %115 = getelementptr bfloat, ptr addrspace(3) %114, i32 %29
- store <8 x bfloat> %113, ptr addrspace(3) %115, align 8
- %116 = select i1 %44, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
- %.idx48 = mul nuw nsw i32 %26, 264
- %117 = getelementptr i8, ptr addrspace(3) @__shared_memory___0, i32 %.idx48
- %118 = getelementptr bfloat, ptr addrspace(3) %117, i32 %29
- store <8 x bfloat> %116, ptr addrspace(3) %118, align 8
- %119 = shl nuw nsw i32 %8, 5
- %120 = or disjoint i32 %119, %9
- %121 = or disjoint i32 %11, 1
- %122 = or disjoint i32 %11, 2
- %123 = or disjoint i32 %11, 3
- %124 = or disjoint i32 %11, 16
- %125 = or disjoint i32 %11, 17
- %126 = or disjoint i32 %11, 18
- %127 = or disjoint i32 %11, 19
- %128 = or disjoint i32 %11, 32
- %129 = or disjoint i32 %11, 33
- %130 = or disjoint i32 %11, 34
- %131 = or disjoint i32 %11, 35
- %132 = or disjoint i32 %11, 48
- %133 = or disjoint i32 %11, 49
- %134 = or disjoint i32 %11, 50
- %135 = or i32 %10, 51
- %136 = or disjoint i32 %11, 64
- %137 = or disjoint i32 %11, 65
- %138 = or disjoint i32 %11, 66
- %139 = or disjoint i32 %11, 67
- %140 = or disjoint i32 %11, 80
- %141 = or disjoint i32 %11, 81
- %142 = or disjoint i32 %11, 82
- %143 = or disjoint i32 %11, 83
- %144 = or disjoint i32 %11, 96
- %145 = or disjoint i32 %11, 97
- %146 = or disjoint i32 %11, 98
- %147 = or disjoint i32 %11, 99
- %148 = or disjoint i32 %11, 112
- %149 = or disjoint i32 %11, 113
- %150 = or disjoint i32 %11, 114
- %151 = or i32 %10, 115
- %152 = lshr i32 %4, 1
- %153 = and i32 %152, 64
- %154 = or disjoint i32 %153, %9
- %invariant.gep = getelementptr bfloat, ptr addrspace(7) %5, i32 %45
- %invariant.gep111 = getelementptr bfloat, ptr addrspace(7) %6, i32 %58
- %155 = mul nuw nsw i32 %11, 68
- %156 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
- %157 = getelementptr bfloat, ptr addrspace(3) %156, i32 %155
- %158 = getelementptr bfloat, ptr addrspace(3) @__shared_memory__, i32 %120
- %159 = getelementptr i8, ptr addrspace(3) %158, i32 32
- %160 = getelementptr bfloat, ptr addrspace(3) %159, i32 %155
- %161 = mul nuw nsw i32 %121, 68
- %162 = getelementptr bfloat, ptr addrspace(3) %156, i32 %161
- %163 = getelementptr bfloat, ptr addrspace(3) %159, i32 %161
- %164 = mul nuw nsw i32 %122, 68
- %165 = getelementptr bfloat, ptr addrspace(3) %156, i32 %164
- %166 = getelementptr bfloat, ptr addrspace(3) %159, i32 %164
- %167 = mul nuw nsw i32 %123, 68
- %168 = getelementptr bfloat, ptr addrspace(3) %156, i32 %167
- %169 = getelementptr bfloat, ptr addrspace(3) %159, i32 %167
- %170 = mul nuw nsw i32 %124, 68
- %171 = getelementptr bfloat, ptr addrspace(3) %156, i32 %170
- %172 = getelementptr bfloat, ptr addrspace(3) %159, i32 %170
- %173 = mul nuw nsw i32 %125, 68
- %174 = getelementptr bfloat, ptr addrspace(3) %156, i32 %173
- %175 = getelementptr bfloat, ptr addrspace(3) %159, i32 %173
- %176 = mul nuw nsw i32 %126, 68
- %177 = getelementptr bfloat, ptr addrspace(3) %156, i32 %176
- %178 = getelementptr bfloat, ptr addrspace(3) %159, i32 %176
- %179 = mul nuw nsw i32 %127, 68
- %180 = getelementptr bfloat, ptr addrspace(3) %156, i32 %179
- %181 = getelementptr bfloat, ptr addrspace(3) %159, i32 %179
- %182 = mul nuw nsw i32 %128, 68
- %183 = getelementptr bfloat, ptr addrspace(3) %156, i32 %182
- %184 = getelementptr bfloat, ptr addrspace(3) %159, i32 %182
- %185 = mul nuw nsw i32 %129, 68
- %186 = getelementptr bfloat, ptr addrspace(3) %156, i32 %185
- %187 = getelementptr bfloat, ptr addrspace(3) %159, i32 %185
- %188 = mul nuw nsw i32 %130, 68
- %189 = getelementptr bfloat, ptr addrspace(3) %156, i32 %188
- %190 = getelementptr bfloat, ptr addrspace(3) %159, i32 %188
- %191 = mul nuw nsw i32 %131, 68
- %192 = getelementptr bfloat, ptr addrspace(3) %156, i32 %191
- %193 = getelementptr bfloat, ptr addrspace(3) %159, i32 %191
- %194 = mul nuw nsw i32 %132, 68
- %195 = getelementptr bfloat, ptr addrspace(3) %156, i32 %194
- %196 = getelementptr bfloat, ptr addrspace(3) %159, i32 %194
- %197 = mul nuw nsw i32 %133, 68
- %198 = getelementptr bfloat, ptr addrspace(3) %156, i32 %197
- %199 = getelementptr bfloat, ptr addrspace(3) %159, i32 %197
- %200 = mul nuw nsw i32 %134, 68
- %201 = getelementptr bfloat, ptr addrspace(3) %156, i32 %200
- %202 = getelementptr bfloat, ptr addrspace(3) %159, i32 %200
- %203 = mul nuw nsw i32 %135, 68
- %204 = getelementptr bfloat, ptr addrspace(3) %156, i32 %203
- %205 = getelementptr bfloat, ptr addrspace(3) %159, i32 %203
- %206 = mul nuw nsw i32 %136, 68
- %207 = getelementptr bfloat, ptr addrspace(3) %156, i32 %206
- %208 = getelementptr bfloat, ptr addrspace(3) %159, i32 %206
- %209 = mul nuw nsw i32 %137, 68
- %210 = getelementptr bfloat, ptr addrspace(3) %156, i32 %209
- %211 = getelementptr bfloat, ptr addrspace(3) %159, i32 %209
- %212 = mul nuw nsw i32 %138, 68
- %213 = getelementptr bfloat, ptr addrspace(3) %156, i32 %212
- %214 = getelementptr bfloat, ptr addrspace(3) %159, i32 %212
- %215 = mul nuw nsw i32 %139, 68
- %216 = getelementptr bfloat, ptr addrspace(3) %156, i32 %215
- %217 = getelementptr bfloat, ptr addrspace(3) %159, i32 %215
- %218 = mul nuw nsw i32 %140, 68
- %219 = getelementptr bfloat, ptr addrspace(3) %156, i32 %218
- %220 = getelementptr bfloat, ptr addrspace(3) %159, i32 %218
- %221 = mul nuw nsw i32 %141, 68
- %222 = getelementptr bfloat, ptr addrspace(3) %156, i32 %221
- %223 = getelementptr bfloat, ptr addrspace(3) %159, i32 %221
- %224 = mul nuw nsw i32 %142, 68
- %225 = getelementptr bfloat, ptr addrspace(3) %156, i32 %224
- %226 = getelementptr bfloat, ptr addrspace(3) %159, i32 %224
- %227 = mul nuw nsw i32 %143, 68
- %228 = getelementptr bfloat, ptr addrspace(3) %156, i32 %227
- %229 = getelementptr bfloat, ptr addrspace(3) %159, i32 %227
- %230 = mul nuw nsw i32 %144, 68
- %231 = getelementptr bfloat, ptr addrspace(3) %156, i32 %230
- %232 = getelementptr bfloat, ptr addrspace(3) %159, i32 %230
- %233 = mul nuw nsw i32 %145, 68
- %234 = getelementptr bfloat, ptr addrspace(3) %156, i32 %233
- %235 = getelementptr bfloat, ptr addrspace(3) %159, i32 %233
- %236 = mul nuw nsw i32 %146, 68
- %237 = getelementptr bfloat, ptr addrspace(3) %156, i32 %236
- %238 = getelementptr bfloat, ptr addrspace(3) %159, i32 %236
- %239 = mul nuw nsw i32 %147, 68
- %240 = getelementptr bfloat, ptr addrspace(3) %156, i32 %239
- %241 = getelementptr bfloat, ptr addrspace(3) %159, i32 %239
- %242 = mul nuw nsw i32 %148, 68
- %243 = getelementptr bfloat, ptr addrspace(3) %156, i32 %242
- %244 = getelementptr bfloat, ptr addrspace(3) %159, i32 %242
- %245 = mul nuw nsw i32 %149, 68
- %246 = getelementptr bfloat, ptr addrspace(3) %156, i32 %245
- %247 = getelementptr bfloat, ptr addrspace(3) %159, i32 %245
- %248 = mul nuw nsw i32 %150, 68
- %249 = getelementptr bfloat, ptr addrspace(3) %156, i32 %248
- %250 = getelementptr bfloat, ptr addrspace(3) %159, i32 %248
- %251 = mul nuw nsw i32 %151, 68
- %252 = getelementptr bfloat, ptr addrspace(3) %156, i32 %251
- %253 = getelementptr bfloat, ptr addrspace(3) %159, i32 %251
- %narrow = mul nuw nsw i32 %11, 132
- %254 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %255 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow
- %256 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %257 = getelementptr i8, ptr addrspace(3) %256, i32 32
- %258 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow
- %259 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %260 = getelementptr i8, ptr addrspace(3) %259, i32 64
- %261 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow
- %262 = getelementptr bfloat, ptr addrspace(3) @__shared_memory___0, i32 %154
- %263 = getelementptr i8, ptr addrspace(3) %262, i32 96
- %264 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow
- %narrow74 = mul nuw nsw i32 %121, 132
- %265 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow74
- %266 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow74
- %267 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow74
- %268 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow74
- %narrow75 = mul nuw nsw i32 %122, 132
- %269 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow75
- %270 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow75
- %271 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow75
- %272 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow75
- %narrow76 = mul nuw nsw i32 %123, 132
- %273 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow76
- %274 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow76
- %275 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow76
- %276 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow76
- %narrow77 = mul nuw nsw i32 %124, 132
- %277 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow77
- %278 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow77
- %279 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow77
- %280 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow77
- %narrow78 = mul nuw nsw i32 %125, 132
- %281 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow78
- %282 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow78
- %283 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow78
- %284 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow78
- %narrow79 = mul nuw nsw i32 %126, 132
- %285 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow79
- %286 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow79
- %287 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow79
- %288 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow79
- %narrow80 = mul nuw nsw i32 %127, 132
- %289 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow80
- %290 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow80
- %291 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow80
- %292 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow80
- %narrow81 = mul nuw nsw i32 %128, 132
- %293 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow81
- %294 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow81
- %295 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow81
- %296 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow81
- %narrow82 = mul nuw nsw i32 %129, 132
- %297 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow82
- %298 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow82
- %299 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow82
- %300 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow82
- %narrow83 = mul nuw nsw i32 %130, 132
- %301 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow83
- %302 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow83
- %303 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow83
- %304 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow83
- %narrow84 = mul nuw nsw i32 %131, 132
- %305 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow84
- %306 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow84
- %307 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow84
- %308 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow84
- %narrow85 = mul nuw nsw i32 %132, 132
- %309 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow85
- %310 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow85
- %311 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow85
- %312 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow85
- %narrow86 = mul nuw nsw i32 %133, 132
- %313 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow86
- %314 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow86
- %315 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow86
- %316 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow86
- %narrow87 = mul nuw nsw i32 %134, 132
- %317 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow87
- %318 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow87
- %319 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow87
- %320 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow87
- %narrow88 = mul nuw nsw i32 %135, 132
- %321 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow88
- %322 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow88
- %323 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow88
- %324 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow88
- %narrow89 = mul nuw nsw i32 %136, 132
- %325 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow89
- %326 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow89
- %327 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow89
- %328 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow89
- %narrow90 = mul nuw nsw i32 %137, 132
- %329 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow90
- %330 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow90
- %331 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow90
- %332 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow90
- %narrow91 = mul nuw nsw i32 %138, 132
- %333 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow91
- %334 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow91
- %335 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow91
- %336 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow91
- %narrow92 = mul nuw nsw i32 %139, 132
- %337 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow92
- %338 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow92
- %339 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow92
- %340 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow92
- %narrow93 = mul nuw nsw i32 %140, 132
- %341 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow93
- %342 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow93
- %343 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow93
- %344 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow93
- %narrow94 = mul nuw nsw i32 %141, 132
- %345 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow94
- %346 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow94
- %347 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow94
- %348 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow94
- %narrow95 = mul nuw nsw i32 %142, 132
- %349 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow95
- %350 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow95
- %351 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow95
- %352 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow95
- %narrow96 = mul nuw nsw i32 %143, 132
- %353 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow96
- %354 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow96
- %355 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow96
- %356 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow96
- %narrow97 = mul nuw nsw i32 %144, 132
- %357 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow97
- %358 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow97
- %359 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow97
- %360 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow97
- %narrow98 = mul nuw nsw i32 %145, 132
- %361 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow98
- %362 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow98
- %363 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow98
- %364 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow98
- %narrow99 = mul nuw nsw i32 %146, 132
- %365 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow99
- %366 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow99
- %367 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow99
- %368 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow99
- %narrow100 = mul nuw nsw i32 %147, 132
- %369 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow100
- %370 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow100
- %371 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow100
- %372 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow100
- %narrow101 = mul nuw nsw i32 %148, 132
- %373 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow101
- %374 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow101
- %375 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow101
- %376 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow101
- %narrow102 = mul nuw nsw i32 %149, 132
- %377 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow102
- %378 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow102
- %379 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow102
- %380 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow102
- %narrow103 = mul nuw nsw i32 %150, 132
- %381 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow103
- %382 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow103
- %383 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow103
- %384 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow103
- %narrow104 = mul nuw nsw i32 %151, 132
- %385 = getelementptr bfloat, ptr addrspace(3) %254, i32 %narrow104
- %386 = getelementptr bfloat, ptr addrspace(3) %257, i32 %narrow104
- %387 = getelementptr bfloat, ptr addrspace(3) %260, i32 %narrow104
- %388 = getelementptr bfloat, ptr addrspace(3) %263, i32 %narrow104
- br label %389
-
-389: ; preds = %3, %389
- %390 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %3 ], [ %1015, %389 ]
- %391 = phi i32 [ 0, %3 ], [ %392, %389 ]
- %392 = add nuw nsw i32 %391, 8
- %393 = shl nuw nsw i32 %392, 4
- %394 = sub nuw nsw i32 1380, %393
- %395 = tail call i32 @llvm.umin.i32(i32 %394, i32 128)
- %396 = tail call i32 @llvm.umin.i32(i32 %395, i32 %12)
- %397 = sub nsw i32 %395, %396
- %398 = tail call i32 @llvm.smin.i32(i32 %397, i32 1)
- %399 = or disjoint i32 %396, %393
- %.idx50 = mul nuw nsw i32 %399, 4032
- %gep = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx50
- %400 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
- %401 = tail call i32 @llvm.umin.i32(i32 %395, i32 %14)
- %402 = sub nsw i32 %395, %401
- %403 = tail call i32 @llvm.smin.i32(i32 %402, i32 1)
- %404 = or disjoint i32 %401, %393
- %.idx52 = mul nuw nsw i32 %404, 4032
- %gep106 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx52
- %405 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
- %406 = tail call i32 @llvm.umin.i32(i32 %395, i32 %16)
- %407 = sub nsw i32 %395, %406
- %408 = tail call i32 @llvm.smin.i32(i32 %407, i32 1)
- %409 = or disjoint i32 %406, %393
- %.idx54 = mul nuw nsw i32 %409, 4032
- %gep108 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx54
- %410 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
- %411 = tail call i32 @llvm.umin.i32(i32 %395, i32 %18)
- %412 = sub nsw i32 %395, %411
- %413 = tail call i32 @llvm.smin.i32(i32 %412, i32 1)
- %414 = or disjoint i32 %411, %393
- %.idx56 = mul nuw nsw i32 %414, 4032
- %gep110 = getelementptr i8, ptr addrspace(7) %invariant.gep, i32 %.idx56
- %415 = load <8 x bfloat>, ptr addrspace(7) %gep110, align 2
- %416 = tail call i32 @llvm.umin.i32(i32 %395, i32 %19)
- %417 = sub nsw i32 %395, %416
- %418 = tail call i32 @llvm.smin.i32(i32 %417, i32 1)
- %419 = or disjoint i32 %416, %393
- %.idx58 = mul nuw nsw i32 %419, 4032
- %gep112 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx58
- %420 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
- %421 = tail call i32 @llvm.umin.i32(i32 %395, i32 %20)
- %422 = sub nsw i32 %395, %421
- %423 = tail call i32 @llvm.smin.i32(i32 %422, i32 1)
- %424 = or disjoint i32 %421, %393
- %.idx60 = mul nuw nsw i32 %424, 4032
- %gep114 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx60
- %425 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
- %426 = tail call i32 @llvm.umin.i32(i32 %395, i32 %21)
- %427 = sub nsw i32 %395, %426
- %428 = tail call i32 @llvm.smin.i32(i32 %427, i32 1)
- %429 = or disjoint i32 %426, %393
- %.idx62 = mul nuw nsw i32 %429, 4032
- %gep116 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx62
- %430 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
- %431 = tail call i32 @llvm.umin.i32(i32 %395, i32 %22)
- %432 = sub nsw i32 %395, %431
- %433 = tail call i32 @llvm.smin.i32(i32 %432, i32 1)
- %434 = or disjoint i32 %431, %393
- %.idx64 = mul nuw nsw i32 %434, 4032
- %gep118 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx64
- %435 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
- %436 = tail call i32 @llvm.umin.i32(i32 %395, i32 %23)
- %437 = sub nsw i32 %395, %436
- %438 = tail call i32 @llvm.smin.i32(i32 %437, i32 1)
- %439 = or disjoint i32 %436, %393
- %.idx66 = mul nuw nsw i32 %439, 4032
- %gep120 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx66
- %440 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
- %441 = tail call i32 @llvm.umin.i32(i32 %395, i32 %24)
- %442 = sub nsw i32 %395, %441
- %443 = tail call i32 @llvm.smin.i32(i32 %442, i32 1)
- %444 = or disjoint i32 %441, %393
- %.idx68 = mul nuw nsw i32 %444, 4032
- %gep122 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx68
- %445 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
- %446 = tail call i32 @llvm.umin.i32(i32 %395, i32 %25)
- %447 = sub nsw i32 %395, %446
- %448 = tail call i32 @llvm.smin.i32(i32 %447, i32 1)
- %449 = or disjoint i32 %446, %393
- %.idx70 = mul nuw nsw i32 %449, 4032
- %gep124 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx70
- %450 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
- %451 = tail call i32 @llvm.umin.i32(i32 %395, i32 %26)
- %452 = sub nsw i32 %395, %451
- %453 = tail call i32 @llvm.smin.i32(i32 %452, i32 1)
- %454 = or disjoint i32 %451, %393
- %.idx72 = mul nuw nsw i32 %454, 4032
- %gep126 = getelementptr i8, ptr addrspace(7) %invariant.gep111, i32 %.idx72
- %455 = load <8 x bfloat>, ptr addrspace(7) %gep126, align 2
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
- %456 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
- %457 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
- %458 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
- %459 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
- %460 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
- %461 = shufflevector <1 x bfloat> %460, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %462 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
- %463 = shufflevector <1 x bfloat> %462, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %464 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
- %465 = shufflevector <1 x bfloat> %464, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %466 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
- %467 = shufflevector <1 x bfloat> %466, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %468 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
- %469 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
- %470 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
- %471 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
- %472 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
- %473 = shufflevector <1 x bfloat> %472, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %474 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
- %475 = shufflevector <1 x bfloat> %474, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %476 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
- %477 = shufflevector <1 x bfloat> %476, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %478 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
- %479 = shufflevector <1 x bfloat> %478, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %480 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
- %481 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
- %482 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
- %483 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
- %484 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
- %485 = shufflevector <1 x bfloat> %484, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %486 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
- %487 = shufflevector <1 x bfloat> %486, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %488 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
- %489 = shufflevector <1 x bfloat> %488, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %490 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
- %491 = shufflevector <1 x bfloat> %490, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %492 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
- %493 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
- %494 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
- %495 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
- %496 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
- %497 = shufflevector <1 x bfloat> %496, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %498 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
- %499 = shufflevector <1 x bfloat> %498, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %500 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
- %501 = shufflevector <1 x bfloat> %500, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %502 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
- %503 = shufflevector <1 x bfloat> %502, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %504 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
- %505 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
- %506 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
- %507 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
- %508 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
- %509 = shufflevector <1 x bfloat> %508, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %510 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
- %511 = shufflevector <1 x bfloat> %510, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %512 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
- %513 = shufflevector <1 x bfloat> %512, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %514 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
- %515 = shufflevector <1 x bfloat> %514, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %516 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
- %517 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
- %518 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
- %519 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
- %520 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
- %521 = shufflevector <1 x bfloat> %520, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %522 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
- %523 = shufflevector <1 x bfloat> %522, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %524 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
- %525 = shufflevector <1 x bfloat> %524, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %526 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
- %527 = shufflevector <1 x bfloat> %526, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %528 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
- %529 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
- %530 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
- %531 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
- %532 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
- %533 = shufflevector <1 x bfloat> %532, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %534 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
- %535 = shufflevector <1 x bfloat> %534, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %536 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
- %537 = shufflevector <1 x bfloat> %536, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %538 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
- %539 = shufflevector <1 x bfloat> %538, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %540 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
- %541 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
- %542 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
- %543 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
- %544 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
- %545 = shufflevector <1 x bfloat> %544, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %546 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
- %547 = shufflevector <1 x bfloat> %546, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %548 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
- %549 = shufflevector <1 x bfloat> %548, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %550 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
- %551 = shufflevector <1 x bfloat> %550, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %552 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
- %553 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
- %554 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
- %555 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
- %556 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
- %557 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
- %558 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
- %559 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
- %560 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
- %561 = shufflevector <1 x bfloat> %560, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %562 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
- %563 = shufflevector <1 x bfloat> %562, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %564 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
- %565 = shufflevector <1 x bfloat> %564, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %566 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
- %567 = shufflevector <1 x bfloat> %566, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %568 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
- %569 = shufflevector <1 x bfloat> %568, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %570 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
- %571 = shufflevector <1 x bfloat> %570, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %572 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
- %573 = shufflevector <1 x bfloat> %572, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %574 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
- %575 = shufflevector <1 x bfloat> %574, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %576 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
- %577 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
- %578 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
- %579 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
- %580 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
- %581 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
- %582 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
- %583 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
- %584 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
- %585 = shufflevector <1 x bfloat> %584, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %586 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
- %587 = shufflevector <1 x bfloat> %586, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %588 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
- %589 = shufflevector <1 x bfloat> %588, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %590 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
- %591 = shufflevector <1 x bfloat> %590, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %592 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
- %593 = shufflevector <1 x bfloat> %592, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %594 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
- %595 = shufflevector <1 x bfloat> %594, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %596 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
- %597 = shufflevector <1 x bfloat> %596, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %598 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
- %599 = shufflevector <1 x bfloat> %598, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %600 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
- %601 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
- %602 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
- %603 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
- %604 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
- %605 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
- %606 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
- %607 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
- %608 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
- %609 = shufflevector <1 x bfloat> %608, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %610 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
- %611 = shufflevector <1 x bfloat> %610, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %612 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
- %613 = shufflevector <1 x bfloat> %612, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %614 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
- %615 = shufflevector <1 x bfloat> %614, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %616 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
- %617 = shufflevector <1 x bfloat> %616, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %618 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
- %619 = shufflevector <1 x bfloat> %618, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %620 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
- %621 = shufflevector <1 x bfloat> %620, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %622 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
- %623 = shufflevector <1 x bfloat> %622, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %624 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
- %625 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
- %626 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
- %627 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
- %628 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
- %629 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
- %630 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
- %631 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
- %632 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
- %633 = shufflevector <1 x bfloat> %632, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %634 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
- %635 = shufflevector <1 x bfloat> %634, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %636 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
- %637 = shufflevector <1 x bfloat> %636, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %638 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
- %639 = shufflevector <1 x bfloat> %638, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %640 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
- %641 = shufflevector <1 x bfloat> %640, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %642 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
- %643 = shufflevector <1 x bfloat> %642, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %644 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
- %645 = shufflevector <1 x bfloat> %644, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %646 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
- %647 = shufflevector <1 x bfloat> %646, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %648 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
- %649 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
- %650 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
- %651 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
- %652 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
- %653 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
- %654 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
- %655 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
- %656 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
- %657 = shufflevector <1 x bfloat> %656, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %658 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
- %659 = shufflevector <1 x bfloat> %658, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %660 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
- %661 = shufflevector <1 x bfloat> %660, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %662 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
- %663 = shufflevector <1 x bfloat> %662, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %664 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
- %665 = shufflevector <1 x bfloat> %664, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %666 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
- %667 = shufflevector <1 x bfloat> %666, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %668 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
- %669 = shufflevector <1 x bfloat> %668, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %670 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
- %671 = shufflevector <1 x bfloat> %670, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %672 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
- %673 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
- %674 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
- %675 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
- %676 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
- %677 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
- %678 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
- %679 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
- %680 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
- %681 = shufflevector <1 x bfloat> %680, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %682 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
- %683 = shufflevector <1 x bfloat> %682, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %684 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
- %685 = shufflevector <1 x bfloat> %684, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %686 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
- %687 = shufflevector <1 x bfloat> %686, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %688 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
- %689 = shufflevector <1 x bfloat> %688, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %690 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
- %691 = shufflevector <1 x bfloat> %690, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %692 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
- %693 = shufflevector <1 x bfloat> %692, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %694 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
- %695 = shufflevector <1 x bfloat> %694, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %696 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
- %697 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
- %698 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
- %699 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
- %700 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
- %701 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
- %702 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
- %703 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
- %704 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
- %705 = shufflevector <1 x bfloat> %704, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %706 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
- %707 = shufflevector <1 x bfloat> %706, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %708 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
- %709 = shufflevector <1 x bfloat> %708, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %710 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
- %711 = shufflevector <1 x bfloat> %710, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %712 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
- %713 = shufflevector <1 x bfloat> %712, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %714 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
- %715 = shufflevector <1 x bfloat> %714, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %716 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
- %717 = shufflevector <1 x bfloat> %716, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %718 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
- %719 = shufflevector <1 x bfloat> %718, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %720 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
- %721 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
- %722 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
- %723 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
- %724 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
- %725 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
- %726 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
- %727 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
- %728 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
- %729 = shufflevector <1 x bfloat> %728, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %730 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
- %731 = shufflevector <1 x bfloat> %730, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %732 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
- %733 = shufflevector <1 x bfloat> %732, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %734 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
- %735 = shufflevector <1 x bfloat> %734, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %736 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
- %737 = shufflevector <1 x bfloat> %736, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %738 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
- %739 = shufflevector <1 x bfloat> %738, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %740 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
- %741 = shufflevector <1 x bfloat> %740, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %742 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
- %743 = shufflevector <1 x bfloat> %742, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %744 = shufflevector <1 x bfloat> %456, <1 x bfloat> %458, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %745 = shufflevector <4 x bfloat> %744, <4 x bfloat> %461, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %746 = shufflevector <4 x bfloat> %745, <4 x bfloat> %465, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %747 = shufflevector <1 x bfloat> %552, <1 x bfloat> %556, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %748 = shufflevector <4 x bfloat> %747, <4 x bfloat> %561, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %749 = shufflevector <4 x bfloat> %748, <4 x bfloat> %569, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %750 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 0, 0
- %751 = bitcast <4 x bfloat> %746 to <4 x i16>
- %752 = bitcast <4 x bfloat> %749 to <4 x i16>
- %753 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %752, <4 x float> %750, i32 0, i32 0, i32 0)
- %754 = shufflevector <1 x bfloat> %553, <1 x bfloat> %557, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %755 = shufflevector <4 x bfloat> %754, <4 x bfloat> %563, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %756 = shufflevector <4 x bfloat> %755, <4 x bfloat> %571, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %757 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 0, 0
- %758 = bitcast <4 x bfloat> %756 to <4 x i16>
- %759 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %758, <4 x float> %757, i32 0, i32 0, i32 0)
- %760 = shufflevector <1 x bfloat> %554, <1 x bfloat> %558, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %761 = shufflevector <4 x bfloat> %760, <4 x bfloat> %565, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %762 = shufflevector <4 x bfloat> %761, <4 x bfloat> %573, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %763 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 0, 0
- %764 = bitcast <4 x bfloat> %762 to <4 x i16>
- %765 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %764, <4 x float> %763, i32 0, i32 0, i32 0)
- %766 = shufflevector <1 x bfloat> %555, <1 x bfloat> %559, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %767 = shufflevector <4 x bfloat> %766, <4 x bfloat> %567, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %768 = shufflevector <4 x bfloat> %767, <4 x bfloat> %575, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %769 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 0, 0
- %770 = bitcast <4 x bfloat> %768 to <4 x i16>
- %771 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %751, <4 x i16> %770, <4 x float> %769, i32 0, i32 0, i32 0)
- %772 = shufflevector <1 x bfloat> %457, <1 x bfloat> %459, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %773 = shufflevector <4 x bfloat> %772, <4 x bfloat> %463, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %774 = shufflevector <4 x bfloat> %773, <4 x bfloat> %467, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %775 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 0, 1, 0
- %776 = bitcast <4 x bfloat> %774 to <4 x i16>
- %777 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %752, <4 x float> %775, i32 0, i32 0, i32 0)
- %778 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 1, 1, 0
- %779 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %758, <4 x float> %778, i32 0, i32 0, i32 0)
- %780 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 2, 1, 0
- %781 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %764, <4 x float> %780, i32 0, i32 0, i32 0)
- %782 = extractvalue [4 x [2 x [1 x <4 x float>]]] %390, 3, 1, 0
- %783 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %776, <4 x i16> %770, <4 x float> %782, i32 0, i32 0, i32 0)
- %784 = shufflevector <1 x bfloat> %468, <1 x bfloat> %470, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %785 = shufflevector <4 x bfloat> %784, <4 x bfloat> %473, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %786 = shufflevector <4 x bfloat> %785, <4 x bfloat> %477, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %787 = shufflevector <1 x bfloat> %576, <1 x bfloat> %580, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %788 = shufflevector <4 x bfloat> %787, <4 x bfloat> %585, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %789 = shufflevector <4 x bfloat> %788, <4 x bfloat> %593, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %790 = bitcast <4 x bfloat> %786 to <4 x i16>
- %791 = bitcast <4 x bfloat> %789 to <4 x i16>
- %792 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %791, <4 x float> %753, i32 0, i32 0, i32 0)
- %793 = shufflevector <1 x bfloat> %577, <1 x bfloat> %581, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %794 = shufflevector <4 x bfloat> %793, <4 x bfloat> %587, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %795 = shufflevector <4 x bfloat> %794, <4 x bfloat> %595, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %796 = bitcast <4 x bfloat> %795 to <4 x i16>
- %797 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %796, <4 x float> %759, i32 0, i32 0, i32 0)
- %798 = shufflevector <1 x bfloat> %578, <1 x bfloat> %582, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %799 = shufflevector <4 x bfloat> %798, <4 x bfloat> %589, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %800 = shufflevector <4 x bfloat> %799, <4 x bfloat> %597, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %801 = bitcast <4 x bfloat> %800 to <4 x i16>
- %802 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %801, <4 x float> %765, i32 0, i32 0, i32 0)
- %803 = shufflevector <1 x bfloat> %579, <1 x bfloat> %583, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %804 = shufflevector <4 x bfloat> %803, <4 x bfloat> %591, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %805 = shufflevector <4 x bfloat> %804, <4 x bfloat> %599, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %806 = bitcast <4 x bfloat> %805 to <4 x i16>
- %807 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %790, <4 x i16> %806, <4 x float> %771, i32 0, i32 0, i32 0)
- %808 = shufflevector <1 x bfloat> %469, <1 x bfloat> %471, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %809 = shufflevector <4 x bfloat> %808, <4 x bfloat> %475, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %810 = shufflevector <4 x bfloat> %809, <4 x bfloat> %479, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %811 = bitcast <4 x bfloat> %810 to <4 x i16>
- %812 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %791, <4 x float> %777, i32 0, i32 0, i32 0)
- %813 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %796, <4 x float> %779, i32 0, i32 0, i32 0)
- %814 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %801, <4 x float> %781, i32 0, i32 0, i32 0)
- %815 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %811, <4 x i16> %806, <4 x float> %783, i32 0, i32 0, i32 0)
- %816 = shufflevector <1 x bfloat> %480, <1 x bfloat> %482, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %817 = shufflevector <4 x bfloat> %816, <4 x bfloat> %485, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %818 = shufflevector <4 x bfloat> %817, <4 x bfloat> %489, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %819 = shufflevector <1 x bfloat> %600, <1 x bfloat> %604, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %820 = shufflevector <4 x bfloat> %819, <4 x bfloat> %609, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %821 = shufflevector <4 x bfloat> %820, <4 x bfloat> %617, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %822 = bitcast <4 x bfloat> %818 to <4 x i16>
- %823 = bitcast <4 x bfloat> %821 to <4 x i16>
- %824 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %823, <4 x float> %792, i32 0, i32 0, i32 0)
- %825 = shufflevector <1 x bfloat> %601, <1 x bfloat> %605, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %826 = shufflevector <4 x bfloat> %825, <4 x bfloat> %611, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %827 = shufflevector <4 x bfloat> %826, <4 x bfloat> %619, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %828 = bitcast <4 x bfloat> %827 to <4 x i16>
- %829 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %828, <4 x float> %797, i32 0, i32 0, i32 0)
- %830 = shufflevector <1 x bfloat> %602, <1 x bfloat> %606, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %831 = shufflevector <4 x bfloat> %830, <4 x bfloat> %613, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %832 = shufflevector <4 x bfloat> %831, <4 x bfloat> %621, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %833 = bitcast <4 x bfloat> %832 to <4 x i16>
- %834 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %833, <4 x float> %802, i32 0, i32 0, i32 0)
- %835 = shufflevector <1 x bfloat> %603, <1 x bfloat> %607, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %836 = shufflevector <4 x bfloat> %835, <4 x bfloat> %615, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %837 = shufflevector <4 x bfloat> %836, <4 x bfloat> %623, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %838 = bitcast <4 x bfloat> %837 to <4 x i16>
- %839 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %822, <4 x i16> %838, <4 x float> %807, i32 0, i32 0, i32 0)
- %840 = shufflevector <1 x bfloat> %481, <1 x bfloat> %483, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %841 = shufflevector <4 x bfloat> %840, <4 x bfloat> %487, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %842 = shufflevector <4 x bfloat> %841, <4 x bfloat> %491, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %843 = bitcast <4 x bfloat> %842 to <4 x i16>
- %844 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %823, <4 x float> %812, i32 0, i32 0, i32 0)
- %845 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %828, <4 x float> %813, i32 0, i32 0, i32 0)
- %846 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %833, <4 x float> %814, i32 0, i32 0, i32 0)
- %847 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %843, <4 x i16> %838, <4 x float> %815, i32 0, i32 0, i32 0)
- %848 = shufflevector <1 x bfloat> %492, <1 x bfloat> %494, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %849 = shufflevector <4 x bfloat> %848, <4 x bfloat> %497, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %850 = shufflevector <4 x bfloat> %849, <4 x bfloat> %501, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %851 = shufflevector <1 x bfloat> %624, <1 x bfloat> %628, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %852 = shufflevector <4 x bfloat> %851, <4 x bfloat> %633, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %853 = shufflevector <4 x bfloat> %852, <4 x bfloat> %641, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %854 = bitcast <4 x bfloat> %850 to <4 x i16>
- %855 = bitcast <4 x bfloat> %853 to <4 x i16>
- %856 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %855, <4 x float> %824, i32 0, i32 0, i32 0)
- %857 = shufflevector <1 x bfloat> %625, <1 x bfloat> %629, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %858 = shufflevector <4 x bfloat> %857, <4 x bfloat> %635, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %859 = shufflevector <4 x bfloat> %858, <4 x bfloat> %643, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %860 = bitcast <4 x bfloat> %859 to <4 x i16>
- %861 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %860, <4 x float> %829, i32 0, i32 0, i32 0)
- %862 = shufflevector <1 x bfloat> %626, <1 x bfloat> %630, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %863 = shufflevector <4 x bfloat> %862, <4 x bfloat> %637, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %864 = shufflevector <4 x bfloat> %863, <4 x bfloat> %645, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %865 = bitcast <4 x bfloat> %864 to <4 x i16>
- %866 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %865, <4 x float> %834, i32 0, i32 0, i32 0)
- %867 = shufflevector <1 x bfloat> %627, <1 x bfloat> %631, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %868 = shufflevector <4 x bfloat> %867, <4 x bfloat> %639, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %869 = shufflevector <4 x bfloat> %868, <4 x bfloat> %647, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %870 = bitcast <4 x bfloat> %869 to <4 x i16>
- %871 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %854, <4 x i16> %870, <4 x float> %839, i32 0, i32 0, i32 0)
- %872 = shufflevector <1 x bfloat> %493, <1 x bfloat> %495, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %873 = shufflevector <4 x bfloat> %872, <4 x bfloat> %499, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %874 = shufflevector <4 x bfloat> %873, <4 x bfloat> %503, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %875 = bitcast <4 x bfloat> %874 to <4 x i16>
- %876 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %855, <4 x float> %844, i32 0, i32 0, i32 0)
- %877 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %860, <4 x float> %845, i32 0, i32 0, i32 0)
- %878 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %865, <4 x float> %846, i32 0, i32 0, i32 0)
- %879 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %875, <4 x i16> %870, <4 x float> %847, i32 0, i32 0, i32 0)
- %880 = shufflevector <1 x bfloat> %504, <1 x bfloat> %506, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %881 = shufflevector <4 x bfloat> %880, <4 x bfloat> %509, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %882 = shufflevector <4 x bfloat> %881, <4 x bfloat> %513, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %883 = shufflevector <1 x bfloat> %648, <1 x bfloat> %652, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %884 = shufflevector <4 x bfloat> %883, <4 x bfloat> %657, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %885 = shufflevector <4 x bfloat> %884, <4 x bfloat> %665, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %886 = bitcast <4 x bfloat> %882 to <4 x i16>
- %887 = bitcast <4 x bfloat> %885 to <4 x i16>
- %888 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %887, <4 x float> %856, i32 0, i32 0, i32 0)
- %889 = shufflevector <1 x bfloat> %649, <1 x bfloat> %653, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %890 = shufflevector <4 x bfloat> %889, <4 x bfloat> %659, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %891 = shufflevector <4 x bfloat> %890, <4 x bfloat> %667, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %892 = bitcast <4 x bfloat> %891 to <4 x i16>
- %893 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %892, <4 x float> %861, i32 0, i32 0, i32 0)
- %894 = shufflevector <1 x bfloat> %650, <1 x bfloat> %654, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %895 = shufflevector <4 x bfloat> %894, <4 x bfloat> %661, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %896 = shufflevector <4 x bfloat> %895, <4 x bfloat> %669, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %897 = bitcast <4 x bfloat> %896 to <4 x i16>
- %898 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %897, <4 x float> %866, i32 0, i32 0, i32 0)
- %899 = shufflevector <1 x bfloat> %651, <1 x bfloat> %655, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %900 = shufflevector <4 x bfloat> %899, <4 x bfloat> %663, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %901 = shufflevector <4 x bfloat> %900, <4 x bfloat> %671, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %902 = bitcast <4 x bfloat> %901 to <4 x i16>
- %903 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %886, <4 x i16> %902, <4 x float> %871, i32 0, i32 0, i32 0)
- %904 = shufflevector <1 x bfloat> %505, <1 x bfloat> %507, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %905 = shufflevector <4 x bfloat> %904, <4 x bfloat> %511, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %906 = shufflevector <4 x bfloat> %905, <4 x bfloat> %515, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %907 = bitcast <4 x bfloat> %906 to <4 x i16>
- %908 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %887, <4 x float> %876, i32 0, i32 0, i32 0)
- %909 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %892, <4 x float> %877, i32 0, i32 0, i32 0)
- %910 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %897, <4 x float> %878, i32 0, i32 0, i32 0)
- %911 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %907, <4 x i16> %902, <4 x float> %879, i32 0, i32 0, i32 0)
- %912 = shufflevector <1 x bfloat> %516, <1 x bfloat> %518, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %913 = shufflevector <4 x bfloat> %912, <4 x bfloat> %521, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %914 = shufflevector <4 x bfloat> %913, <4 x bfloat> %525, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %915 = shufflevector <1 x bfloat> %672, <1 x bfloat> %676, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %916 = shufflevector <4 x bfloat> %915, <4 x bfloat> %681, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %917 = shufflevector <4 x bfloat> %916, <4 x bfloat> %689, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %918 = bitcast <4 x bfloat> %914 to <4 x i16>
- %919 = bitcast <4 x bfloat> %917 to <4 x i16>
- %920 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %919, <4 x float> %888, i32 0, i32 0, i32 0)
- %921 = shufflevector <1 x bfloat> %673, <1 x bfloat> %677, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %922 = shufflevector <4 x bfloat> %921, <4 x bfloat> %683, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %923 = shufflevector <4 x bfloat> %922, <4 x bfloat> %691, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %924 = bitcast <4 x bfloat> %923 to <4 x i16>
- %925 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %924, <4 x float> %893, i32 0, i32 0, i32 0)
- %926 = shufflevector <1 x bfloat> %674, <1 x bfloat> %678, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %927 = shufflevector <4 x bfloat> %926, <4 x bfloat> %685, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %928 = shufflevector <4 x bfloat> %927, <4 x bfloat> %693, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %929 = bitcast <4 x bfloat> %928 to <4 x i16>
- %930 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %929, <4 x float> %898, i32 0, i32 0, i32 0)
- %931 = shufflevector <1 x bfloat> %675, <1 x bfloat> %679, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %932 = shufflevector <4 x bfloat> %931, <4 x bfloat> %687, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %933 = shufflevector <4 x bfloat> %932, <4 x bfloat> %695, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %934 = bitcast <4 x bfloat> %933 to <4 x i16>
- %935 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %918, <4 x i16> %934, <4 x float> %903, i32 0, i32 0, i32 0)
- %936 = shufflevector <1 x bfloat> %517, <1 x bfloat> %519, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %937 = shufflevector <4 x bfloat> %936, <4 x bfloat> %523, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %938 = shufflevector <4 x bfloat> %937, <4 x bfloat> %527, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %939 = bitcast <4 x bfloat> %938 to <4 x i16>
- %940 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %919, <4 x float> %908, i32 0, i32 0, i32 0)
- %941 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %924, <4 x float> %909, i32 0, i32 0, i32 0)
- %942 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %929, <4 x float> %910, i32 0, i32 0, i32 0)
- %943 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %939, <4 x i16> %934, <4 x float> %911, i32 0, i32 0, i32 0)
- %944 = shufflevector <1 x bfloat> %528, <1 x bfloat> %530, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %945 = shufflevector <4 x bfloat> %944, <4 x bfloat> %533, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %946 = shufflevector <4 x bfloat> %945, <4 x bfloat> %537, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %947 = shufflevector <1 x bfloat> %696, <1 x bfloat> %700, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %948 = shufflevector <4 x bfloat> %947, <4 x bfloat> %705, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %949 = shufflevector <4 x bfloat> %948, <4 x bfloat> %713, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %950 = bitcast <4 x bfloat> %946 to <4 x i16>
- %951 = bitcast <4 x bfloat> %949 to <4 x i16>
- %952 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %951, <4 x float> %920, i32 0, i32 0, i32 0)
- %953 = shufflevector <1 x bfloat> %697, <1 x bfloat> %701, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %954 = shufflevector <4 x bfloat> %953, <4 x bfloat> %707, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %955 = shufflevector <4 x bfloat> %954, <4 x bfloat> %715, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %956 = bitcast <4 x bfloat> %955 to <4 x i16>
- %957 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %956, <4 x float> %925, i32 0, i32 0, i32 0)
- %958 = shufflevector <1 x bfloat> %698, <1 x bfloat> %702, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %959 = shufflevector <4 x bfloat> %958, <4 x bfloat> %709, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %960 = shufflevector <4 x bfloat> %959, <4 x bfloat> %717, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %961 = bitcast <4 x bfloat> %960 to <4 x i16>
- %962 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %961, <4 x float> %930, i32 0, i32 0, i32 0)
- %963 = shufflevector <1 x bfloat> %699, <1 x bfloat> %703, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %964 = shufflevector <4 x bfloat> %963, <4 x bfloat> %711, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %965 = shufflevector <4 x bfloat> %964, <4 x bfloat> %719, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %966 = bitcast <4 x bfloat> %965 to <4 x i16>
- %967 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %950, <4 x i16> %966, <4 x float> %935, i32 0, i32 0, i32 0)
- %968 = shufflevector <1 x bfloat> %529, <1 x bfloat> %531, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %969 = shufflevector <4 x bfloat> %968, <4 x bfloat> %535, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %970 = shufflevector <4 x bfloat> %969, <4 x bfloat> %539, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %971 = bitcast <4 x bfloat> %970 to <4 x i16>
- %972 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %951, <4 x float> %940, i32 0, i32 0, i32 0)
- %973 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %956, <4 x float> %941, i32 0, i32 0, i32 0)
- %974 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %961, <4 x float> %942, i32 0, i32 0, i32 0)
- %975 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %971, <4 x i16> %966, <4 x float> %943, i32 0, i32 0, i32 0)
- %976 = shufflevector <1 x bfloat> %540, <1 x bfloat> %542, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %977 = shufflevector <4 x bfloat> %976, <4 x bfloat> %545, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %978 = shufflevector <4 x bfloat> %977, <4 x bfloat> %549, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %979 = shufflevector <1 x bfloat> %720, <1 x bfloat> %724, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %980 = shufflevector <4 x bfloat> %979, <4 x bfloat> %729, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %981 = shufflevector <4 x bfloat> %980, <4 x bfloat> %737, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %982 = bitcast <4 x bfloat> %978 to <4 x i16>
- %983 = bitcast <4 x bfloat> %981 to <4 x i16>
- %984 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %983, <4 x float> %952, i32 0, i32 0, i32 0)
- %985 = shufflevector <1 x bfloat> %721, <1 x bfloat> %725, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %986 = shufflevector <4 x bfloat> %985, <4 x bfloat> %731, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %987 = shufflevector <4 x bfloat> %986, <4 x bfloat> %739, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %988 = bitcast <4 x bfloat> %987 to <4 x i16>
- %989 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %988, <4 x float> %957, i32 0, i32 0, i32 0)
- %990 = shufflevector <1 x bfloat> %722, <1 x bfloat> %726, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %991 = shufflevector <4 x bfloat> %990, <4 x bfloat> %733, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %992 = shufflevector <4 x bfloat> %991, <4 x bfloat> %741, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %993 = bitcast <4 x bfloat> %992 to <4 x i16>
- %994 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %993, <4 x float> %962, i32 0, i32 0, i32 0)
- %995 = shufflevector <1 x bfloat> %723, <1 x bfloat> %727, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %996 = shufflevector <4 x bfloat> %995, <4 x bfloat> %735, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %997 = shufflevector <4 x bfloat> %996, <4 x bfloat> %743, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %998 = bitcast <4 x bfloat> %997 to <4 x i16>
- %999 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %982, <4 x i16> %998, <4 x float> %967, i32 0, i32 0, i32 0)
- %1000 = shufflevector <1 x bfloat> %541, <1 x bfloat> %543, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1001 = shufflevector <4 x bfloat> %1000, <4 x bfloat> %547, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1002 = shufflevector <4 x bfloat> %1001, <4 x bfloat> %551, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1003 = bitcast <4 x bfloat> %1002 to <4 x i16>
- %1004 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %983, <4 x float> %972, i32 0, i32 0, i32 0)
- %1005 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %988, <4 x float> %973, i32 0, i32 0, i32 0)
- %1006 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %993, <4 x float> %974, i32 0, i32 0, i32 0)
- %1007 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1003, <4 x i16> %998, <4 x float> %975, i32 0, i32 0, i32 0)
- %1008 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %984, 0, 0, 0
- %1009 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1008, <4 x float> %989, 1, 0, 0
- %1010 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1009, <4 x float> %994, 2, 0, 0
- %1011 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1010, <4 x float> %999, 3, 0, 0
- %1012 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1011, <4 x float> %1004, 0, 1, 0
- %1013 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1012, <4 x float> %1005, 1, 1, 0
- %1014 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1013, <4 x float> %1006, 2, 1, 0
- %1015 = insertvalue [4 x [2 x [1 x <4 x float>]]] %1014, <4 x float> %1007, 3, 1, 0
- %1016 = trunc i32 %398 to i1
- %1017 = and i1 %41, %1016
- %1018 = select i1 %1017, <8 x bfloat> %400, <8 x bfloat> zeroinitializer
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
+ br label %73
+73: ; preds = %73, %72
+ %74 = phi <1 x bfloat> [ zeroinitializer, %72 ], [ %339, %73 ]
+ %75 = phi [4 x [2 x [1 x <4 x float>]]] [ zeroinitializer, %72 ], [ %304, %73 ]
+ %76 = phi i32 [ 0, %72 ], [ %77, %73 ]
+ %77 = add i32 %76, 1
+ %78 = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %79 = tail call i32 @llvm.umin.i32(i32 %77, i32 %78)
+ %80 = sub i32 0, %79
+ %81 = tail call i32 @llvm.smin.i32(i32 %80, i32 0)
+ %82 = load <8 x bfloat>, ptr addrspace(7) %gep, align 2
+ %83 = sub i32 0, %76
+ %84 = tail call i32 @llvm.umin.i32(i32 %83, i32 1)
+ %85 = tail call i32 @llvm.umin.i32(i32 %84, i32 %0)
+ %86 = load <8 x bfloat>, ptr addrspace(7) %gep106, align 2
+ %87 = or i32 %78, 512
+ %88 = lshr i32 %87, 1
+ %89 = tail call i32 @llvm.umin.i32(i32 %83, i32 %88)
+ %90 = sub i32 0, %89
+ %91 = tail call i32 @llvm.smin.i32(i32 %90, i32 0)
+ %92 = load <8 x bfloat>, ptr addrspace(7) %gep108, align 2
+ %93 = or i32 %78, 768
+ %94 = lshr i32 %93, 1
+ %95 = tail call i32 @llvm.umin.i32(i32 %83, i32 %94)
+ %96 = sub i32 0, %95
+ %97 = tail call i32 @llvm.smin.i32(i32 %96, i32 0)
+ %98 = load <8 x bfloat>, ptr addrspace(7) %invariant.gep, align 2
+ %99 = lshr i32 %78, 1
+ %100 = tail call i32 @llvm.umin.i32(i32 %77, i32 %99)
+ %101 = load <8 x bfloat>, ptr addrspace(7) %gep112, align 2
+ %102 = or i32 %78, 256
+ %103 = tail call i32 @llvm.umin.i32(i32 %83, i32 %102)
+ %104 = sub i32 0, %103
+ %105 = tail call i32 @llvm.smin.i32(i32 %104, i32 0)
+ %106 = load <8 x bfloat>, ptr addrspace(7) %gep114, align 2
+ %107 = tail call i32 @llvm.umin.i32(i32 %83, i32 %87)
+ %108 = load <8 x bfloat>, ptr addrspace(7) %gep116, align 2
+ %109 = tail call i32 @llvm.umin.i32(i32 %83, i32 %93)
+ %110 = load <8 x bfloat>, ptr addrspace(7) %gep118, align 2
+ %111 = or i32 %78, 64
+ %112 = tail call i32 @llvm.umin.i32(i32 %83, i32 %111)
+ %113 = load <8 x bfloat>, ptr addrspace(7) %gep120, align 2
+ %114 = or i32 %78, 80
+ %115 = tail call i32 @llvm.umin.i32(i32 %83, i32 %114)
+ %116 = load <8 x bfloat>, ptr addrspace(7) %gep122, align 2
+ %117 = or i32 %78, 96
+ %118 = tail call i32 @llvm.umin.i32(i32 %83, i32 %117)
+ %119 = load <8 x bfloat>, ptr addrspace(7) %gep124, align 2
+ %120 = or i32 %78, 1
+ %121 = tail call i32 @llvm.umin.i32(i32 %77, i32 %120)
+ %122 = load <8 x bfloat>, ptr addrspace(7) null, align 2
+ fence syncscope("workgroup") release
+ %123 = load <1 x bfloat>, ptr addrspace(3) %13, align 2
+ %124 = load <1 x bfloat>, ptr addrspace(3) %14, align 2
+ %125 = load <1 x bfloat>, ptr addrspace(3) %15, align 2
+ %126 = load <1 x bfloat>, ptr addrspace(3) %16, align 2
+ %127 = load <1 x bfloat>, ptr addrspace(3) %17, align 2
+ %128 = shufflevector <1 x bfloat> %127, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %129 = load <1 x bfloat>, ptr addrspace(3) %18, align 2
+ %130 = load <1 x bfloat>, ptr addrspace(3) %19, align 2
+ %131 = shufflevector <1 x bfloat> %130, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %132 = load <1 x bfloat>, ptr addrspace(3) %20, align 2
+ %133 = load <1 x bfloat>, ptr addrspace(3) %21, align 2
+ %134 = load <1 x bfloat>, ptr addrspace(3) %22, align 2
+ %135 = load <1 x bfloat>, ptr addrspace(3) %23, align 2
+ %136 = load <1 x bfloat>, ptr addrspace(3) %24, align 2
+ %137 = load <1 x bfloat>, ptr addrspace(3) %40, align 2
+ %138 = load <1 x bfloat>, ptr addrspace(3) %25, align 2
+ %139 = shufflevector <1 x bfloat> %138, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %140 = load <1 x bfloat>, ptr addrspace(3) %26, align 2
+ %141 = load <1 x bfloat>, ptr addrspace(3) %27, align 2
+ %142 = load <1 x bfloat>, ptr addrspace(3) %28, align 2
+ %143 = load <1 x bfloat>, ptr addrspace(3) %29, align 2
+ %144 = load <1 x bfloat>, ptr addrspace(3) %31, align 2
+ %145 = load <1 x bfloat>, ptr addrspace(3) %32, align 2
+ %146 = load <1 x bfloat>, ptr addrspace(3) %33, align 2
+ %147 = load <1 x bfloat>, ptr addrspace(3) %34, align 2
+ %148 = load <1 x bfloat>, ptr addrspace(3) %35, align 2
+ %149 = load <1 x bfloat>, ptr addrspace(3) %36, align 2
+ %150 = load <1 x bfloat>, ptr addrspace(3) %37, align 2
+ %151 = shufflevector <1 x bfloat> %150, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %152 = load <1 x bfloat>, ptr addrspace(3) %38, align 2
+ %153 = shufflevector <1 x bfloat> %152, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %154 = load <1 x bfloat>, ptr addrspace(3) %39, align 2
+ %155 = shufflevector <1 x bfloat> %154, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %156 = load <1 x bfloat>, ptr addrspace(3) %9, align 2
+ %157 = load <1 x bfloat>, ptr addrspace(3) %43, align 2
+ %158 = load <1 x bfloat>, ptr addrspace(3) %44, align 2
+ %159 = load <1 x bfloat>, ptr addrspace(3) %45, align 2
+ %160 = load <1 x bfloat>, ptr addrspace(3) %46, align 2
+ %161 = load <1 x bfloat>, ptr addrspace(3) %47, align 2
+ %162 = load <1 x bfloat>, ptr addrspace(3) %48, align 2
+ %163 = load <1 x bfloat>, ptr addrspace(3) %49, align 2
+ %164 = load <1 x bfloat>, ptr addrspace(3) %50, align 2
+ %165 = shufflevector <1 x bfloat> %164, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %166 = load <1 x bfloat>, ptr addrspace(3) %51, align 2
+ %167 = load <1 x bfloat>, ptr addrspace(3) %7, align 2
+ %168 = shufflevector <1 x bfloat> %167, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %169 = load <1 x bfloat>, ptr addrspace(3) %53, align 2
+ %170 = load <1 x bfloat>, ptr addrspace(3) %54, align 2
+ %171 = load <1 x bfloat>, ptr addrspace(3) %55, align 2
+ %172 = load <1 x bfloat>, ptr addrspace(3) %56, align 2
+ %173 = load <1 x bfloat>, ptr addrspace(3) %3, align 2
+ %174 = load <1 x bfloat>, ptr addrspace(3) %6, align 2
+ %175 = load <1 x bfloat>, ptr addrspace(3) %5, align 2
+ %176 = load <1 x bfloat>, ptr addrspace(3) %57, align 2
+ %177 = load <1 x bfloat>, ptr addrspace(3) %8, align 2
+ %178 = load <1 x bfloat>, ptr addrspace(3) %11, align 2
+ %179 = load <1 x bfloat>, ptr addrspace(3) %30, align 2
+ %180 = load <1 x bfloat>, ptr addrspace(3) %59, align 2
+ %181 = load <1 x bfloat>, ptr addrspace(3) %58, align 2
+ %182 = load <1 x bfloat>, ptr addrspace(3) %52, align 2
+ %183 = load <1 x bfloat>, ptr addrspace(3) %42, align 2
+ %184 = load <1 x bfloat>, ptr addrspace(3) %1, align 2
+ %185 = load <1 x bfloat>, ptr addrspace(3) %4, align 2
+ %186 = load <1 x bfloat>, ptr addrspace(3) %10, align 2
+ %187 = load <1 x bfloat>, ptr addrspace(3) %41, align 2
+ %188 = load <1 x bfloat>, ptr addrspace(3) %2, align 2
+ %189 = load <1 x bfloat>, ptr addrspace(3) %12, align 2
+ %190 = shufflevector <1 x bfloat> %74, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %191 = shufflevector <4 x bfloat> zeroinitializer, <4 x bfloat> %190, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %192 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 3, 1, 0
+ %193 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %67, <4 x float> %192, i32 0, i32 0, i32 0)
+ %194 = shufflevector <1 x bfloat> %123, <1 x bfloat> %125, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %195 = shufflevector <4 x bfloat> %194, <4 x bfloat> %128, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %196 = shufflevector <1 x bfloat> %156, <1 x bfloat> %160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %197 = shufflevector <4 x bfloat> %196, <4 x bfloat> %165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %198 = shufflevector <1 x bfloat> %170, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %199 = shufflevector <4 x bfloat> %195, <4 x bfloat> %131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %200 = bitcast <4 x bfloat> %199 to <4 x i16>
+ %201 = shufflevector <4 x bfloat> %197, <4 x bfloat> %198, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %202 = bitcast <4 x bfloat> %201 to <4 x i16>
+ %203 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 0, 0, 0
+ %204 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %61, <4 x float> %203, i32 0, i32 0, i32 0)
+ %205 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %200, <4 x i16> %202, <4 x float> %204, i32 0, i32 0, i32 0)
+ %206 = shufflevector <1 x bfloat> %157, <1 x bfloat> %161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %207 = shufflevector <1 x bfloat> %166, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %208 = shufflevector <4 x bfloat> %206, <4 x bfloat> %207, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %209 = shufflevector <1 x bfloat> %171, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %210 = shufflevector <4 x bfloat> %208, <4 x bfloat> %209, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %211 = shufflevector <1 x bfloat> %158, <1 x bfloat> %162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %212 = shufflevector <4 x bfloat> %211, <4 x bfloat> %168, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %213 = shufflevector <1 x bfloat> %172, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %214 = shufflevector <1 x bfloat> %159, <1 x bfloat> %163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %215 = shufflevector <1 x bfloat> %169, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %216 = shufflevector <4 x bfloat> %214, <4 x bfloat> %215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %217 = shufflevector <1 x bfloat> %173, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %218 = shufflevector <4 x bfloat> %216, <4 x bfloat> %217, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %219 = bitcast <4 x bfloat> %218 to <4 x i16>
+ %220 = shufflevector <1 x bfloat> %124, <1 x bfloat> %126, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %221 = shufflevector <1 x bfloat> %129, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %222 = bitcast <4 x bfloat> %210 to <4 x i16>
+ %223 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 1, 1, 0
+ %224 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %222, <4 x float> %223, i32 0, i32 0, i32 0)
+ %225 = shufflevector <4 x bfloat> %220, <4 x bfloat> %221, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %226 = shufflevector <1 x bfloat> %132, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %227 = shufflevector <4 x bfloat> %225, <4 x bfloat> %226, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %228 = bitcast <4 x bfloat> %227 to <4 x i16>
+ %229 = shufflevector <4 x bfloat> %212, <4 x bfloat> %213, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %230 = bitcast <4 x bfloat> %229 to <4 x i16>
+ %231 = bitcast <4 x bfloat> %191 to <4 x i16>
+ %232 = extractvalue [4 x [2 x [1 x <4 x float>]]] %75, 2, 1, 0
+ %233 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %62, <4 x i16> %231, <4 x float> %232, i32 0, i32 0, i32 0)
+ %234 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %228, <4 x i16> %230, <4 x float> %233, i32 0, i32 0, i32 0)
+ %235 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %219, <4 x float> %193, i32 0, i32 0, i32 0)
+ %236 = shufflevector <1 x bfloat> %133, <1 x bfloat> %135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %237 = shufflevector <1 x bfloat> %137, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %238 = shufflevector <4 x bfloat> %236, <4 x bfloat> %237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %239 = shufflevector <4 x bfloat> %238, <4 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %240 = shufflevector <1 x bfloat> %174, <1 x bfloat> %178, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %241 = shufflevector <1 x bfloat> %182, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %242 = shufflevector <4 x bfloat> %240, <4 x bfloat> %241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %243 = shufflevector <1 x bfloat> %185, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %244 = shufflevector <4 x bfloat> %242, <4 x bfloat> %243, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %245 = bitcast <4 x bfloat> %239 to <4 x i16>
+ %246 = bitcast <4 x bfloat> %244 to <4 x i16>
+ %247 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %245, <4 x i16> %246, <4 x float> %205, i32 0, i32 0, i32 0)
+ %248 = shufflevector <1 x bfloat> %175, <1 x bfloat> %179, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %249 = shufflevector <1 x bfloat> %183, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %250 = shufflevector <4 x bfloat> %248, <4 x bfloat> %249, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %251 = shufflevector <1 x bfloat> %186, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %252 = shufflevector <4 x bfloat> %250, <4 x bfloat> %251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %253 = bitcast <4 x bfloat> %252 to <4 x i16>
+ %254 = shufflevector <1 x bfloat> %176, <1 x bfloat> %180, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %255 = shufflevector <4 x bfloat> %254, <4 x bfloat> %60, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %256 = shufflevector <1 x bfloat> %187, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %257 = shufflevector <4 x bfloat> %255, <4 x bfloat> %256, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %258 = bitcast <4 x bfloat> %257 to <4 x i16>
+ %259 = shufflevector <1 x bfloat> %177, <1 x bfloat> %181, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %260 = shufflevector <1 x bfloat> %184, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %261 = shufflevector <4 x bfloat> %259, <4 x bfloat> %260, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %262 = shufflevector <1 x bfloat> %188, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %263 = shufflevector <4 x bfloat> %261, <4 x bfloat> %262, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %264 = bitcast <4 x bfloat> %263 to <4 x i16>
+ %265 = shufflevector <1 x bfloat> %134, <1 x bfloat> %136, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %266 = shufflevector <4 x bfloat> %265, <4 x bfloat> %139, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %267 = bitcast <4 x bfloat> %266 to <4 x i16>
+ %268 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %267, <4 x i16> %253, <4 x float> %224, i32 0, i32 0, i32 0)
+ %269 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %258, <4 x float> %234, i32 0, i32 0, i32 0)
+ %270 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %264, <4 x float> %235, i32 0, i32 0, i32 0)
+ %271 = shufflevector <1 x bfloat> %140, <1 x bfloat> %141, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %272 = shufflevector <1 x bfloat> %142, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %273 = shufflevector <4 x bfloat> %271, <4 x bfloat> %272, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %274 = shufflevector <1 x bfloat> %143, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %275 = shufflevector <4 x bfloat> %273, <4 x bfloat> %274, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %276 = shufflevector <1 x bfloat> splat (bfloat 0xR3F80), <1 x bfloat> %189, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %277 = bitcast <4 x bfloat> %275 to <4 x i16>
+ %278 = bitcast <4 x bfloat> %276 to <4 x i16>
+ %279 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %277, <4 x i16> %278, <4 x float> %247, i32 0, i32 0, i32 0)
+ %280 = shufflevector <1 x bfloat> %144, <1 x bfloat> %145, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %281 = shufflevector <1 x bfloat> %146, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %282 = shufflevector <4 x bfloat> %280, <4 x bfloat> %281, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %283 = shufflevector <1 x bfloat> %147, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %284 = shufflevector <4 x bfloat> %282, <4 x bfloat> %283, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %285 = bitcast <4 x bfloat> %284 to <4 x i16>
+ %286 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %285, <4 x i16> %63, <4 x float> %279, i32 0, i32 0, i32 0)
+ %287 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %69, <4 x float> %268, i32 0, i32 0, i32 0)
+ %288 = shufflevector <1 x bfloat> %148, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+ %289 = shufflevector <1 x bfloat> %149, <1 x bfloat> zeroinitializer, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+ %290 = shufflevector <4 x bfloat> %288, <4 x bfloat> %289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
+ %291 = shufflevector <4 x bfloat> %290, <4 x bfloat> %153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %292 = bitcast <4 x bfloat> %291 to <4 x i16>
+ %293 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %292, <4 x i16> splat (i16 1), <4 x float> %286, i32 0, i32 0, i32 0)
+ %294 = shufflevector <4 x bfloat> %151, <4 x bfloat> %155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %295 = bitcast <4 x bfloat> %294 to <4 x i16>
+ %296 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %70, <4 x i16> zeroinitializer, <4 x float> %293, i32 0, i32 0, i32 0)
+ %297 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %65, <4 x i16> zeroinitializer, <4 x float> %296, i32 0, i32 0, i32 0)
+ %298 = insertvalue [4 x [2 x [1 x <4 x float>]]] zeroinitializer, <4 x float> %297, 0, 0, 0
+ %299 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %295, <4 x i16> zeroinitializer, <4 x float> %287, i32 0, i32 0, i32 0)
+ %300 = insertvalue [4 x [2 x [1 x <4 x float>]]] %298, <4 x float> %299, 1, 1, 0
+ %301 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> %64, <4 x float> %269, i32 0, i32 0, i32 0)
+ %302 = insertvalue [4 x [2 x [1 x <4 x float>]]] %300, <4 x float> %301, 2, 1, 0
+ %303 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %68, <4 x i16> zeroinitializer, <4 x float> %270, i32 0, i32 0, i32 0)
+ %304 = insertvalue [4 x [2 x [1 x <4 x float>]]] %302, <4 x float> %303, 3, 1, 0
+ %305 = trunc i32 %81 to i1
+ %306 = select i1 %305, <8 x bfloat> %82, <8 x bfloat> zeroinitializer
tail call void @llvm.amdgcn.sched.barrier(i32 0)
- store <8 x bfloat> %1018, ptr addrspace(3) %85, align 8
- %1019 = trunc i32 %403 to i1
- %1020 = and i1 %41, %1019
- %1021 = select i1 %1020, <8 x bfloat> %405, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1021, ptr addrspace(3) %88, align 8
- %1022 = trunc i32 %408 to i1
- %1023 = and i1 %41, %1022
- %1024 = select i1 %1023, <8 x bfloat> %410, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1024, ptr addrspace(3) %91, align 8
- %1025 = trunc i32 %413 to i1
- %1026 = and i1 %41, %1025
- %1027 = select i1 %1026, <8 x bfloat> %415, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1027, ptr addrspace(3) %94, align 8
- %1028 = trunc i32 %418 to i1
- %1029 = and i1 %44, %1028
- %1030 = select i1 %1029, <8 x bfloat> %420, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1030, ptr addrspace(3) %97, align 8
- %1031 = trunc i32 %423 to i1
- %1032 = and i1 %44, %1031
- %1033 = select i1 %1032, <8 x bfloat> %425, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1033, ptr addrspace(3) %100, align 8
- %1034 = trunc i32 %428 to i1
- %1035 = and i1 %44, %1034
- %1036 = select i1 %1035, <8 x bfloat> %430, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1036, ptr addrspace(3) %103, align 8
- %1037 = trunc i32 %433 to i1
- %1038 = and i1 %44, %1037
- %1039 = select i1 %1038, <8 x bfloat> %435, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1039, ptr addrspace(3) %106, align 8
- %1040 = trunc i32 %438 to i1
- %1041 = and i1 %44, %1040
- %1042 = select i1 %1041, <8 x bfloat> %440, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1042, ptr addrspace(3) %109, align 8
- %1043 = trunc i32 %443 to i1
- %1044 = and i1 %44, %1043
- %1045 = select i1 %1044, <8 x bfloat> %445, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1045, ptr addrspace(3) %112, align 8
- %1046 = trunc i32 %448 to i1
- %1047 = and i1 %44, %1046
- %1048 = select i1 %1047, <8 x bfloat> %450, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1048, ptr addrspace(3) %115, align 8
- %1049 = trunc i32 %453 to i1
- %1050 = and i1 %44, %1049
- %1051 = select i1 %1050, <8 x bfloat> %455, <8 x bfloat> zeroinitializer
- store <8 x bfloat> %1051, ptr addrspace(3) %118, align 8
- %1052 = icmp samesign ult i32 %391, 72
- br i1 %1052, label %389, label %1053
-
-1053: ; preds = %389
- %1054 = tail call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) nonnull %2, i16 0, i64 8128512, i32 159744)
- %1055 = shl nuw nsw i32 %8, 5
- %.scalar = or disjoint i32 %1055, %11
- %1056 = insertelement <4 x i32> poison, i32 %.scalar, i64 0
- %1057 = shufflevector <4 x i32> %1056, <4 x i32> poison, <4 x i32> zeroinitializer
- %1058 = or disjoint <4 x i32> %1057, <i32 0, i32 1, i32 2, i32 3>
- %1059 = or disjoint <4 x i32> %1057, <i32 16, i32 17, i32 18, i32 19>
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
- %1060 = load <1 x bfloat>, ptr addrspace(3) %157, align 2
- %1061 = load <1 x bfloat>, ptr addrspace(3) %160, align 2
- %1062 = load <1 x bfloat>, ptr addrspace(3) %162, align 2
- %1063 = load <1 x bfloat>, ptr addrspace(3) %163, align 2
- %1064 = load <1 x bfloat>, ptr addrspace(3) %165, align 2
- %1065 = shufflevector <1 x bfloat> %1064, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1066 = load <1 x bfloat>, ptr addrspace(3) %166, align 2
- %1067 = shufflevector <1 x bfloat> %1066, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1068 = load <1 x bfloat>, ptr addrspace(3) %168, align 2
- %1069 = shufflevector <1 x bfloat> %1068, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1070 = load <1 x bfloat>, ptr addrspace(3) %169, align 2
- %1071 = shufflevector <1 x bfloat> %1070, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1072 = load <1 x bfloat>, ptr addrspace(3) %171, align 2
- %1073 = load <1 x bfloat>, ptr addrspace(3) %172, align 2
- %1074 = load <1 x bfloat>, ptr addrspace(3) %174, align 2
- %1075 = load <1 x bfloat>, ptr addrspace(3) %175, align 2
- %1076 = load <1 x bfloat>, ptr addrspace(3) %177, align 2
- %1077 = shufflevector <1 x bfloat> %1076, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1078 = load <1 x bfloat>, ptr addrspace(3) %178, align 2
- %1079 = shufflevector <1 x bfloat> %1078, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1080 = load <1 x bfloat>, ptr addrspace(3) %180, align 2
- %1081 = shufflevector <1 x bfloat> %1080, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1082 = load <1 x bfloat>, ptr addrspace(3) %181, align 2
- %1083 = shufflevector <1 x bfloat> %1082, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1084 = load <1 x bfloat>, ptr addrspace(3) %183, align 2
- %1085 = load <1 x bfloat>, ptr addrspace(3) %184, align 2
- %1086 = load <1 x bfloat>, ptr addrspace(3) %186, align 2
- %1087 = load <1 x bfloat>, ptr addrspace(3) %187, align 2
- %1088 = load <1 x bfloat>, ptr addrspace(3) %189, align 2
- %1089 = shufflevector <1 x bfloat> %1088, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1090 = load <1 x bfloat>, ptr addrspace(3) %190, align 2
- %1091 = shufflevector <1 x bfloat> %1090, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1092 = load <1 x bfloat>, ptr addrspace(3) %192, align 2
- %1093 = shufflevector <1 x bfloat> %1092, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1094 = load <1 x bfloat>, ptr addrspace(3) %193, align 2
- %1095 = shufflevector <1 x bfloat> %1094, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1096 = load <1 x bfloat>, ptr addrspace(3) %195, align 2
- %1097 = load <1 x bfloat>, ptr addrspace(3) %196, align 2
- %1098 = load <1 x bfloat>, ptr addrspace(3) %198, align 2
- %1099 = load <1 x bfloat>, ptr addrspace(3) %199, align 2
- %1100 = load <1 x bfloat>, ptr addrspace(3) %201, align 2
- %1101 = shufflevector <1 x bfloat> %1100, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1102 = load <1 x bfloat>, ptr addrspace(3) %202, align 2
- %1103 = shufflevector <1 x bfloat> %1102, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1104 = load <1 x bfloat>, ptr addrspace(3) %204, align 2
- %1105 = shufflevector <1 x bfloat> %1104, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1106 = load <1 x bfloat>, ptr addrspace(3) %205, align 2
- %1107 = shufflevector <1 x bfloat> %1106, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1108 = load <1 x bfloat>, ptr addrspace(3) %207, align 2
- %1109 = load <1 x bfloat>, ptr addrspace(3) %208, align 2
- %1110 = load <1 x bfloat>, ptr addrspace(3) %210, align 2
- %1111 = load <1 x bfloat>, ptr addrspace(3) %211, align 2
- %1112 = load <1 x bfloat>, ptr addrspace(3) %213, align 2
- %1113 = shufflevector <1 x bfloat> %1112, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1114 = load <1 x bfloat>, ptr addrspace(3) %214, align 2
- %1115 = shufflevector <1 x bfloat> %1114, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1116 = load <1 x bfloat>, ptr addrspace(3) %216, align 2
- %1117 = shufflevector <1 x bfloat> %1116, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1118 = load <1 x bfloat>, ptr addrspace(3) %217, align 2
- %1119 = shufflevector <1 x bfloat> %1118, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1120 = load <1 x bfloat>, ptr addrspace(3) %219, align 2
- %1121 = load <1 x bfloat>, ptr addrspace(3) %220, align 2
- %1122 = load <1 x bfloat>, ptr addrspace(3) %222, align 2
- %1123 = load <1 x bfloat>, ptr addrspace(3) %223, align 2
- %1124 = load <1 x bfloat>, ptr addrspace(3) %225, align 2
- %1125 = shufflevector <1 x bfloat> %1124, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1126 = load <1 x bfloat>, ptr addrspace(3) %226, align 2
- %1127 = shufflevector <1 x bfloat> %1126, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1128 = load <1 x bfloat>, ptr addrspace(3) %228, align 2
- %1129 = shufflevector <1 x bfloat> %1128, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1130 = load <1 x bfloat>, ptr addrspace(3) %229, align 2
- %1131 = shufflevector <1 x bfloat> %1130, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1132 = load <1 x bfloat>, ptr addrspace(3) %231, align 2
- %1133 = load <1 x bfloat>, ptr addrspace(3) %232, align 2
- %1134 = load <1 x bfloat>, ptr addrspace(3) %234, align 2
- %1135 = load <1 x bfloat>, ptr addrspace(3) %235, align 2
- %1136 = load <1 x bfloat>, ptr addrspace(3) %237, align 2
- %1137 = shufflevector <1 x bfloat> %1136, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1138 = load <1 x bfloat>, ptr addrspace(3) %238, align 2
- %1139 = shufflevector <1 x bfloat> %1138, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1140 = load <1 x bfloat>, ptr addrspace(3) %240, align 2
- %1141 = shufflevector <1 x bfloat> %1140, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1142 = load <1 x bfloat>, ptr addrspace(3) %241, align 2
- %1143 = shufflevector <1 x bfloat> %1142, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1144 = load <1 x bfloat>, ptr addrspace(3) %243, align 2
- %1145 = load <1 x bfloat>, ptr addrspace(3) %244, align 2
- %1146 = load <1 x bfloat>, ptr addrspace(3) %246, align 2
- %1147 = load <1 x bfloat>, ptr addrspace(3) %247, align 2
- %1148 = load <1 x bfloat>, ptr addrspace(3) %249, align 2
- %1149 = shufflevector <1 x bfloat> %1148, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1150 = load <1 x bfloat>, ptr addrspace(3) %250, align 2
- %1151 = shufflevector <1 x bfloat> %1150, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1152 = load <1 x bfloat>, ptr addrspace(3) %252, align 2
- %1153 = shufflevector <1 x bfloat> %1152, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1154 = load <1 x bfloat>, ptr addrspace(3) %253, align 2
- %1155 = shufflevector <1 x bfloat> %1154, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1156 = load <1 x bfloat>, ptr addrspace(3) %255, align 2
- %1157 = load <1 x bfloat>, ptr addrspace(3) %258, align 2
- %1158 = load <1 x bfloat>, ptr addrspace(3) %261, align 2
- %1159 = load <1 x bfloat>, ptr addrspace(3) %264, align 2
- %1160 = load <1 x bfloat>, ptr addrspace(3) %265, align 2
- %1161 = load <1 x bfloat>, ptr addrspace(3) %266, align 2
- %1162 = load <1 x bfloat>, ptr addrspace(3) %267, align 2
- %1163 = load <1 x bfloat>, ptr addrspace(3) %268, align 2
- %1164 = load <1 x bfloat>, ptr addrspace(3) %269, align 2
- %1165 = shufflevector <1 x bfloat> %1164, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1166 = load <1 x bfloat>, ptr addrspace(3) %270, align 2
- %1167 = shufflevector <1 x bfloat> %1166, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1168 = load <1 x bfloat>, ptr addrspace(3) %271, align 2
- %1169 = shufflevector <1 x bfloat> %1168, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1170 = load <1 x bfloat>, ptr addrspace(3) %272, align 2
- %1171 = shufflevector <1 x bfloat> %1170, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1172 = load <1 x bfloat>, ptr addrspace(3) %273, align 2
- %1173 = shufflevector <1 x bfloat> %1172, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1174 = load <1 x bfloat>, ptr addrspace(3) %274, align 2
- %1175 = shufflevector <1 x bfloat> %1174, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1176 = load <1 x bfloat>, ptr addrspace(3) %275, align 2
- %1177 = shufflevector <1 x bfloat> %1176, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1178 = load <1 x bfloat>, ptr addrspace(3) %276, align 2
- %1179 = shufflevector <1 x bfloat> %1178, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1180 = load <1 x bfloat>, ptr addrspace(3) %277, align 2
- %1181 = load <1 x bfloat>, ptr addrspace(3) %278, align 2
- %1182 = load <1 x bfloat>, ptr addrspace(3) %279, align 2
- %1183 = load <1 x bfloat>, ptr addrspace(3) %280, align 2
- %1184 = load <1 x bfloat>, ptr addrspace(3) %281, align 2
- %1185 = load <1 x bfloat>, ptr addrspace(3) %282, align 2
- %1186 = load <1 x bfloat>, ptr addrspace(3) %283, align 2
- %1187 = load <1 x bfloat>, ptr addrspace(3) %284, align 2
- %1188 = load <1 x bfloat>, ptr addrspace(3) %285, align 2
- %1189 = shufflevector <1 x bfloat> %1188, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1190 = load <1 x bfloat>, ptr addrspace(3) %286, align 2
- %1191 = shufflevector <1 x bfloat> %1190, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1192 = load <1 x bfloat>, ptr addrspace(3) %287, align 2
- %1193 = shufflevector <1 x bfloat> %1192, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1194 = load <1 x bfloat>, ptr addrspace(3) %288, align 2
- %1195 = shufflevector <1 x bfloat> %1194, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1196 = load <1 x bfloat>, ptr addrspace(3) %289, align 2
- %1197 = shufflevector <1 x bfloat> %1196, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1198 = load <1 x bfloat>, ptr addrspace(3) %290, align 2
- %1199 = shufflevector <1 x bfloat> %1198, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1200 = load <1 x bfloat>, ptr addrspace(3) %291, align 2
- %1201 = shufflevector <1 x bfloat> %1200, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1202 = load <1 x bfloat>, ptr addrspace(3) %292, align 2
- %1203 = shufflevector <1 x bfloat> %1202, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1204 = load <1 x bfloat>, ptr addrspace(3) %293, align 2
- %1205 = load <1 x bfloat>, ptr addrspace(3) %294, align 2
- %1206 = load <1 x bfloat>, ptr addrspace(3) %295, align 2
- %1207 = load <1 x bfloat>, ptr addrspace(3) %296, align 2
- %1208 = load <1 x bfloat>, ptr addrspace(3) %297, align 2
- %1209 = load <1 x bfloat>, ptr addrspace(3) %298, align 2
- %1210 = load <1 x bfloat>, ptr addrspace(3) %299, align 2
- %1211 = load <1 x bfloat>, ptr addrspace(3) %300, align 2
- %1212 = load <1 x bfloat>, ptr addrspace(3) %301, align 2
- %1213 = shufflevector <1 x bfloat> %1212, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1214 = load <1 x bfloat>, ptr addrspace(3) %302, align 2
- %1215 = shufflevector <1 x bfloat> %1214, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1216 = load <1 x bfloat>, ptr addrspace(3) %303, align 2
- %1217 = shufflevector <1 x bfloat> %1216, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1218 = load <1 x bfloat>, ptr addrspace(3) %304, align 2
- %1219 = shufflevector <1 x bfloat> %1218, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1220 = load <1 x bfloat>, ptr addrspace(3) %305, align 2
- %1221 = shufflevector <1 x bfloat> %1220, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1222 = load <1 x bfloat>, ptr addrspace(3) %306, align 2
- %1223 = shufflevector <1 x bfloat> %1222, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1224 = load <1 x bfloat>, ptr addrspace(3) %307, align 2
- %1225 = shufflevector <1 x bfloat> %1224, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1226 = load <1 x bfloat>, ptr addrspace(3) %308, align 2
- %1227 = shufflevector <1 x bfloat> %1226, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1228 = load <1 x bfloat>, ptr addrspace(3) %309, align 2
- %1229 = load <1 x bfloat>, ptr addrspace(3) %310, align 2
- %1230 = load <1 x bfloat>, ptr addrspace(3) %311, align 2
- %1231 = load <1 x bfloat>, ptr addrspace(3) %312, align 2
- %1232 = load <1 x bfloat>, ptr addrspace(3) %313, align 2
- %1233 = load <1 x bfloat>, ptr addrspace(3) %314, align 2
- %1234 = load <1 x bfloat>, ptr addrspace(3) %315, align 2
- %1235 = load <1 x bfloat>, ptr addrspace(3) %316, align 2
- %1236 = load <1 x bfloat>, ptr addrspace(3) %317, align 2
- %1237 = shufflevector <1 x bfloat> %1236, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1238 = load <1 x bfloat>, ptr addrspace(3) %318, align 2
- %1239 = shufflevector <1 x bfloat> %1238, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1240 = load <1 x bfloat>, ptr addrspace(3) %319, align 2
- %1241 = shufflevector <1 x bfloat> %1240, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1242 = load <1 x bfloat>, ptr addrspace(3) %320, align 2
- %1243 = shufflevector <1 x bfloat> %1242, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1244 = load <1 x bfloat>, ptr addrspace(3) %321, align 2
- %1245 = shufflevector <1 x bfloat> %1244, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1246 = load <1 x bfloat>, ptr addrspace(3) %322, align 2
- %1247 = shufflevector <1 x bfloat> %1246, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1248 = load <1 x bfloat>, ptr addrspace(3) %323, align 2
- %1249 = shufflevector <1 x bfloat> %1248, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1250 = load <1 x bfloat>, ptr addrspace(3) %324, align 2
- %1251 = shufflevector <1 x bfloat> %1250, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1252 = load <1 x bfloat>, ptr addrspace(3) %325, align 2
- %1253 = load <1 x bfloat>, ptr addrspace(3) %326, align 2
- %1254 = load <1 x bfloat>, ptr addrspace(3) %327, align 2
- %1255 = load <1 x bfloat>, ptr addrspace(3) %328, align 2
- %1256 = load <1 x bfloat>, ptr addrspace(3) %329, align 2
- %1257 = load <1 x bfloat>, ptr addrspace(3) %330, align 2
- %1258 = load <1 x bfloat>, ptr addrspace(3) %331, align 2
- %1259 = load <1 x bfloat>, ptr addrspace(3) %332, align 2
- %1260 = load <1 x bfloat>, ptr addrspace(3) %333, align 2
- %1261 = shufflevector <1 x bfloat> %1260, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1262 = load <1 x bfloat>, ptr addrspace(3) %334, align 2
- %1263 = shufflevector <1 x bfloat> %1262, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1264 = load <1 x bfloat>, ptr addrspace(3) %335, align 2
- %1265 = shufflevector <1 x bfloat> %1264, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1266 = load <1 x bfloat>, ptr addrspace(3) %336, align 2
- %1267 = shufflevector <1 x bfloat> %1266, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1268 = load <1 x bfloat>, ptr addrspace(3) %337, align 2
- %1269 = shufflevector <1 x bfloat> %1268, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1270 = load <1 x bfloat>, ptr addrspace(3) %338, align 2
- %1271 = shufflevector <1 x bfloat> %1270, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1272 = load <1 x bfloat>, ptr addrspace(3) %339, align 2
- %1273 = shufflevector <1 x bfloat> %1272, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1274 = load <1 x bfloat>, ptr addrspace(3) %340, align 2
- %1275 = shufflevector <1 x bfloat> %1274, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1276 = load <1 x bfloat>, ptr addrspace(3) %341, align 2
- %1277 = load <1 x bfloat>, ptr addrspace(3) %342, align 2
- %1278 = load <1 x bfloat>, ptr addrspace(3) %343, align 2
- %1279 = load <1 x bfloat>, ptr addrspace(3) %344, align 2
- %1280 = load <1 x bfloat>, ptr addrspace(3) %345, align 2
- %1281 = load <1 x bfloat>, ptr addrspace(3) %346, align 2
- %1282 = load <1 x bfloat>, ptr addrspace(3) %347, align 2
- %1283 = load <1 x bfloat>, ptr addrspace(3) %348, align 2
- %1284 = load <1 x bfloat>, ptr addrspace(3) %349, align 2
- %1285 = shufflevector <1 x bfloat> %1284, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1286 = load <1 x bfloat>, ptr addrspace(3) %350, align 2
- %1287 = shufflevector <1 x bfloat> %1286, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1288 = load <1 x bfloat>, ptr addrspace(3) %351, align 2
- %1289 = shufflevector <1 x bfloat> %1288, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1290 = load <1 x bfloat>, ptr addrspace(3) %352, align 2
- %1291 = shufflevector <1 x bfloat> %1290, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1292 = load <1 x bfloat>, ptr addrspace(3) %353, align 2
- %1293 = shufflevector <1 x bfloat> %1292, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1294 = load <1 x bfloat>, ptr addrspace(3) %354, align 2
- %1295 = shufflevector <1 x bfloat> %1294, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1296 = load <1 x bfloat>, ptr addrspace(3) %355, align 2
- %1297 = shufflevector <1 x bfloat> %1296, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1298 = load <1 x bfloat>, ptr addrspace(3) %356, align 2
- %1299 = shufflevector <1 x bfloat> %1298, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1300 = load <1 x bfloat>, ptr addrspace(3) %357, align 2
- %1301 = load <1 x bfloat>, ptr addrspace(3) %358, align 2
- %1302 = load <1 x bfloat>, ptr addrspace(3) %359, align 2
- %1303 = load <1 x bfloat>, ptr addrspace(3) %360, align 2
- %1304 = load <1 x bfloat>, ptr addrspace(3) %361, align 2
- %1305 = load <1 x bfloat>, ptr addrspace(3) %362, align 2
- %1306 = load <1 x bfloat>, ptr addrspace(3) %363, align 2
- %1307 = load <1 x bfloat>, ptr addrspace(3) %364, align 2
- %1308 = load <1 x bfloat>, ptr addrspace(3) %365, align 2
- %1309 = shufflevector <1 x bfloat> %1308, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1310 = load <1 x bfloat>, ptr addrspace(3) %366, align 2
- %1311 = shufflevector <1 x bfloat> %1310, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1312 = load <1 x bfloat>, ptr addrspace(3) %367, align 2
- %1313 = shufflevector <1 x bfloat> %1312, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1314 = load <1 x bfloat>, ptr addrspace(3) %368, align 2
- %1315 = shufflevector <1 x bfloat> %1314, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1316 = load <1 x bfloat>, ptr addrspace(3) %369, align 2
- %1317 = shufflevector <1 x bfloat> %1316, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1318 = load <1 x bfloat>, ptr addrspace(3) %370, align 2
- %1319 = shufflevector <1 x bfloat> %1318, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1320 = load <1 x bfloat>, ptr addrspace(3) %371, align 2
- %1321 = shufflevector <1 x bfloat> %1320, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1322 = load <1 x bfloat>, ptr addrspace(3) %372, align 2
- %1323 = shufflevector <1 x bfloat> %1322, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1324 = load <1 x bfloat>, ptr addrspace(3) %373, align 2
- %1325 = load <1 x bfloat>, ptr addrspace(3) %374, align 2
- %1326 = load <1 x bfloat>, ptr addrspace(3) %375, align 2
- %1327 = load <1 x bfloat>, ptr addrspace(3) %376, align 2
- %1328 = load <1 x bfloat>, ptr addrspace(3) %377, align 2
- %1329 = load <1 x bfloat>, ptr addrspace(3) %378, align 2
- %1330 = load <1 x bfloat>, ptr addrspace(3) %379, align 2
- %1331 = load <1 x bfloat>, ptr addrspace(3) %380, align 2
- %1332 = load <1 x bfloat>, ptr addrspace(3) %381, align 2
- %1333 = shufflevector <1 x bfloat> %1332, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1334 = load <1 x bfloat>, ptr addrspace(3) %382, align 2
- %1335 = shufflevector <1 x bfloat> %1334, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1336 = load <1 x bfloat>, ptr addrspace(3) %383, align 2
- %1337 = shufflevector <1 x bfloat> %1336, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1338 = load <1 x bfloat>, ptr addrspace(3) %384, align 2
- %1339 = shufflevector <1 x bfloat> %1338, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1340 = load <1 x bfloat>, ptr addrspace(3) %385, align 2
- %1341 = shufflevector <1 x bfloat> %1340, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1342 = load <1 x bfloat>, ptr addrspace(3) %386, align 2
- %1343 = shufflevector <1 x bfloat> %1342, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1344 = load <1 x bfloat>, ptr addrspace(3) %387, align 2
- %1345 = shufflevector <1 x bfloat> %1344, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1346 = load <1 x bfloat>, ptr addrspace(3) %388, align 2
- %1347 = shufflevector <1 x bfloat> %1346, <1 x bfloat> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
- %1348 = shufflevector <1 x bfloat> %1060, <1 x bfloat> %1062, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1349 = shufflevector <4 x bfloat> %1348, <4 x bfloat> %1065, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1350 = shufflevector <4 x bfloat> %1349, <4 x bfloat> %1069, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1351 = shufflevector <1 x bfloat> %1156, <1 x bfloat> %1160, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1352 = shufflevector <4 x bfloat> %1351, <4 x bfloat> %1165, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1353 = shufflevector <4 x bfloat> %1352, <4 x bfloat> %1173, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1354 = bitcast <4 x bfloat> %1350 to <4 x i16>
- %1355 = bitcast <4 x bfloat> %1353 to <4 x i16>
- %1356 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1355, <4 x float> %984, i32 0, i32 0, i32 0)
- %1357 = shufflevector <1 x bfloat> %1157, <1 x bfloat> %1161, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1358 = shufflevector <4 x bfloat> %1357, <4 x bfloat> %1167, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1359 = shufflevector <4 x bfloat> %1358, <4 x bfloat> %1175, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1360 = bitcast <4 x bfloat> %1359 to <4 x i16>
- %1361 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1360, <4 x float> %989, i32 0, i32 0, i32 0)
- %1362 = shufflevector <1 x bfloat> %1158, <1 x bfloat> %1162, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1363 = shufflevector <4 x bfloat> %1362, <4 x bfloat> %1169, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1364 = shufflevector <4 x bfloat> %1363, <4 x bfloat> %1177, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1365 = bitcast <4 x bfloat> %1364 to <4 x i16>
- %1366 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1365, <4 x float> %994, i32 0, i32 0, i32 0)
- %1367 = shufflevector <1 x bfloat> %1159, <1 x bfloat> %1163, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1368 = shufflevector <4 x bfloat> %1367, <4 x bfloat> %1171, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1369 = shufflevector <4 x bfloat> %1368, <4 x bfloat> %1179, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1370 = bitcast <4 x bfloat> %1369 to <4 x i16>
- %1371 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1354, <4 x i16> %1370, <4 x float> %999, i32 0, i32 0, i32 0)
- %1372 = shufflevector <1 x bfloat> %1061, <1 x bfloat> %1063, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1373 = shufflevector <4 x bfloat> %1372, <4 x bfloat> %1067, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1374 = shufflevector <4 x bfloat> %1373, <4 x bfloat> %1071, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1375 = bitcast <4 x bfloat> %1374 to <4 x i16>
- %1376 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1355, <4 x float> %1004, i32 0, i32 0, i32 0)
- %1377 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1360, <4 x float> %1005, i32 0, i32 0, i32 0)
- %1378 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1365, <4 x float> %1006, i32 0, i32 0, i32 0)
- %1379 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1375, <4 x i16> %1370, <4 x float> %1007, i32 0, i32 0, i32 0)
- %1380 = shufflevector <1 x bfloat> %1072, <1 x bfloat> %1074, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1381 = shufflevector <4 x bfloat> %1380, <4 x bfloat> %1077, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1382 = shufflevector <4 x bfloat> %1381, <4 x bfloat> %1081, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1383 = shufflevector <1 x bfloat> %1180, <1 x bfloat> %1184, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1384 = shufflevector <4 x bfloat> %1383, <4 x bfloat> %1189, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1385 = shufflevector <4 x bfloat> %1384, <4 x bfloat> %1197, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1386 = bitcast <4 x bfloat> %1382 to <4 x i16>
- %1387 = bitcast <4 x bfloat> %1385 to <4 x i16>
- %1388 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1387, <4 x float> %1356, i32 0, i32 0, i32 0)
- %1389 = shufflevector <1 x bfloat> %1181, <1 x bfloat> %1185, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1390 = shufflevector <4 x bfloat> %1389, <4 x bfloat> %1191, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1391 = shufflevector <4 x bfloat> %1390, <4 x bfloat> %1199, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1392 = bitcast <4 x bfloat> %1391 to <4 x i16>
- %1393 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1392, <4 x float> %1361, i32 0, i32 0, i32 0)
- %1394 = shufflevector <1 x bfloat> %1182, <1 x bfloat> %1186, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1395 = shufflevector <4 x bfloat> %1394, <4 x bfloat> %1193, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1396 = shufflevector <4 x bfloat> %1395, <4 x bfloat> %1201, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1397 = bitcast <4 x bfloat> %1396 to <4 x i16>
- %1398 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1397, <4 x float> %1366, i32 0, i32 0, i32 0)
- %1399 = shufflevector <1 x bfloat> %1183, <1 x bfloat> %1187, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1400 = shufflevector <4 x bfloat> %1399, <4 x bfloat> %1195, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1401 = shufflevector <4 x bfloat> %1400, <4 x bfloat> %1203, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1402 = bitcast <4 x bfloat> %1401 to <4 x i16>
- %1403 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1386, <4 x i16> %1402, <4 x float> %1371, i32 0, i32 0, i32 0)
- %1404 = shufflevector <1 x bfloat> %1073, <1 x bfloat> %1075, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1405 = shufflevector <4 x bfloat> %1404, <4 x bfloat> %1079, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1406 = shufflevector <4 x bfloat> %1405, <4 x bfloat> %1083, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1407 = bitcast <4 x bfloat> %1406 to <4 x i16>
- %1408 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1387, <4 x float> %1376, i32 0, i32 0, i32 0)
- %1409 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1392, <4 x float> %1377, i32 0, i32 0, i32 0)
- %1410 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1397, <4 x float> %1378, i32 0, i32 0, i32 0)
- %1411 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1407, <4 x i16> %1402, <4 x float> %1379, i32 0, i32 0, i32 0)
- %1412 = shufflevector <1 x bfloat> %1084, <1 x bfloat> %1086, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1413 = shufflevector <4 x bfloat> %1412, <4 x bfloat> %1089, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1414 = shufflevector <4 x bfloat> %1413, <4 x bfloat> %1093, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1415 = shufflevector <1 x bfloat> %1204, <1 x bfloat> %1208, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1416 = shufflevector <4 x bfloat> %1415, <4 x bfloat> %1213, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1417 = shufflevector <4 x bfloat> %1416, <4 x bfloat> %1221, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1418 = bitcast <4 x bfloat> %1414 to <4 x i16>
- %1419 = bitcast <4 x bfloat> %1417 to <4 x i16>
- %1420 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1419, <4 x float> %1388, i32 0, i32 0, i32 0)
- %1421 = shufflevector <1 x bfloat> %1205, <1 x bfloat> %1209, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1422 = shufflevector <4 x bfloat> %1421, <4 x bfloat> %1215, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1423 = shufflevector <4 x bfloat> %1422, <4 x bfloat> %1223, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1424 = bitcast <4 x bfloat> %1423 to <4 x i16>
- %1425 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1424, <4 x float> %1393, i32 0, i32 0, i32 0)
- %1426 = shufflevector <1 x bfloat> %1206, <1 x bfloat> %1210, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1427 = shufflevector <4 x bfloat> %1426, <4 x bfloat> %1217, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1428 = shufflevector <4 x bfloat> %1427, <4 x bfloat> %1225, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1429 = bitcast <4 x bfloat> %1428 to <4 x i16>
- %1430 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1429, <4 x float> %1398, i32 0, i32 0, i32 0)
- %1431 = shufflevector <1 x bfloat> %1207, <1 x bfloat> %1211, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1432 = shufflevector <4 x bfloat> %1431, <4 x bfloat> %1219, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1433 = shufflevector <4 x bfloat> %1432, <4 x bfloat> %1227, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1434 = bitcast <4 x bfloat> %1433 to <4 x i16>
- %1435 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1418, <4 x i16> %1434, <4 x float> %1403, i32 0, i32 0, i32 0)
- %1436 = shufflevector <1 x bfloat> %1085, <1 x bfloat> %1087, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1437 = shufflevector <4 x bfloat> %1436, <4 x bfloat> %1091, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1438 = shufflevector <4 x bfloat> %1437, <4 x bfloat> %1095, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1439 = bitcast <4 x bfloat> %1438 to <4 x i16>
- %1440 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1419, <4 x float> %1408, i32 0, i32 0, i32 0)
- %1441 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1424, <4 x float> %1409, i32 0, i32 0, i32 0)
- %1442 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1429, <4 x float> %1410, i32 0, i32 0, i32 0)
- %1443 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1439, <4 x i16> %1434, <4 x float> %1411, i32 0, i32 0, i32 0)
- %1444 = shufflevector <1 x bfloat> %1096, <1 x bfloat> %1098, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1445 = shufflevector <4 x bfloat> %1444, <4 x bfloat> %1101, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1446 = shufflevector <4 x bfloat> %1445, <4 x bfloat> %1105, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1447 = shufflevector <1 x bfloat> %1228, <1 x bfloat> %1232, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1448 = shufflevector <4 x bfloat> %1447, <4 x bfloat> %1237, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1449 = shufflevector <4 x bfloat> %1448, <4 x bfloat> %1245, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1450 = bitcast <4 x bfloat> %1446 to <4 x i16>
- %1451 = bitcast <4 x bfloat> %1449 to <4 x i16>
- %1452 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1451, <4 x float> %1420, i32 0, i32 0, i32 0)
- %1453 = shufflevector <1 x bfloat> %1229, <1 x bfloat> %1233, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1454 = shufflevector <4 x bfloat> %1453, <4 x bfloat> %1239, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1455 = shufflevector <4 x bfloat> %1454, <4 x bfloat> %1247, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1456 = bitcast <4 x bfloat> %1455 to <4 x i16>
- %1457 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1456, <4 x float> %1425, i32 0, i32 0, i32 0)
- %1458 = shufflevector <1 x bfloat> %1230, <1 x bfloat> %1234, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1459 = shufflevector <4 x bfloat> %1458, <4 x bfloat> %1241, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1460 = shufflevector <4 x bfloat> %1459, <4 x bfloat> %1249, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1461 = bitcast <4 x bfloat> %1460 to <4 x i16>
- %1462 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1461, <4 x float> %1430, i32 0, i32 0, i32 0)
- %1463 = shufflevector <1 x bfloat> %1231, <1 x bfloat> %1235, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1464 = shufflevector <4 x bfloat> %1463, <4 x bfloat> %1243, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1465 = shufflevector <4 x bfloat> %1464, <4 x bfloat> %1251, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1466 = bitcast <4 x bfloat> %1465 to <4 x i16>
- %1467 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1450, <4 x i16> %1466, <4 x float> %1435, i32 0, i32 0, i32 0)
- %1468 = shufflevector <1 x bfloat> %1097, <1 x bfloat> %1099, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1469 = shufflevector <4 x bfloat> %1468, <4 x bfloat> %1103, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1470 = shufflevector <4 x bfloat> %1469, <4 x bfloat> %1107, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1471 = bitcast <4 x bfloat> %1470 to <4 x i16>
- %1472 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1451, <4 x float> %1440, i32 0, i32 0, i32 0)
- %1473 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1456, <4 x float> %1441, i32 0, i32 0, i32 0)
- %1474 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1461, <4 x float> %1442, i32 0, i32 0, i32 0)
- %1475 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1471, <4 x i16> %1466, <4 x float> %1443, i32 0, i32 0, i32 0)
- %1476 = shufflevector <1 x bfloat> %1108, <1 x bfloat> %1110, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1477 = shufflevector <4 x bfloat> %1476, <4 x bfloat> %1113, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1478 = shufflevector <4 x bfloat> %1477, <4 x bfloat> %1117, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1479 = shufflevector <1 x bfloat> %1252, <1 x bfloat> %1256, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1480 = shufflevector <4 x bfloat> %1479, <4 x bfloat> %1261, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1481 = shufflevector <4 x bfloat> %1480, <4 x bfloat> %1269, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1482 = bitcast <4 x bfloat> %1478 to <4 x i16>
- %1483 = bitcast <4 x bfloat> %1481 to <4 x i16>
- %1484 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1483, <4 x float> %1452, i32 0, i32 0, i32 0)
- %1485 = shufflevector <1 x bfloat> %1253, <1 x bfloat> %1257, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1486 = shufflevector <4 x bfloat> %1485, <4 x bfloat> %1263, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1487 = shufflevector <4 x bfloat> %1486, <4 x bfloat> %1271, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1488 = bitcast <4 x bfloat> %1487 to <4 x i16>
- %1489 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1488, <4 x float> %1457, i32 0, i32 0, i32 0)
- %1490 = shufflevector <1 x bfloat> %1254, <1 x bfloat> %1258, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1491 = shufflevector <4 x bfloat> %1490, <4 x bfloat> %1265, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1492 = shufflevector <4 x bfloat> %1491, <4 x bfloat> %1273, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1493 = bitcast <4 x bfloat> %1492 to <4 x i16>
- %1494 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1493, <4 x float> %1462, i32 0, i32 0, i32 0)
- %1495 = shufflevector <1 x bfloat> %1255, <1 x bfloat> %1259, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1496 = shufflevector <4 x bfloat> %1495, <4 x bfloat> %1267, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1497 = shufflevector <4 x bfloat> %1496, <4 x bfloat> %1275, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1498 = bitcast <4 x bfloat> %1497 to <4 x i16>
- %1499 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1482, <4 x i16> %1498, <4 x float> %1467, i32 0, i32 0, i32 0)
- %1500 = shufflevector <1 x bfloat> %1109, <1 x bfloat> %1111, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1501 = shufflevector <4 x bfloat> %1500, <4 x bfloat> %1115, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1502 = shufflevector <4 x bfloat> %1501, <4 x bfloat> %1119, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1503 = bitcast <4 x bfloat> %1502 to <4 x i16>
- %1504 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1483, <4 x float> %1472, i32 0, i32 0, i32 0)
- %1505 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1488, <4 x float> %1473, i32 0, i32 0, i32 0)
- %1506 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1493, <4 x float> %1474, i32 0, i32 0, i32 0)
- %1507 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1503, <4 x i16> %1498, <4 x float> %1475, i32 0, i32 0, i32 0)
- %1508 = shufflevector <1 x bfloat> %1120, <1 x bfloat> %1122, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1509 = shufflevector <4 x bfloat> %1508, <4 x bfloat> %1125, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1510 = shufflevector <4 x bfloat> %1509, <4 x bfloat> %1129, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1511 = shufflevector <1 x bfloat> %1276, <1 x bfloat> %1280, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1512 = shufflevector <4 x bfloat> %1511, <4 x bfloat> %1285, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1513 = shufflevector <4 x bfloat> %1512, <4 x bfloat> %1293, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1514 = bitcast <4 x bfloat> %1510 to <4 x i16>
- %1515 = bitcast <4 x bfloat> %1513 to <4 x i16>
- %1516 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1515, <4 x float> %1484, i32 0, i32 0, i32 0)
- %1517 = shufflevector <1 x bfloat> %1277, <1 x bfloat> %1281, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1518 = shufflevector <4 x bfloat> %1517, <4 x bfloat> %1287, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1519 = shufflevector <4 x bfloat> %1518, <4 x bfloat> %1295, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1520 = bitcast <4 x bfloat> %1519 to <4 x i16>
- %1521 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1520, <4 x float> %1489, i32 0, i32 0, i32 0)
- %1522 = shufflevector <1 x bfloat> %1278, <1 x bfloat> %1282, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1523 = shufflevector <4 x bfloat> %1522, <4 x bfloat> %1289, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1524 = shufflevector <4 x bfloat> %1523, <4 x bfloat> %1297, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1525 = bitcast <4 x bfloat> %1524 to <4 x i16>
- %1526 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1525, <4 x float> %1494, i32 0, i32 0, i32 0)
- %1527 = shufflevector <1 x bfloat> %1279, <1 x bfloat> %1283, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1528 = shufflevector <4 x bfloat> %1527, <4 x bfloat> %1291, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1529 = shufflevector <4 x bfloat> %1528, <4 x bfloat> %1299, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1530 = bitcast <4 x bfloat> %1529 to <4 x i16>
- %1531 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1514, <4 x i16> %1530, <4 x float> %1499, i32 0, i32 0, i32 0)
- %1532 = shufflevector <1 x bfloat> %1121, <1 x bfloat> %1123, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1533 = shufflevector <4 x bfloat> %1532, <4 x bfloat> %1127, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1534 = shufflevector <4 x bfloat> %1533, <4 x bfloat> %1131, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1535 = bitcast <4 x bfloat> %1534 to <4 x i16>
- %1536 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1515, <4 x float> %1504, i32 0, i32 0, i32 0)
- %1537 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1520, <4 x float> %1505, i32 0, i32 0, i32 0)
- %1538 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1525, <4 x float> %1506, i32 0, i32 0, i32 0)
- %1539 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1535, <4 x i16> %1530, <4 x float> %1507, i32 0, i32 0, i32 0)
- %1540 = shufflevector <1 x bfloat> %1132, <1 x bfloat> %1134, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1541 = shufflevector <4 x bfloat> %1540, <4 x bfloat> %1137, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1542 = shufflevector <4 x bfloat> %1541, <4 x bfloat> %1141, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1543 = shufflevector <1 x bfloat> %1300, <1 x bfloat> %1304, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1544 = shufflevector <4 x bfloat> %1543, <4 x bfloat> %1309, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1545 = shufflevector <4 x bfloat> %1544, <4 x bfloat> %1317, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1546 = bitcast <4 x bfloat> %1542 to <4 x i16>
- %1547 = bitcast <4 x bfloat> %1545 to <4 x i16>
- %1548 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1547, <4 x float> %1516, i32 0, i32 0, i32 0)
- %1549 = shufflevector <1 x bfloat> %1301, <1 x bfloat> %1305, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1550 = shufflevector <4 x bfloat> %1549, <4 x bfloat> %1311, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1551 = shufflevector <4 x bfloat> %1550, <4 x bfloat> %1319, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1552 = bitcast <4 x bfloat> %1551 to <4 x i16>
- %1553 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1552, <4 x float> %1521, i32 0, i32 0, i32 0)
- %1554 = shufflevector <1 x bfloat> %1302, <1 x bfloat> %1306, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1555 = shufflevector <4 x bfloat> %1554, <4 x bfloat> %1313, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1556 = shufflevector <4 x bfloat> %1555, <4 x bfloat> %1321, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1557 = bitcast <4 x bfloat> %1556 to <4 x i16>
- %1558 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1557, <4 x float> %1526, i32 0, i32 0, i32 0)
- %1559 = shufflevector <1 x bfloat> %1303, <1 x bfloat> %1307, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1560 = shufflevector <4 x bfloat> %1559, <4 x bfloat> %1315, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1561 = shufflevector <4 x bfloat> %1560, <4 x bfloat> %1323, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1562 = bitcast <4 x bfloat> %1561 to <4 x i16>
- %1563 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1546, <4 x i16> %1562, <4 x float> %1531, i32 0, i32 0, i32 0)
- %1564 = shufflevector <1 x bfloat> %1133, <1 x bfloat> %1135, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1565 = shufflevector <4 x bfloat> %1564, <4 x bfloat> %1139, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1566 = shufflevector <4 x bfloat> %1565, <4 x bfloat> %1143, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1567 = bitcast <4 x bfloat> %1566 to <4 x i16>
- %1568 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1547, <4 x float> %1536, i32 0, i32 0, i32 0)
- %1569 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1552, <4 x float> %1537, i32 0, i32 0, i32 0)
- %1570 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1557, <4 x float> %1538, i32 0, i32 0, i32 0)
- %1571 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1567, <4 x i16> %1562, <4 x float> %1539, i32 0, i32 0, i32 0)
- %1572 = shufflevector <1 x bfloat> %1144, <1 x bfloat> %1146, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1573 = shufflevector <4 x bfloat> %1572, <4 x bfloat> %1149, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1574 = shufflevector <4 x bfloat> %1573, <4 x bfloat> %1153, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1575 = shufflevector <1 x bfloat> %1324, <1 x bfloat> %1328, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1576 = shufflevector <4 x bfloat> %1575, <4 x bfloat> %1333, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1577 = shufflevector <4 x bfloat> %1576, <4 x bfloat> %1341, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1578 = bitcast <4 x bfloat> %1574 to <4 x i16>
- %1579 = bitcast <4 x bfloat> %1577 to <4 x i16>
- %1580 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1579, <4 x float> %1548, i32 0, i32 0, i32 0)
- %1581 = shufflevector <1 x bfloat> %1325, <1 x bfloat> %1329, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1582 = shufflevector <4 x bfloat> %1581, <4 x bfloat> %1335, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1583 = shufflevector <4 x bfloat> %1582, <4 x bfloat> %1343, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1584 = bitcast <4 x bfloat> %1583 to <4 x i16>
- %1585 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1584, <4 x float> %1553, i32 0, i32 0, i32 0)
- %1586 = shufflevector <4 x float> %1585, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1587 = shufflevector <1 x bfloat> %1326, <1 x bfloat> %1330, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1588 = shufflevector <4 x bfloat> %1587, <4 x bfloat> %1337, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1589 = shufflevector <4 x bfloat> %1588, <4 x bfloat> %1345, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1590 = bitcast <4 x bfloat> %1589 to <4 x i16>
- %1591 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1590, <4 x float> %1558, i32 0, i32 0, i32 0)
- %1592 = shufflevector <4 x float> %1591, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1593 = shufflevector <1 x bfloat> %1327, <1 x bfloat> %1331, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1594 = shufflevector <4 x bfloat> %1593, <4 x bfloat> %1339, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1595 = shufflevector <4 x bfloat> %1594, <4 x bfloat> %1347, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1596 = bitcast <4 x bfloat> %1595 to <4 x i16>
- %1597 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1578, <4 x i16> %1596, <4 x float> %1563, i32 0, i32 0, i32 0)
- %1598 = shufflevector <4 x float> %1597, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1599 = shufflevector <1 x bfloat> %1145, <1 x bfloat> %1147, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
- %1600 = shufflevector <4 x bfloat> %1599, <4 x bfloat> %1151, <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
- %1601 = shufflevector <4 x bfloat> %1600, <4 x bfloat> %1155, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
- %1602 = bitcast <4 x bfloat> %1601 to <4 x i16>
- %1603 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1579, <4 x float> %1568, i32 0, i32 0, i32 0)
- %1604 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1584, <4 x float> %1569, i32 0, i32 0, i32 0)
- %1605 = shufflevector <4 x float> %1604, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1606 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1590, <4 x float> %1570, i32 0, i32 0, i32 0)
- %1607 = shufflevector <4 x float> %1606, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1608 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> %1602, <4 x i16> %1596, <4 x float> %1571, i32 0, i32 0, i32 0)
- %1609 = shufflevector <4 x float> %1608, <4 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1610 = shufflevector <4 x float> %1580, <4 x float> %1603, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1611 = shufflevector <32 x float> %1610, <32 x float> %1586, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1612 = shufflevector <32 x float> %1611, <32 x float> %1605, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1613 = shufflevector <32 x float> %1612, <32 x float> %1592, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1614 = shufflevector <32 x float> %1613, <32 x float> %1607, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
- %1615 = shufflevector <32 x float> %1614, <32 x float> %1598, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 poison, i32 poison>
- %1616 = shufflevector <32 x float> %1615, <32 x float> %1609, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 34, i32 35>
- %1617 = insertelement <4 x i32> poison, i32 %38, i64 0
- %1618 = insertelement <4 x i32> poison, i32 %32, i64 0
- %1619 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1620 = shufflevector <4 x i32> %1618, <4 x i32> poison, <8 x i32> zeroinitializer
- %1621 = or disjoint <8 x i32> %1619, %1620
- %1622 = shufflevector <4 x i32> %1058, <4 x i32> %1059, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1623 = shufflevector <4 x i32> %1617, <4 x i32> poison, <8 x i32> zeroinitializer
- %1624 = icmp samesign ult <8 x i32> %1622, %1623
- br label %1625
-
-1625: ; preds = %1053, %1625
- %1626 = phi i32 [ 0, %1053 ], [ %1676, %1625 ]
- %1627 = shl nuw nsw i32 %1626, 3
- %1628 = extractelement <32 x float> %1616, i32 %1627
- %1629 = insertelement <4 x float> poison, float %1628, i64 0
- %1630 = or disjoint i32 %1627, 1
- %1631 = extractelement <32 x float> %1616, i32 %1630
- %1632 = insertelement <4 x float> %1629, float %1631, i64 1
- %1633 = or disjoint i32 %1627, 2
- %1634 = extractelement <32 x float> %1616, i32 %1633
- %1635 = insertelement <4 x float> %1632, float %1634, i64 2
- %1636 = or disjoint i32 %1627, 3
- %1637 = extractelement <32 x float> %1616, i32 %1636
- %1638 = insertelement <4 x float> %1635, float %1637, i64 3
- %1639 = or disjoint i32 %1627, 4
- %1640 = extractelement <32 x float> %1616, i32 %1639
- %1641 = insertelement <4 x float> poison, float %1640, i64 0
- %1642 = or disjoint i32 %1627, 5
- %1643 = extractelement <32 x float> %1616, i32 %1642
- %1644 = insertelement <4 x float> %1641, float %1643, i64 1
- %1645 = or disjoint i32 %1627, 6
- %1646 = extractelement <32 x float> %1616, i32 %1645
- %1647 = insertelement <4 x float> %1644, float %1646, i64 2
- %1648 = or disjoint i32 %1627, 7
- %1649 = extractelement <32 x float> %1616, i32 %1648
- %1650 = insertelement <4 x float> %1647, float %1649, i64 3
- %1651 = bitcast <4 x float> %1638 to <4 x i32>
- %1652 = bitcast <4 x float> %1650 to <4 x i32>
- %1653 = shl nuw nsw i32 %1626, 4
- %.reass = add nuw nsw i32 %1653, %154
- %1654 = icmp samesign ult i32 %.reass, %36
- %1655 = insertelement <4 x i1> poison, i1 %1654, i64 0
- %1656 = add nuw nsw i32 %.reass, %34
- %1657 = mul nuw nsw i32 %1656, 2016
- %1658 = insertelement <4 x i32> poison, i32 %1657, i64 0
- %1659 = shufflevector <4 x i32> %1658, <4 x i32> poison, <8 x i32> zeroinitializer
- %1660 = add nuw nsw <8 x i32> %1621, %1659
- %1661 = shufflevector <4 x i1> %1655, <4 x i1> poison, <8 x i32> zeroinitializer
- %1662 = and <8 x i1> %1624, %1661
- %1663 = shufflevector <4 x float> %1638, <4 x float> %1650, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1664 = fcmp uno <8 x float> %1663, zeroinitializer
- %1665 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1666 = add <8 x i32> %1665, splat (i32 32767)
- %1667 = shufflevector <4 x i32> %1651, <4 x i32> %1652, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
- %1668 = lshr <8 x i32> %1667, splat (i32 16)
- %1669 = and <8 x i32> %1668, splat (i32 1)
- %1670 = add <8 x i32> %1666, %1669
- %1671 = lshr <8 x i32> %1670, splat (i32 16)
- %1672 = trunc nuw <8 x i32> %1671 to <8 x i16>
- %1673 = bitcast <8 x i16> %1672 to <8 x bfloat>
- %1674 = select <8 x i1> %1664, <8 x bfloat> splat (bfloat 0xR7FC0), <8 x bfloat> %1673
- %1675 = getelementptr bfloat, ptr addrspace(7) %1054, <8 x i32> %1660
- tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %1674, <8 x ptr addrspace(7)> align 2 %1675, <8 x i1> %1662)
- %1676 = add nuw nsw i32 %1626, 1
- %exitcond.not = icmp eq i32 %1676, 4
- br i1 %exitcond.not, label %1677, label %1625
+ %307 = and i32 %78, 56
+ %308 = getelementptr bfloat, ptr addrspace(3) null, i32 %307
+ store <8 x bfloat> %306, ptr addrspace(3) %308, align 8
+ %309 = trunc i32 %85 to i1
+ %310 = select i1 %309, <8 x bfloat> %86, <8 x bfloat> zeroinitializer
+ %311 = getelementptr bfloat, ptr addrspace(3) %1, i32 %307
+ store <8 x bfloat> %310, ptr addrspace(3) %311, align 8
+ %312 = trunc i32 %91 to i1
+ %313 = select i1 %312, <8 x bfloat> %92, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %313, ptr addrspace(3) null, align 8
+ %314 = trunc i32 %97 to i1
+ %315 = select i1 %314, <8 x bfloat> %98, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %315, ptr addrspace(3) %1, align 8
+ %316 = trunc i32 %100 to i1
+ %317 = select i1 %316, <8 x bfloat> %101, <8 x bfloat> zeroinitializer
+ %318 = and i32 %78, 1
+ %319 = getelementptr bfloat, ptr addrspace(3) %3, i32 %318
+ store <8 x bfloat> %317, ptr addrspace(3) %319, align 8
+ %320 = trunc i32 %105 to i1
+ %321 = select i1 %320, <8 x bfloat> %106, <8 x bfloat> zeroinitializer
+ %322 = getelementptr bfloat, ptr addrspace(3) %4, i32 %318
+ store <8 x bfloat> %321, ptr addrspace(3) %322, align 8
+ %323 = trunc i32 %107 to i1
+ %324 = select i1 %323, <8 x bfloat> %108, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %324, ptr addrspace(3) null, align 8
+ %325 = trunc i32 %109 to i1
+ %326 = select i1 %325, <8 x bfloat> %110, <8 x bfloat> zeroinitializer
+ %327 = getelementptr bfloat, ptr addrspace(3) null, i32 %318
+ store <8 x bfloat> %326, ptr addrspace(3) %327, align 8
+ %328 = trunc i32 %112 to i1
+ %329 = select i1 %328, <8 x bfloat> %113, <8 x bfloat> zeroinitializer
+ %330 = getelementptr bfloat, ptr addrspace(3) %5, i32 %318
+ store <8 x bfloat> %329, ptr addrspace(3) %330, align 8
+ %331 = trunc i32 %115 to i1
+ %332 = select i1 %331, <8 x bfloat> %116, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %332, ptr addrspace(3) %1, align 8
+ %333 = trunc i32 %118 to i1
+ %334 = select i1 %333, <8 x bfloat> %119, <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %334, ptr addrspace(3) null, align 8
+ %335 = trunc i32 %121 to i1
+ %336 = select i1 %335, <8 x bfloat> splat (bfloat 0xR3F80), <8 x bfloat> zeroinitializer
+ store <8 x bfloat> %122, ptr addrspace(3) %1, align 8
+ %337 = bitcast <8 x bfloat> %336 to i128
+ %338 = trunc i128 %337 to i16
+ %339 = bitcast i16 %338 to <1 x bfloat>
+ %340 = icmp ult i32 %76, 1
+ br i1 %340, label %73, label %341
-1677: ; preds = %1625
- fence syncscope("workgroup") release, !mmra !3
- tail call void @llvm.amdgcn.s.barrier()
- fence syncscope("workgroup") acquire, !mmra !3
- ret void
+341: ; preds = %341, %73
+ %342 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16> zeroinitializer, <4 x i16> zeroinitializer, <4 x float> splat (float 0x7FF8000000000000), i32 0, i32 0, i32 0)
+ %343 = shufflevector <4 x float> %342, <4 x float> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %344 = extractelement <32 x float> %343, i32 %0
+ %345 = insertelement <4 x float> zeroinitializer, float %344, i64 0
+ %346 = shufflevector <4 x float> %345, <4 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %347 = fcmp uno <8 x float> %346, zeroinitializer
+ %348 = select <8 x i1> %347, <8 x bfloat> splat (bfloat 0xR3F80), <8 x bfloat> zeroinitializer
+ tail call void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat> %348, <8 x ptr addrspace(7)> zeroinitializer, <8 x i1> %66)
+ br label %341
}
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
-declare void @llvm.assume(i1 noundef) #2
-
-; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) readnone, i16, i64, i32) #3
-
-; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare i32 @llvm.umin.i32(i32, i32) #3
-
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
-declare noundef i32 @llvm.amdgcn.workgroup.id.x() #1
-
-; Function Attrs: alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
-declare i32 @llvm.smin.i32(i32, i32) #3
-
-; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
-declare void @llvm.amdgcn.s.barrier() #4
-
-; Function Attrs: alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
-declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16bf16.1k(<4 x i16>, <4 x i16>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #5
-
-; Function Attrs: alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write)
-declare void @llvm.masked.scatter.v8bf16.v8p7(<8 x bfloat>, <8 x ptr addrspace(7)>, <8 x i1>) #6
-
-; Function Attrs: alwaysinline convergent mustprogress nocallback nofree nounwind willreturn
-declare void @llvm.amdgcn.sched.barrier(i32 immarg) #4
-
-attributes #0 = { alwaysinline nofree norecurse nounwind "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="256,256" "amdgpu-no-cluster-id-x" "amdgpu-no-cluster-id-y" "amdgpu-no-cluster-id-z" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-flat-scratch-init" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" }
-attributes #1 = { alwaysinline mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #2 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
-attributes #3 = { alwaysinline mustprogress nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
-attributes #4 = { alwaysinline convergent mustprogress nocallback nofree nounwind willreturn }
-attributes #5 = { alwaysinline convergent mustprogress nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
-attributes #6 = { alwaysinline mustprogress nocallback nofree nosync nounwind willreturn memory(write) }
-
-!llvm.module.flags = !{!0, !1}
-
-!0 = !{i32 2, !"Debug Info Version", i32 3}
-!1 = !{i32 1, !"amdhsa_code_object_version", i32 500}
-!2 = !{i32 256, i32 1, i32 1}
-!3 = !{!"amdgpu-synchronize-as", !"local"}
+attributes #0 = { "amdgpu-flat-work-group-size"="256,256" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(write) }
+attributes #5 = { convergent nocallback nofree nounwind willreturn }
>From a435c90e8b2718c402496cc63c536d609643098d Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Tue, 27 Jan 2026 08:49:52 -0600
Subject: [PATCH 09/13] Convert rewrite cost analysis to use double rather than
int64 to clean up logic
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 48 ++++++++-------------
llvm/lib/Target/AMDGPU/GCNSchedStrategy.h | 2 +-
2 files changed, 18 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 277ac7d4ecf4c..e1ad56a951b2e 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1330,11 +1330,11 @@ bool RewriteMFMAFormStage::initGCNSchedStage() {
if (!initHeuristics(RewriteCands, CopyForUse, CopyForDef))
return false;
- int64_t Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
+ double Cost = getRewriteCost(RewriteCands, CopyForUse, CopyForDef);
// If we haven't found the beneficial conditions, prefer the VGPR form which
// may result in less cross RC copies.
- if (Cost > 0)
+ if (Cost > 0.0)
return false;
return rewrite(RewriteCands);
@@ -2251,15 +2251,14 @@ bool RewriteMFMAFormStage::initHeuristics(
return Changed;
}
-int64_t RewriteMFMAFormStage::getRewriteCost(
+double RewriteMFMAFormStage::getRewriteCost(
const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef) {
MachineBlockFrequencyInfo *MBFI = DAG.MBFI;
- int64_t BestSpillCost = 0;
- int64_t Cost = 0;
- uint64_t EntryFreq = MBFI->getEntryFreq().getFrequency();
+ double BestSpillCost = 0.0;
+ double Cost = 0.0;
std::pair<unsigned, unsigned> MaxVectorRegs =
ST.getMaxNumVectorRegs(MF.getFunction());
@@ -2283,28 +2282,18 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
unsigned SpillCostAfter = PressureAfter.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
- uint64_t BlockFreq =
- MBFI->getBlockFreq(DAG.Regions[Region].first->getParent())
- .getFrequency();
-
- bool RelativeFreqIsDenom = EntryFreq > BlockFreq;
- uint64_t RelativeFreq = EntryFreq && BlockFreq
- ? (RelativeFreqIsDenom ? EntryFreq / BlockFreq
- : BlockFreq / EntryFreq)
- : 1;
+ MachineBasicBlock *MBB = DAG.Regions[Region].first->getParent();
+ double BlockFreq = MBFI->getBlockFreqRelativeToEntryBlock(MBB);
// This assumes perfect spilling / splitting -- using one spill / copy
// instruction and one restoreFrom / copy for each excess register,
- int64_t SpillCost = ((int)SpillCostAfter - (int)SpillCostBefore) * 2;
+ double SpillCost = ((double)SpillCostAfter - (double)SpillCostBefore) * 2;
// Also account for the block frequency.
- if (RelativeFreqIsDenom)
- SpillCost /= (int64_t)RelativeFreq;
- else
- SpillCost *= (int64_t)RelativeFreq;
+ SpillCost *= BlockFreq;
// If we have increased spilling in any block, just bail.
- if (SpillCost > 0)
+ if (SpillCost > 0.0)
return SpillCost;
if (SpillCost < BestSpillCost)
@@ -2314,31 +2303,28 @@ int64_t RewriteMFMAFormStage::getRewriteCost(
// Set the cost to the largest decrease in spill cost in order to not double
// count spill reductions.
Cost = BestSpillCost;
- assert(Cost <= 0);
+ assert(Cost <= 0.0);
- unsigned CopyCost = 0;
+ double CopyCost = 0.0;
// For each CopyForDef, increase the cost by the register size while
// accounting for block frequency.
for (MachineInstr *DefMI : CopyForDef) {
Register DefReg = DefMI->getOperand(0).getReg();
- uint64_t DefFreq =
- EntryFreq
- ? MBFI->getBlockFreq(DefMI->getParent()).getFrequency() / EntryFreq
- : 1;
+ MachineBasicBlock *DefMBB = DefMI->getParent();
+ double DefFreq = MBFI->getBlockFreqRelativeToEntryBlock(DefMBB);
const TargetRegisterClass *RC = DAG.MRI.getRegClass(DefReg);
- CopyCost += RC->getCopyCost() * DefFreq;
+ CopyCost += (double) RC->getCopyCost() * DefFreq;
}
// Account for CopyForUse copies in each block that the register is used.
for (auto &[UseBlock, UseRegs] : CopyForUse) {
- uint64_t UseFreq =
- EntryFreq ? MBFI->getBlockFreq(UseBlock).getFrequency() / EntryFreq : 1;
+ uint64_t UseFreq = MBFI->getBlockFreqRelativeToEntryBlock(UseBlock);
for (Register UseReg : UseRegs) {
const TargetRegisterClass *RC = DAG.MRI.getRegClass(UseReg);
- CopyCost += RC->getCopyCost() * UseFreq;
+ CopyCost += (double) RC->getCopyCost() * UseFreq;
}
}
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 6b6a40365b52d..0f73cc4fda10f 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -442,7 +442,7 @@ class RewriteMFMAFormStage : public GCNSchedStage {
/// Calculate the rewrite cost and undo the state change (e.g. rewriting) done
/// in initHeuristics. Uses \p CopyForUse and \p CopyForDef to calculate copy
/// costs, and \p RewriteCands to undo rewriting.
- int64_t getRewriteCost(
+ double getRewriteCost(
const std::vector<std::pair<MachineInstr *, unsigned>> &RewriteCands,
const DenseMap<MachineBasicBlock *, std::set<Register>> &CopyForUse,
const SmallPtrSetImpl<MachineInstr *> &CopyForDef);
>From 7e966261ce9693809d9e4ee83a5754b468cc62a0 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Mon, 2 Feb 2026 08:58:03 -0600
Subject: [PATCH 10/13] Add some additional debug prints
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index e1ad56a951b2e..c3e24d9a02f07 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2273,6 +2273,8 @@ double RewriteMFMAFormStage::getRewriteCost(
GCNRegPressure &PressureBefore = DAG.Pressure[Region];
unsigned SpillCostBefore = PressureBefore.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Region " << Region
+ << " spill cost before: " << SpillCostBefore << "\n");
// For the cases we care about (i.e. ArchVGPR usage is greater than the
// addressable limit), rewriting alone should bring pressure to manageable
@@ -2281,6 +2283,8 @@ double RewriteMFMAFormStage::getRewriteCost(
GCNRegPressure PressureAfter = DAG.getRealRegPressure(Region);
unsigned SpillCostAfter = PressureAfter.getVGPRSpills(
MF, ArchVGPRThreshold, AGPRThreshold, CombinedThreshold);
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Region " << Region
+ << " spill cost after: " << SpillCostAfter << "\n");
MachineBasicBlock *MBB = DAG.Regions[Region].first->getParent();
double BlockFreq = MBFI->getBlockFreqRelativeToEntryBlock(MBB);
@@ -2302,32 +2306,37 @@ double RewriteMFMAFormStage::getRewriteCost(
// Set the cost to the largest decrease in spill cost in order to not double
// count spill reductions.
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: BestSpillCost: " << BestSpillCost << "\n");
Cost = BestSpillCost;
assert(Cost <= 0.0);
- double CopyCost = 0.0;
-
// For each CopyForDef, increase the cost by the register size while
// accounting for block frequency.
+ double DefCopyCost = 0.0;
for (MachineInstr *DefMI : CopyForDef) {
Register DefReg = DefMI->getOperand(0).getReg();
MachineBasicBlock *DefMBB = DefMI->getParent();
double DefFreq = MBFI->getBlockFreqRelativeToEntryBlock(DefMBB);
const TargetRegisterClass *RC = DAG.MRI.getRegClass(DefReg);
- CopyCost += (double) RC->getCopyCost() * DefFreq;
+ DefCopyCost += (double)RC->getCopyCost() * DefFreq;
}
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Def copy Costs: " << DefCopyCost << "\n");
// Account for CopyForUse copies in each block that the register is used.
+ double UseCopyCost = 0.0;
for (auto &[UseBlock, UseRegs] : CopyForUse) {
uint64_t UseFreq = MBFI->getBlockFreqRelativeToEntryBlock(UseBlock);
for (Register UseReg : UseRegs) {
const TargetRegisterClass *RC = DAG.MRI.getRegClass(UseReg);
- CopyCost += (double) RC->getCopyCost() * UseFreq;
+ UseCopyCost += (double)RC->getCopyCost() * UseFreq;
}
}
+ LLVM_DEBUG(dbgs() << "RewriteMFMA: Use copy Costs: " << UseCopyCost << "\n");
+ double CopyCost = UseCopyCost + DefCopyCost;
+
// Reset the classes that were changed to AGPR for better RB analysis.
// We must do rewriting after copy-insertion, as some defs of the register
// may require VGPR. Additionally, if we bail out and don't perform the
>From 6f7efc471fc470a69d2ba69cbfa8ba4cd2b80b47 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Tue, 10 Feb 2026 08:50:46 -0600
Subject: [PATCH 11/13] Reenable AMDGPU MFMA rewrite scheduler stage.
After performance testing, it was determined that the large number of copies that
are inserted outside the loop are more than offset by better allocation within
the loop as a result of the rewrite. Additionally, there are two other fairly
minor changes in this commit:
- Fix for upstream issue #177696. The need for this fix will go away when we
remove the need to do an early conversion to MFMA form in order to get
proper spill cost analysis of the potential rewritten code.
- Convert cost logic to double from int64. This cleans up the logic a bit.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index c3e24d9a02f07..79603ccbb5ead 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,7 +2184,6 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
-
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
>From f94f878364e270cb7b16dde61b771706461d7a93 Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Wed, 11 Feb 2026 17:14:58 -0600
Subject: [PATCH 12/13] Reenable AMDGPU MFMA rewrite scheduler stage.
After performance testing, it was determined that the large number of
copies that are inserted outside the loop are more than offset by better
allocation within the loop as a result of the rewrite. Additionally,
there is a minor cleanup of the cost analysis code.
This reverts commit e47677c3bc38af91e237a390dc09545939be5cdb.
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 79603ccbb5ead..688e1ddd54fa2 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -2184,6 +2184,7 @@ bool RewriteMFMAFormStage::initHeuristics(
int ReplacementOp = AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode());
assert(ReplacementOp != -1);
+
RewriteCands.push_back({&MI, MI.getOpcode()});
MI.setDesc(TII->get(ReplacementOp));
@@ -2350,8 +2351,6 @@ double RewriteMFMAFormStage::getRewriteCost(
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
- if (!Src2->isReg())
- continue;
// Have to get src types separately since subregs may cause C and D
// registers to be different types even though the actual operand is
>From e36d4cdcac2ee293d04ce2bb822fcb4ca5b1743b Mon Sep 17 00:00:00 2001
From: Tony Linthicum <tlinthic at gmail.com>
Date: Thu, 12 Feb 2026 12:27:18 -0600
Subject: [PATCH 13/13] Fix rebase merge errors
---
llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 688e1ddd54fa2..783a6f8440d95 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -101,7 +101,7 @@ static cl::opt<bool> PrintMaxRPRegUsageAfterScheduler(
static cl::opt<bool> DisableRewriteMFMAFormSchedStage(
"amdgpu-disable-rewrite-mfma-form-sched-stage", cl::Hidden,
- cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(true));
+ cl::desc("Disable rewrie mfma rewrite scheduling stage"), cl::init(false));
const unsigned ScheduleMetrics::ScaleFactor = 100;
@@ -2351,6 +2351,8 @@ double RewriteMFMAFormStage::getRewriteCost(
MachineOperand *Src2 = TII->getNamedOperand(*MI, AMDGPU::OpName::src2);
assert(Src2);
+ if (!Src2->isReg())
+ continue;
// Have to get src types separately since subregs may cause C and D
// registers to be different types even though the actual operand is
More information about the llvm-commits
mailing list