[llvm] [MachineScheduler] Fix stale read-undef flags when rescheduling physreg copies (PR #227897)
Nick Riasanovsky via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 5 17:01:00 PDT 2026
https://github.com/njriasan updated https://github.com/llvm/llvm-project/pull/227897
>From 8240e523d740cbf734589801223ef0e6e4ce4b0d Mon Sep 17 00:00:00 2001
From: Nick Riasanovsky <njriasan at meta.com>
Date: Wed, 30 Sep 2026 16:00:12 -0700
Subject: [PATCH 1/3] Fix stale flag bug
---
llvm/lib/CodeGen/MachineScheduler.cpp | 53 +++++
.../reschedule-physreg-copy-stale-undef.ll | 210 ++++++++++++++++++
.../AMDGPU/shufflevector-physreg-copy.ll | 8 +-
3 files changed, 267 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index facf741f12400..e71736d2fd1b0 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -4266,7 +4266,60 @@ void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
continue;
LLVM_DEBUG(dbgs() << " Rescheduling physreg copy ";
DAG->dumpNode(*Dep.getSUnit()));
+ // Moving an already-scheduled copy can invalidate read-undef flags: lanes
+ // that were dead when the flags were set may now be live across the
+ // subregister defs the copy moves past (or after the copy itself). A stale
+ // read-undef makes a later DAG build drop true data dependencies, which
+ // can orphan the moved copy and crash LiveIntervals. Recompute the affected
+ // flags from LiveIntervals after the move.
+ LiveIntervals *LIS = DAG->getLIS();
+ bool FixUndef = LIS && DAG->hasVRegLiveness() && shouldTrackLaneMasks();
+ MachineBasicBlock::iterator OldNext = Copy->getParent()->end();
+ SlotIndex OldIdx;
+ SmallVector<Register, 8> CopyVRegs;
+ if (FixUndef) {
+ OldIdx = LIS->getInstructionIndex(*Copy);
+ OldNext = std::next(Copy->getIterator());
+ for (const MachineOperand &MO : Copy->operands())
+ if (MO.isReg() && MO.getReg().isVirtual())
+ CopyVRegs.push_back(MO.getReg());
+ }
DAG->moveInstruction(Copy, InsertPos);
+ if (!FixUndef || CopyVRegs.empty())
+ continue;
+
+ auto FixVRegUndef = [&](MachineInstr &MI) {
+ bool HasAffectedDef = false;
+ for (MachineOperand &MO : MI.all_defs()) {
+ if (!MO.getReg().isVirtual() || MO.getSubReg() == 0 ||
+ !llvm::is_contained(CopyVRegs, MO.getReg()))
+ continue;
+ MO.setIsUndef(false);
+ HasAffectedDef = true;
+ }
+ if (!HasAffectedDef)
+ return;
+
+ // adjustLaneLiveness only adds read-undef flags, so clear them first.
+ // Re-add the valid flags to avoid introducing reads of undefined lanes.
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ RegisterOperands RegOpers;
+ RegOpers.collect(MI, *TRI, MRI, /*TrackLaneMasks=*/true,
+ /*IgnoreDead=*/false);
+ RegOpers.adjustLaneLiveness(*LIS, MRI, MI);
+ };
+ FixVRegUndef(*Copy);
+ MachineBasicBlock::iterator NewIt = Copy->getIterator();
+ SlotIndex NewIdx = LIS->getInstructionIndex(*Copy);
+ if (NewIdx < OldIdx) {
+ for (auto I = std::next(NewIt); I != OldNext; ++I)
+ if (!I->isDebugInstr())
+ FixVRegUndef(*I);
+ } else if (OldIdx < NewIdx) {
+ for (auto I = OldNext; I != NewIt; ++I)
+ if (!I->isDebugInstr())
+ FixVRegUndef(*I);
+ }
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll b/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll
new file mode 100644
index 0000000000000..193b0d85387f2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll
@@ -0,0 +1,210 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-machineinstrs %s -o - | FileCheck %s
+;
+; Rescheduling an already-scheduled physreg copy next to its user must not
+; leave stale read-undef flags behind. A stale flag makes a later GCN
+; reschedule stage drop a true data dependency, orphaning the copy and
+; crashing in LiveIntervals::HMEditor::handleMoveDown with "Must have
+; following segment".
+; Recomputing the flags must also preserve read-undef on the first subregister
+; definition, so that it does not read undefined lanes.
+;
+; CHECK-LABEL: kernel:
+; CHECK: v_cvt_scalef32_pk32_fp6_bf16 v[0:5], v[8:23]
+; CHECK: ds_write_b128
+; CHECK: s_endpgm
+
+target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
+target triple = "amdgpu9.50-amd-amdhsa"
+
+define amdgpu_kernel void @kernel(ptr addrspace(1) inreg %0, ptr addrspace(1) %1, float %2, float %3, float %4, float %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13, i1 %14, <8 x half> %15, float %16, float %17, float %18, <8 x half> %19, half %20, <8 x half> %21, half %22, float %23, i32 %24, <2 x float> %25, <2 x float> %26, i64 %27, i32 %28) {
+ %30 = tail call i32 @llvm.amdgcn.workgroup.id.x()
+ %31 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 0, i32 0, i32 0)
+ %32 = fmul float %9, 0.000000e+00
+ %33 = fmul float %7, 0.000000e+00
+ %34 = fmul float %2, 0.000000e+00
+ %35 = fmul float %8, 0.000000e+00
+ %36 = fmul float %6, 0.000000e+00
+ %37 = fmul float %5, 0.000000e+00
+ %38 = fmul float %4, 0.000000e+00
+ %39 = fmul float %3, 0.000000e+00
+ %40 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %0, i16 0, i64 0, i32 0)
+ %41 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) %40, i32 0, i32 0, i32 0)
+ %42 = zext i1 %14 to i32
+ %43 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %42, i32 0, i32 0)
+ %44 = bitcast <4 x i32> %43 to <8 x half>
+ %45 = extractelement <8 x half> %44, i64 3
+ %46 = bitcast <4 x i32> %41 to <8 x half>
+ %47 = extractelement <8 x half> %46, i64 0
+ %48 = fpext half %47 to float
+ %49 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) %40, i32 1, i32 0, i32 0)
+ %50 = tail call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) %1, i16 0, i64 0, i32 0)
+ %51 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) %50, i32 0, i32 0, i32 0)
+ %52 = bitcast <4 x i32> %49 to <8 x half>
+ %53 = extractelement <8 x half> %52, i64 2
+ %54 = bitcast <4 x i32> %51 to <8 x half>
+ %55 = extractelement <8 x half> %54, i64 2
+ %56 = extractelement <8 x half> %54, i64 5
+ %57 = fpext half %56 to float
+ %58 = fadd float %57, 0.000000e+00
+ %59 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 -2147483648, i32 0, i32 0)
+ %60 = select i1 %14, float %2, float 0.000000e+00
+ %61 = fadd float %11, 0.000000e+00
+ %62 = extractelement <8 x half> %44, i64 0
+ %63 = fpext half %62 to float
+ %64 = fadd float %61, %63
+ %65 = extractelement <8 x half> %52, i64 0
+ %66 = fpext half %65 to float
+ %67 = extractelement <8 x half> %54, i64 0
+ %68 = fpext half %67 to float
+ %69 = fadd float %66, %68
+ %70 = bitcast <4 x i32> %59 to <8 x half>
+ %71 = extractelement <8 x half> %70, i64 0
+ %72 = fpext half %71 to float
+ %73 = fadd float %5, 0.000000e+00
+ %74 = fadd float %9, 0.000000e+00
+ %75 = extractelement <8 x half> %44, i64 4
+ %76 = fpext half %75 to float
+ %77 = fadd float %74, %76
+ %78 = fadd float %10, 0.000000e+00
+ %79 = extractelement <8 x half> %70, i64 4
+ %80 = fpext half %79 to float
+ %81 = fadd float %12, 0.000000e+00
+ %82 = extractelement <8 x half> %44, i64 7
+ %83 = fpext half %82 to float
+ %84 = extractelement <8 x half> %70, i64 7
+ %85 = fadd float %4, 0.000000e+00
+ %86 = fadd float %7, 0.000000e+00
+ %87 = fpext half %45 to float
+ %88 = fadd float %86, %87
+ %89 = fadd float %85, %88
+ %90 = fadd float %8, 0.000000e+00
+ %91 = fadd float %3, 0.000000e+00
+ %92 = fadd float %48, 0.000000e+00
+ %93 = fadd float %91, %92
+ %94 = fpext half %53 to float
+ %95 = fpext half %55 to float
+ %96 = fadd float %94, %95
+ %97 = fadd float %93, %96
+ %98 = fadd float %60, %64
+ %99 = fadd float %69, %72
+ %100 = fadd float %98, %99
+ %101 = icmp slt i32 %28, 0
+ %102 = select i1 %101, float +qnan, float %4
+ %103 = fadd float %100, %102
+ %104 = select i1 %14, float 1.000000e+00, float 0.000000e+00
+ %105 = fadd float %2, 0.000000e+00
+ %106 = fadd float %104, %105
+ %107 = extractelement <8 x half> %54, i64 6
+ %108 = fpext half %107 to float
+ %109 = fadd float %18, %3
+ %110 = fadd float %109, %58
+ %111 = fadd float %73, %77
+ %112 = fadd float %78, %80
+ %113 = fadd float %111, %112
+ %114 = fadd float %89, %90
+ %115 = fadd float %97, %103
+ %116 = fadd float %114, %115
+ %117 = fadd float %113, %116
+ %118 = fadd float %6, 0.000000e+00
+ %119 = fadd float %81, %83
+ %120 = fadd float %118, %119
+ %121 = fadd float %16, 0.000000e+00
+ %122 = fpext half %84 to float
+ %123 = fadd float %121, %122
+ %124 = fadd float %120, %123
+ %125 = fadd float %106, %108
+ %126 = fadd float %110, %117
+ %127 = fadd float %125, %126
+ %128 = fadd float %124, %127
+ store float %128, ptr addrspace(3) null, align 4
+ %129 = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %130 = and i32 %129, 255
+ %131 = getelementptr [4 x i8], ptr addrspace(3) null, i32 %130
+ store float 0.000000e+00, ptr addrspace(3) %131, align 4
+ %132 = fadd float %32, %33
+ %133 = fadd float %34, %132
+ %134 = fadd float %35, %133
+ %135 = fadd float %36, %134
+ %136 = fadd float %37, %135
+ %137 = fadd float %38, %136
+ %138 = fadd float %39, %137
+ store float %138, ptr addrspace(3) null, align 4
+ %139 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 0, i32 0, i32 0)
+ %140 = bitcast <4 x i32> %139 to <8 x half>
+ %141 = shufflevector <8 x half> %46, <8 x half> zeroinitializer, <2 x i32> <i32 4, i32 5>
+ %142 = fpext <2 x half> %141 to <2 x float>
+ %143 = shufflevector <8 x half> %140, <8 x half> zeroinitializer, <2 x i32> <i32 4, i32 5>
+ %144 = fpext <2 x half> %143 to <2 x float>
+ %145 = fadd <2 x float> %142, %144
+ %146 = bitcast <4 x i32> %31 to <8 x half>
+ %147 = shufflevector <8 x half> %146, <8 x half> zeroinitializer, <2 x i32> <i32 4, i32 5>
+ %148 = fpext <2 x half> %147 to <2 x float>
+ %149 = fadd <2 x float> %145, %148
+ %150 = load <4 x i32>, ptr addrspace(3) null, align 16
+ %151 = fptrunc <2 x float> %149 to <2 x bfloat>
+ %152 = bitcast <2 x bfloat> %151 to i32
+ %153 = insertelement <1 x i32> zeroinitializer, i32 %152, i64 0
+ store <1 x i32> %153, ptr addrspace(3) null, align 4
+ %154 = load <4 x i32>, ptr addrspace(3) null, align 16
+ %155 = extractelement <4 x i32> %154, i64 3
+ tail call void @llvm.amdgcn.wave.barrier()
+ %156 = load <4 x i32>, ptr addrspace(3) null, align 16
+ %157 = extractelement <4 x i32> %156, i64 3
+ %158 = fadd <2 x float> %25, %26
+ %159 = fptrunc <2 x float> %158 to <2 x bfloat>
+ %160 = bitcast <2 x bfloat> %159 to i32
+ %161 = insertelement <1 x i32> zeroinitializer, i32 %160, i64 0
+ store <1 x i32> %161, ptr addrspace(3) null, align 4
+ %162 = extractelement <4 x i32> %150, i64 0
+ %163 = extractelement <4 x i32> %156, i64 0
+ %164 = extractelement <4 x i32> %154, i64 0
+ %165 = load <4 x i32>, ptr addrspace(3) null, align 16
+ %166 = extractelement <4 x i32> %165, i64 0
+ %167 = extractelement <4 x i32> %150, i64 2
+ %168 = extractelement <4 x i32> %156, i64 2
+ %169 = extractelement <4 x i32> %154, i64 2
+ %170 = extractelement <4 x i32> %150, i64 1
+ %171 = extractelement <4 x i32> %156, i64 1
+ %172 = extractelement <4 x i32> %154, i64 1
+ %173 = extractelement <4 x i32> %165, i64 1
+ %174 = extractelement <4 x i32> %150, i64 3
+ %175 = tail call { i64, i64, i64 } asm "v_cvt_scalef32_pk32_fp6_bf16 v[0:5], v[8:23], $19", "=&{v[0:1]},=&{v[2:3]},=&{v[4:5]},{v8},{v9},{v10},{v11},{v12},{v13},{v14},{v15},{v16},{v17},{v18},{v19},{v20},{v21},{v22},{v23},v"(i32 %162, i32 %163, i32 %164, i32 %166, i32 %167, i32 %168, i32 %169, i32 0, i32 %170, i32 %171, i32 %172, i32 %173, i32 %174, i32 %157, i32 %155, i32 0, float 0.000000e+00)
+ %176 = extractvalue { i64, i64, i64 } %175, 2
+ %177 = insertelement <2 x i64> zeroinitializer, i64 %176, i64 0
+ store <2 x i64> %177, ptr addrspace(3) null, align 16
+ tail call void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64 %27, ptr addrspace(8) null, i32 -2147483648, i32 0, i32 0)
+ ret void
+
+; uselistorder directives
+ uselistorder <8 x half> %44, { 0, 1, 3, 2 }
+ uselistorder <8 x half> %52, { 1, 0 }
+ uselistorder <8 x half> %54, { 0, 3, 1, 2 }
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #0
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.amdgcn.workgroup.id.x() #0
+
+; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+declare ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p8.p1(ptr addrspace(1) readnone, i16, i64, i32) #1
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: read)
+declare <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) readonly captures(none), i32, i32, i32 immarg) #2
+
+; Function Attrs: convergent nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.wave.barrier() #3
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: write)
+declare void @llvm.amdgcn.raw.ptr.buffer.store.i64(i64, ptr addrspace(8) writeonly captures(none), i32, i32, i32 immarg) #4
+
+; uselistorder directives
+uselistorder ptr @llvm.amdgcn.make.buffer.rsrc.p8.p1, { 1, 0 }
+uselistorder ptr @llvm.amdgcn.raw.ptr.buffer.load.v4i32, { 6, 5, 4, 3, 2, 1, 0 }
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #1 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+attributes #3 = { convergent nocallback nofree nounwind willreturn }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
index 38b8e4026a8ef..0da7fa6eaf6d1 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector-physreg-copy.ll
@@ -735,10 +735,10 @@ define i32 @shufflevector_v4i32_3210_physreg_even_vgpr_quad_copy_other_use_elt(p
; GFX900-NEXT: ;;#ASMSTART
; GFX900-NEXT: ; def v4, v5, v6, v7
; GFX900-NEXT: ;;#ASMEND
-; GFX900-NEXT: v_mov_b32_e32 v9, v5
-; GFX900-NEXT: v_mov_b32_e32 v8, v6
-; GFX900-NEXT: v_mov_b32_e32 v10, v4
-; GFX900-NEXT: global_store_dwordx4 v0, v[7:10], s[16:17]
+; GFX900-NEXT: v_mov_b32_e32 v3, v5
+; GFX900-NEXT: v_mov_b32_e32 v2, v6
+; GFX900-NEXT: v_mov_b32_e32 v1, v7
+; GFX900-NEXT: global_store_dwordx4 v0, v[1:4], s[16:17]
; GFX900-NEXT: v_mov_b32_e32 v0, v6
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
>From e26507ace0f65dafd3e911b35fb4b113551822bc Mon Sep 17 00:00:00 2001
From: Nick Riasanovsky <njriasan at meta.com>
Date: Mon, 5 Oct 2026 15:47:16 -0700
Subject: [PATCH 2/3] Fix tests to autogenerate results
---
.../reschedule-physreg-copy-stale-undef.ll | 168 +++++++++++++++++-
1 file changed, 164 insertions(+), 4 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll b/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll
index 193b0d85387f2..4ed95d57834c7 100644
--- a/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/reschedule-physreg-copy-stale-undef.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -verify-machineinstrs %s -o - | FileCheck %s
;
; Rescheduling an already-scheduled physreg copy next to its user must not
@@ -8,15 +9,174 @@
; Recomputing the flags must also preserve read-undef on the first subregister
; definition, so that it does not read undefined lanes.
;
-; CHECK-LABEL: kernel:
-; CHECK: v_cvt_scalef32_pk32_fp6_bf16 v[0:5], v[8:23]
-; CHECK: ds_write_b128
-; CHECK: s_endpgm
target datalayout = "e-m:e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
target triple = "amdgpu9.50-amd-amdhsa"
define amdgpu_kernel void @kernel(ptr addrspace(1) inreg %0, ptr addrspace(1) %1, float %2, float %3, float %4, float %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13, i1 %14, <8 x half> %15, float %16, float %17, float %18, <8 x half> %19, half %20, <8 x half> %21, half %22, float %23, i32 %24, <2 x float> %25, <2 x float> %26, i64 %27, i32 %28) {
+; CHECK-LABEL: kernel:
+; CHECK: ; %bb.1:
+; CHECK-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_branch .LBB0_0
+; CHECK-NEXT: .p2align 8
+; CHECK-NEXT: ; %bb.2:
+; CHECK-NEXT: .LBB0_0:
+; CHECK-NEXT: s_load_dword s6, s[4:5], 0x40
+; CHECK-NEXT: s_load_dword s7, s[4:5], 0x68
+; CHECK-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0xb0
+; CHECK-NEXT: s_load_dwordx4 s[20:23], s[4:5], 0x30
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_load_dword s23, s[4:5], 0xc8
+; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x8
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: s_bitcmp1_b32 s6, 0
+; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
+; CHECK-NEXT: s_and_b32 s6, s6, 1
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_and_b32 s13, s13, 0xffff
+; CHECK-NEXT: s_mov_b32 s14, s0
+; CHECK-NEXT: s_mov_b32 s15, s0
+; CHECK-NEXT: s_mov_b32 s1, s0
+; CHECK-NEXT: s_mov_b32 s2, s0
+; CHECK-NEXT: s_mov_b32 s3, s0
+; CHECK-NEXT: v_mov_b32_e32 v1, s6
+; CHECK-NEXT: s_and_b32 s9, s9, 0xffff
+; CHECK-NEXT: s_mov_b32 s10, s0
+; CHECK-NEXT: s_mov_b32 s11, s0
+; CHECK-NEXT: buffer_load_dwordx4 v[14:17], off, s[12:15], 0
+; CHECK-NEXT: buffer_load_dwordx4 v[18:21], off, s[8:11], 0 offset:1
+; CHECK-NEXT: v_bfrev_b32_e32 v26, 1
+; CHECK-NEXT: buffer_load_dwordx4 v[10:13], v1, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dwordx4 v[22:25], off, s[0:3], 0
+; CHECK-NEXT: buffer_load_dwordx4 v[6:9], v26, s[0:3], 0 offen
+; CHECK-NEXT: buffer_load_dwordx4 v[20:23], off, s[0:3], 0
+; CHECK-NEXT: buffer_load_dwordx4 v[2:5], off, s[8:11], 0
+; CHECK-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v3, 2
+; CHECK-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; CHECK-NEXT: v_add_f32_e64 v32, s21, 0
+; CHECK-NEXT: s_cmp_gt_i32 s23, -1
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v3, s8
+; CHECK-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; CHECK-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; CHECK-NEXT: s_load_dword s6, s[4:5], 0x60
+; CHECK-NEXT: v_add_f32_e64 v34, s15, 0
+; CHECK-NEXT: v_add_f32_e64 v38, s13, 0
+; CHECK-NEXT: v_add_f32_e64 v40, s9, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v41, 0, 1.0, vcc
+; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
+; CHECK-NEXT: v_add_f32_e64 v33, s11, 0
+; CHECK-NEXT: v_add_f32_e64 v35, s20, 0
+; CHECK-NEXT: v_add_f32_e64 v37, s10, 0
+; CHECK-NEXT: v_add_f32_e64 v39, s14, 0
+; CHECK-NEXT: v_add_f32_e64 v36, s22, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_add_f32_e64 v42, s6, 0
+; CHECK-NEXT: v_mov_b32_e32 v27, 0
+; CHECK-NEXT: v_mul_f32_e64 v5, s15, 0
+; CHECK-NEXT: v_mul_f32_e64 v7, s13, 0
+; CHECK-NEXT: v_mul_f32_e64 v23, s8, 0
+; CHECK-NEXT: v_mul_f32_e64 v25, s14, 0
+; CHECK-NEXT: v_mul_f32_e64 v28, s12, 0
+; CHECK-NEXT: v_mul_f32_e64 v29, s11, 0
+; CHECK-NEXT: v_mul_f32_e64 v30, s10, 0
+; CHECK-NEXT: v_mul_f32_e64 v31, s9, 0
+; CHECK-NEXT: v_mov_b32_e32 v20, s18
+; CHECK-NEXT: v_mov_b32_e32 v21, s19
+; CHECK-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xc0
+; CHECK-NEXT: v_fma_mix_f32 v16, v16, 1.0, 0 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_fma_mix_f32 v14, v18, 1.0, v14 op_sel_hi:[1,1,1]
+; CHECK-NEXT: v_mov_b32_e32 v18, s10
+; CHECK-NEXT: v_fma_mix_f32 v10, v10, 1.0, v32 op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_fma_mix_f32 v15, v19, 1.0, v15 op_sel_hi:[1,1,1]
+; CHECK-NEXT: v_fma_mix_f32 v6, v6, 1.0, v14 op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_add_f32_e32 v3, v3, v10
+; CHECK-NEXT: v_fma_mix_f32 v2, v2, 1.0, 0 op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_cndmask_b32_e32 v1, v1, v18, vcc
+; CHECK-NEXT: v_fma_mix_f32 v12, v12, 1.0, v34 op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_fma_mix_f32 v11, v11, 1.0, v38 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_add_f32_e32 v2, v40, v2
+; CHECK-NEXT: v_add_f32_e32 v3, v3, v6
+; CHECK-NEXT: v_add_f32_e64 v19, s8, 0
+; CHECK-NEXT: v_fma_mix_f32 v8, v8, 1.0, v35 op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_add_f32_e32 v11, v37, v11
+; CHECK-NEXT: v_add_f32_e32 v2, v2, v15
+; CHECK-NEXT: v_add_f32_e32 v10, v33, v12
+; CHECK-NEXT: v_add_f32_e32 v1, v3, v1
+; CHECK-NEXT: v_add_f32_e32 v19, v41, v19
+; CHECK-NEXT: v_mov_b32_e32 v41, s9
+; CHECK-NEXT: v_add_f32_e32 v6, v10, v8
+; CHECK-NEXT: v_add_f32_e32 v8, v11, v39
+; CHECK-NEXT: v_add_f32_e32 v1, v2, v1
+; CHECK-NEXT: v_add_f32_e32 v41, s7, v41
+; CHECK-NEXT: v_add_f32_e32 v1, v8, v1
+; CHECK-NEXT: v_fma_mix_f32 v17, v17, 1.0, v19 op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_add_f32_e64 v19, s12, 0
+; CHECK-NEXT: v_add_f32_e32 v14, v41, v16
+; CHECK-NEXT: v_fma_mix_f32 v13, v13, 1.0, v36 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_add_f32_e32 v1, v6, v1
+; CHECK-NEXT: v_fma_mix_f32 v9, v9, 1.0, v42 op_sel:[1,0,0] op_sel_hi:[1,1,0]
+; CHECK-NEXT: v_add_f32_e32 v12, v19, v13
+; CHECK-NEXT: v_add_f32_e32 v1, v14, v1
+; CHECK-NEXT: v_add_f32_e32 v9, v12, v9
+; CHECK-NEXT: v_add_f32_e32 v1, v17, v1
+; CHECK-NEXT: v_add_f32_e32 v1, v9, v1
+; CHECK-NEXT: ds_write_b32 v27, v1
+; CHECK-NEXT: ds_write_b32 v0, v27
+; CHECK-NEXT: v_add_f32_e32 v0, v5, v7
+; CHECK-NEXT: v_add_f32_e32 v0, v23, v0
+; CHECK-NEXT: v_add_f32_e32 v0, v25, v0
+; CHECK-NEXT: v_add_f32_e32 v0, v28, v0
+; CHECK-NEXT: v_add_f32_e32 v0, v29, v0
+; CHECK-NEXT: v_add_f32_e32 v6, v30, v0
+; CHECK-NEXT: v_cvt_f32_f16_e32 v0, v4
+; CHECK-NEXT: v_cvt_f32_f16_sdwa v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; CHECK-NEXT: v_cvt_f32_f16_e32 v2, v24
+; CHECK-NEXT: v_cvt_f32_f16_sdwa v3, v24 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; CHECK-NEXT: v_cvt_f32_f16_e32 v4, v22
+; CHECK-NEXT: v_cvt_f32_f16_sdwa v5, v22 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; CHECK-NEXT: v_add_f32_e32 v6, v31, v6
+; CHECK-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; CHECK-NEXT: ds_write_b32 v27, v6
+; CHECK-NEXT: v_pk_add_f32 v[4:5], v[0:1], v[4:5]
+; CHECK-NEXT: v_pk_add_f32 v[8:9], s[16:17], v[20:21]
+; CHECK-NEXT: v_cvt_pk_bf16_f32 v4, v4, v5
+; CHECK-NEXT: ds_read_b128 v[0:3], v27
+; CHECK-NEXT: ds_write_b32 v27, v4
+; CHECK-NEXT: v_cvt_pk_bf16_f32 v8, v8, v9
+; CHECK-NEXT: ds_read_b128 v[4:7], v27
+; CHECK-NEXT: ; wave barrier
+; CHECK-NEXT: ds_read_b128 v[22:25], v27
+; CHECK-NEXT: ds_write_b32 v27, v8
+; CHECK-NEXT: ds_read_b128 v[18:21], v27
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v8, v0
+; CHECK-NEXT: v_mov_b32_e32 v10, v4
+; CHECK-NEXT: v_mov_b32_e32 v9, v22
+; CHECK-NEXT: v_mov_b32_e32 v12, v2
+; CHECK-NEXT: v_mov_b32_e32 v11, v18
+; CHECK-NEXT: v_mov_b32_e32 v13, v24
+; CHECK-NEXT: v_mov_b32_e32 v14, v6
+; CHECK-NEXT: v_mov_b32_e32 v15, 0
+; CHECK-NEXT: v_mov_b32_e32 v16, v1
+; CHECK-NEXT: v_mov_b32_e32 v17, v23
+; CHECK-NEXT: v_mov_b32_e32 v18, v5
+; CHECK-NEXT: v_mov_b32_e32 v20, v3
+; CHECK-NEXT: v_mov_b32_e32 v21, v25
+; CHECK-NEXT: v_mov_b32_e32 v22, v7
+; CHECK-NEXT: v_mov_b32_e32 v23, 0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: v_cvt_scalef32_pk32_fp6_bf16 v[0:5], v[8:23], v27
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: v_mov_b32_e32 v6, s0
+; CHECK-NEXT: v_mov_b32_e32 v7, s0
+; CHECK-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; CHECK-NEXT: ds_write_b128 v27, v[4:7]
+; CHECK-NEXT: buffer_store_dwordx2 v[0:1], v26, s[0:3], 0 offen
+; CHECK-NEXT: s_endpgm
%30 = tail call i32 @llvm.amdgcn.workgroup.id.x()
%31 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 0, i32 0, i32 0)
%32 = fmul float %9, 0.000000e+00
>From 45e81bd65b168f1e964c2be2dc86c8db7c49906b Mon Sep 17 00:00:00 2001
From: Nick Riasanovsky <njriasan at meta.com>
Date: Mon, 5 Oct 2026 17:00:42 -0700
Subject: [PATCH 3/3] apply the change
---
llvm/lib/CodeGen/MachineScheduler.cpp | 90 +++++++++++++++++----------
1 file changed, 56 insertions(+), 34 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index e71736d2fd1b0..e3e4727003ed2 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -4247,6 +4247,32 @@ SUnit *GenericScheduler::pickNode(bool &IsTopNode) {
return SU;
}
+/// Clear potentially-stale read-undef flags on subregister defs of \p VRegs in
+/// \p MI, then re-add the ones still valid according to \p LIS. Instructions
+/// without an affected def are left untouched. The clear must come first
+/// because adjustLaneLiveness only adds flags; the re-add avoids introducing
+/// reads of undefined lanes.
+static void recomputeReadUndefFlags(MachineInstr &MI, ArrayRef<Register> VRegs,
+ LiveIntervals &LIS,
+ const TargetRegisterInfo &TRI) {
+ bool HasAffectedDef = false;
+ for (MachineOperand &MO : MI.all_defs()) {
+ if (!MO.getReg().isVirtual() || MO.getSubReg() == 0 ||
+ !llvm::is_contained(VRegs, MO.getReg()))
+ continue;
+ MO.setIsUndef(false);
+ HasAffectedDef = true;
+ }
+ if (!HasAffectedDef)
+ return;
+
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ RegisterOperands RegOpers;
+ RegOpers.collect(MI, TRI, MRI, /*TrackLaneMasks=*/true,
+ /*IgnoreDead=*/false);
+ RegOpers.adjustLaneLiveness(LIS, MRI, MI);
+}
+
void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
MachineBasicBlock::iterator InsertPos = SU->getInstr();
if (!isTop)
@@ -4272,54 +4298,50 @@ void GenericScheduler::reschedulePhysReg(SUnit *SU, bool isTop) {
// read-undef makes a later DAG build drop true data dependencies, which
// can orphan the moved copy and crash LiveIntervals. Recompute the affected
// flags from LiveIntervals after the move.
+ //
+ // 1) Check whether any read-undef flags could have changed. The recompute
+ // needs live intervals to recompute from, vreg liveness, and lane-mask
+ // tracking for the flags to be meaningful. A copy mentioning no vregs
+ // cannot affect vreg subregister defs.
LiveIntervals *LIS = DAG->getLIS();
- bool FixUndef = LIS && DAG->hasVRegLiveness() && shouldTrackLaneMasks();
- MachineBasicBlock::iterator OldNext = Copy->getParent()->end();
- SlotIndex OldIdx;
SmallVector<Register, 8> CopyVRegs;
- if (FixUndef) {
- OldIdx = LIS->getInstructionIndex(*Copy);
- OldNext = std::next(Copy->getIterator());
+ if (LIS && DAG->hasVRegLiveness() && shouldTrackLaneMasks())
for (const MachineOperand &MO : Copy->operands())
if (MO.isReg() && MO.getReg().isVirtual())
CopyVRegs.push_back(MO.getReg());
+ bool FixUndef = !CopyVRegs.empty();
+
+ // 2) Cache the copy's old location. This is purely an optimization: it
+ // narrows step 3 to the instructions the copy moves past. Recomputing a
+ // wider range would be equally correct, since the recompute reproduces
+ // identical flags wherever relative order did not change.
+ SlotIndex OldIdx;
+ MachineBasicBlock::iterator OldNext;
+ if (FixUndef) {
+ OldIdx = LIS->getInstructionIndex(*Copy);
+ OldNext = std::next(Copy->getIterator());
}
+
DAG->moveInstruction(Copy, InsertPos);
- if (!FixUndef || CopyVRegs.empty())
+ if (!FixUndef)
continue;
- auto FixVRegUndef = [&](MachineInstr &MI) {
- bool HasAffectedDef = false;
- for (MachineOperand &MO : MI.all_defs()) {
- if (!MO.getReg().isVirtual() || MO.getSubReg() == 0 ||
- !llvm::is_contained(CopyVRegs, MO.getReg()))
- continue;
- MO.setIsUndef(false);
- HasAffectedDef = true;
- }
- if (!HasAffectedDef)
- return;
-
- // adjustLaneLiveness only adds read-undef flags, so clear them first.
- // Re-add the valid flags to avoid introducing reads of undefined lanes.
- const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
- RegisterOperands RegOpers;
- RegOpers.collect(MI, *TRI, MRI, /*TrackLaneMasks=*/true,
- /*IgnoreDead=*/false);
- RegOpers.adjustLaneLiveness(*LIS, MRI, MI);
- };
- FixVRegUndef(*Copy);
+ // 3) Recompute the read-undef flags invalidated by the move, on the copy
+ // itself and on every instruction it moved past.
+ recomputeReadUndefFlags(*Copy, CopyVRegs, *LIS, *TRI);
MachineBasicBlock::iterator NewIt = Copy->getIterator();
SlotIndex NewIdx = LIS->getInstructionIndex(*Copy);
+ MachineBasicBlock::iterator FixBegin = NewIt, FixEnd = NewIt;
if (NewIdx < OldIdx) {
- for (auto I = std::next(NewIt); I != OldNext; ++I)
- if (!I->isDebugInstr())
- FixVRegUndef(*I);
+ FixBegin = std::next(NewIt);
+ FixEnd = OldNext;
} else if (OldIdx < NewIdx) {
- for (auto I = OldNext; I != NewIt; ++I)
- if (!I->isDebugInstr())
- FixVRegUndef(*I);
+ FixBegin = OldNext;
+ FixEnd = NewIt;
}
+ for (auto I = FixBegin; I != FixEnd; ++I)
+ if (!I->isDebugInstr())
+ recomputeReadUndefFlags(*I, CopyVRegs, *LIS, *TRI);
}
}
More information about the llvm-commits
mailing list