[llvm] [AMDGPU] SIFixSGPRCopies avoid placing V_READFIRSTLANE inside the loop. (PR #186400)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 04:44:21 PDT 2026
https://github.com/alex-t updated https://github.com/llvm/llvm-project/pull/186400
>From e1e1bedb57a26aa86a7625b8f84828d2ab5a6a48 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Fri, 13 Mar 2026 03:03:26 +0000
Subject: [PATCH 1/5] [AMDGPU] SIFixSGPRCopies cost model update to make avoid
placing V_READFIRSTLANE inside the loop.
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 42 +++++++++++--
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 17 +++++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 5 ++
.../fix-sgpr-copies-v2s-readfirstlane-loop.ll | 63 +++++++++++++++++++
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 ++
5 files changed, 128 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 8782fc5fc9bb7..6fc6022596500 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -70,6 +70,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Target/TargetMachine.h"
@@ -98,6 +99,12 @@ class V2SCopyInfo {
// Actual count of v_readfirstlane_b32
// which need to be inserted to keep SChain SALU
unsigned NumReadfirstlanes = 0;
+ // Count of v_readfirstlane_b32 that legalizeOperands would insert at
+ // unconvertible use sites if the chain is moved to VALU.
+ unsigned NumUnavoidableReadfirstlanes = 0;
+ // Penalty for unavoidable readfirstlanes placed inside a loop when the
+ // original COPY was outside.
+ unsigned LoopDepthPenalty = 0;
// Current score state. To speedup selection V2SCopyInfos for processing
bool NeedToBeConvertedToVALU = false;
// Unique ID. Used as a key for mapping to keep permanent order.
@@ -114,6 +121,8 @@ class V2SCopyInfo {
void dump() const {
dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
<< "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
+ << "\n\tURFL: " << NumUnavoidableReadfirstlanes
+ << "\n\tLDP: " << LoopDepthPenalty
<< "\nScore: " << Score << "\n";
}
#endif
@@ -121,6 +130,7 @@ class V2SCopyInfo {
class SIFixSGPRCopies {
MachineDominatorTree *MDT;
+ MachineLoopInfo *MLI;
SmallVector<MachineInstr*, 4> SCCCopies;
SmallVector<MachineInstr*, 4> RegSequences;
SmallVector<MachineInstr*, 4> PHINodes;
@@ -135,7 +145,8 @@ class SIFixSGPRCopies {
const SIRegisterInfo *TRI;
const SIInstrInfo *TII;
- SIFixSGPRCopies(MachineDominatorTree *MDT) : MDT(MDT) {}
+ SIFixSGPRCopies(MachineDominatorTree *MDT, MachineLoopInfo *MLI)
+ : MDT(MDT), MLI(MLI) {}
bool run(MachineFunction &MF);
void fixSCCCopies(MachineFunction &MF);
@@ -170,7 +181,9 @@ class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
bool runOnMachineFunction(MachineFunction &MF) override {
MachineDominatorTree *MDT =
&getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
- SIFixSGPRCopies Impl(MDT);
+ MachineLoopInfo *MLI =
+ &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
+ SIFixSGPRCopies Impl(MDT, MLI);
return Impl.run(MF);
}
@@ -178,7 +191,9 @@ class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineDominatorTreeWrapperPass>();
+ AU.addRequired<MachineLoopInfoWrapperPass>();
AU.addPreserved<MachineDominatorTreeWrapperPass>();
+ AU.addPreserved<MachineLoopInfoWrapperPass>();
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -189,6 +204,7 @@ class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
INITIALIZE_PASS_BEGIN(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
false, false)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
INITIALIZE_PASS_END(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
false, false)
@@ -1010,6 +1026,22 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
for (auto *U : Users) {
if (TII->isSALU(*U))
Info.SChain.insert(U);
+
+ if (TII->needsReadlaneOnVALUConversion(*U)) {
+ unsigned SrcIdx = U->getOpcode() == AMDGPU::SI_INIT_M0 ? 0 : 1;
+ MachineOperand &Src = U->getOperand(SrcIdx);
+ unsigned Width = Src.isReg() && Src.getReg().isVirtual()
+ ? TRI->getRegSizeInBits(*MRI->getRegClass(Src.getReg()))
+ : 32;
+ unsigned RFLCount = Width / 32;
+ Info.NumUnavoidableReadfirstlanes += RFLCount;
+
+ unsigned CopyDepth = MLI->getLoopDepth(Info.Copy->getParent());
+ unsigned UseDepth = MLI->getLoopDepth(U->getParent());
+ if (UseDepth > CopyDepth)
+ Info.LoopDepthPenalty += RFLCount * (UseDepth - CopyDepth) * 10;
+ }
+
AnalysisWorklist.push_back(U);
}
}
@@ -1050,7 +1082,8 @@ bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
unsigned Penalty =
Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
- unsigned Profit = Info->SChain.size();
+ unsigned Profit = Info->SChain.size() +
+ Info->NumUnavoidableReadfirstlanes + Info->LoopDepthPenalty;
Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
Info->NeedToBeConvertedToVALU = Info->Score < 3;
return Info->NeedToBeConvertedToVALU;
@@ -1212,7 +1245,8 @@ PreservedAnalyses
SIFixSGPRCopiesPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
- SIFixSGPRCopies Impl(&MDT);
+ MachineLoopInfo &MLI = MFAM.getResult<MachineLoopAnalysis>(MF);
+ SIFixSGPRCopies Impl(&MDT, &MLI);
bool Changed = Impl.run(MF);
if (!Changed)
return PreservedAnalyses::all();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 96ad12337d62a..c4d5b79824881 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -6744,6 +6744,23 @@ void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
fixImplicitOperands(MI);
}
+bool SIInstrInfo::needsReadlaneOnVALUConversion(
+ const MachineInstr &MI) const {
+ switch (MI.getOpcode()) {
+ case AMDGPU::SI_INIT_M0:
+ case AMDGPU::S_BITREPLICATE_B64_B32:
+ case AMDGPU::S_QUADMASK_B32:
+ case AMDGPU::S_QUADMASK_B64:
+ case AMDGPU::S_WQM_B32:
+ case AMDGPU::S_WQM_B64:
+ case AMDGPU::S_INVERSE_BALLOT_U32:
+ case AMDGPU::S_INVERSE_BALLOT_U64:
+ return true;
+ default:
+ return false;
+ }
+}
+
// Legalize VOP3 operands. All operand types are supported for any operand
// but only one literal constant and only starting from GFX10.
void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index f363560784730..bbd01be1ec288 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1457,6 +1457,11 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// Fix operands in \p MI to satisfy constant bus requirements.
void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
+ /// Returns true if moveToVALU would fall through to legalizeOperands for
+ /// this instruction, resulting in a readlane insertion at the use site
+ /// rather than a genuine VALU conversion.
+ bool needsReadlaneOnVALUConversion(const MachineInstr &MI) const;
+
/// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
/// for the dst register (\p DstRC) can be optionally supplied. This function
/// can only be used when it is know that the value in SrcReg is same across
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
new file mode 100644
index 0000000000000..9b6f278477770
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
@@ -0,0 +1,63 @@
+; RUN: llc -march=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck %s
+
+; Test that the SIFixSGPRCopies cost model keeps V_READFIRSTLANE_B32 outside
+; the loop when it feeds an unconvertible SALU instruction (SI_INIT_M0) inside
+; the loop.
+;
+; Without the cost model accounting for unavoidable readfirstlanes and loop
+; depth penalty, the VGPR-to-SGPR copy would be converted to VALU, causing
+; legalizeOperands to insert V_READFIRSTLANE_B32 at the SI_INIT_M0 use site
+; inside the loop.
+
+; The v_readfirstlane_b32 must appear before the loop, not inside it.
+; CHECK-LABEL: readfirstlane_in_loop:
+; CHECK: ; %bb.0:
+; CHECK: v_readfirstlane_b32
+; CHECK: ; %loop
+; CHECK-NOT: v_readfirstlane_b32
+; CHECK: s_mov_b32 m0,
+; CHECK-NOT: v_readfirstlane_b32
+; CHECK: s_endpgm
+
+target triple = "amdgcn-amd-amdhsa"
+
+ at a = internal unnamed_addr addrspace(3) global [2048 x i8] poison, align 16
+
+declare void @llvm.amdgcn.load.to.lds.p1(ptr addrspace(1), ptr addrspace(3), i32, i32, i32)
+
+define amdgpu_kernel void @readfirstlane_in_loop(ptr addrspace(1) noalias noundef nocapture readonly %inp, ptr addrspace(1) noalias noundef nocapture writeonly %out) #0 !reqd_work_group_size !0 {
+entry:
+ %tid = call range(i32 0, 128) i32 @llvm.amdgcn.workitem.id.x()
+ %wgid = call i32 @llvm.amdgcn.workgroup.id.x()
+ %wgid.zext = zext nneg i32 %wgid to i64
+ %wgoff = shl nsw i64 %wgid.zext, 11
+ %tidoff = shl nsw i32 %tid, 4
+ %tidoff.zext = zext nneg i32 %tid to i64
+ %globaloff = add nsw nuw i64 %wgoff, %tidoff.zext
+ %src.base = getelementptr inbounds nuw i8, ptr addrspace(1) %inp, i64 %globaloff
+ %dst.base = getelementptr inbounds nuw i8, ptr addrspace(1) %out, i64 %globaloff
+ %waveid = lshr i32 %tid, 6
+ %waveoff = shl nsw nuw i32 %waveid, 4
+ %our.lds = getelementptr inbounds nuw i8, ptr addrspace(3) @a, i32 %waveoff
+ %my.lds = getelementptr inbounds nuw i8, ptr addrspace(3) @a, i32 %tidoff
+ br label %loop
+
+loop:
+ %k = phi i32 [%k.next, %loop], [0, %entry]
+ %src = phi ptr addrspace(1) [%src.next, %loop], [%src.base, %entry]
+ %dst = phi ptr addrspace(1) [%dst.next, %loop], [%dst.base, %entry]
+ call void @llvm.amdgcn.load.to.lds.p1(ptr addrspace(1) %src, ptr addrspace(3) %our.lds, i32 16, i32 0, i32 0)
+ %v = load <4 x i32>, ptr addrspace(3) %my.lds, align 16
+ store <4 x i32> %v, ptr addrspace(1) %dst, align 16
+ %k.next = add nsw nuw i32 %k, 1
+ %src.next = getelementptr inbounds nuw i8, ptr addrspace(1) %src, i64 65536
+ %dst.next = getelementptr inbounds nuw i8, ptr addrspace(1) %dst, i64 65536
+ %cond = icmp samesign ult i32 %k, 9
+ br i1 %cond, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="128,128" }
+!0 = !{i32 128, i32 1, i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index abb6ccc5faadb..264a8e1561bc2 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -103,6 +103,7 @@
; GCN-O0-NEXT: Assignment Tracking Analysis
; GCN-O0-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection
; GCN-O0-NEXT: MachineDominator Tree Construction
+; GCN-O0-NEXT: Machine Natural Loop Construction
; GCN-O0-NEXT: SI Fix SGPR copies
; GCN-O0-NEXT: MachinePostDominator Tree Construction
; GCN-O0-NEXT: SI Lower i1 Copies
@@ -315,6 +316,7 @@
; GCN-O1-NEXT: Lazy Block Frequency Analysis
; GCN-O1-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection
; GCN-O1-NEXT: MachineDominator Tree Construction
+; GCN-O1-NEXT: Machine Natural Loop Construction
; GCN-O1-NEXT: SI Fix SGPR copies
; GCN-O1-NEXT: MachinePostDominator Tree Construction
; GCN-O1-NEXT: SI Lower i1 Copies
@@ -627,6 +629,7 @@
; GCN-O1-OPTS-NEXT: Lazy Block Frequency Analysis
; GCN-O1-OPTS-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection
; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction
+; GCN-O1-OPTS-NEXT: Machine Natural Loop Construction
; GCN-O1-OPTS-NEXT: SI Fix SGPR copies
; GCN-O1-OPTS-NEXT: MachinePostDominator Tree Construction
; GCN-O1-OPTS-NEXT: SI Lower i1 Copies
@@ -950,6 +953,7 @@
; GCN-O2-NEXT: Lazy Block Frequency Analysis
; GCN-O2-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection
; GCN-O2-NEXT: MachineDominator Tree Construction
+; GCN-O2-NEXT: Machine Natural Loop Construction
; GCN-O2-NEXT: SI Fix SGPR copies
; GCN-O2-NEXT: MachinePostDominator Tree Construction
; GCN-O2-NEXT: SI Lower i1 Copies
@@ -1287,6 +1291,7 @@
; GCN-O3-NEXT: Lazy Block Frequency Analysis
; GCN-O3-NEXT: AMDGPU DAG->DAG Pattern Instruction Selection
; GCN-O3-NEXT: MachineDominator Tree Construction
+; GCN-O3-NEXT: Machine Natural Loop Construction
; GCN-O3-NEXT: SI Fix SGPR copies
; GCN-O3-NEXT: MachinePostDominator Tree Construction
; GCN-O3-NEXT: SI Lower i1 Copies
>From 0de9309a5df0b3d0ff1e0f49b451c58a1298eaf1 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Wed, 18 Mar 2026 09:34:42 +0000
Subject: [PATCH 2/5] Address review feedback: deduplicate opcode list, fix
test RUN line
Refactor legalizeOperands to use needsReadlaneOnVALUConversion instead
of duplicating the opcode list. Fix test to use -mtriple and drop
-verify-machineinstrs.
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 22 +++++--------------
.../fix-sgpr-copies-v2s-readfirstlane-loop.ll | 2 +-
2 files changed, 6 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index c4d5b79824881..6d9113f87111a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7461,23 +7461,11 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
return CreatedBB;
}
- // Legalize SI_INIT_M0
- if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
- MachineOperand &Src = MI.getOperand(0);
- if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
- Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
- return CreatedBB;
- }
-
- // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
- if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
- MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
- MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
- MI.getOpcode() == AMDGPU::S_WQM_B32 ||
- MI.getOpcode() == AMDGPU::S_WQM_B64 ||
- MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
- MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
- MachineOperand &Src = MI.getOperand(1);
+ // Legalize instructions with no VALU equivalent — insert readlane for
+ // any VGPR source operand.
+ if (needsReadlaneOnVALUConversion(MI)) {
+ unsigned SrcIdx = MI.getOpcode() == AMDGPU::SI_INIT_M0 ? 0 : 1;
+ MachineOperand &Src = MI.getOperand(SrcIdx);
if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
return CreatedBB;
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
index 9b6f278477770..5eaf8bd7a701d 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx950 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck %s
; Test that the SIFixSGPRCopies cost model keeps V_READFIRSTLANE_B32 outside
; the loop when it feeds an unconvertible SALU instruction (SI_INIT_M0) inside
>From ebbc9d637507f78b7fa685063b107b9fe4e086f8 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Tue, 24 Mar 2026 09:33:46 +0000
Subject: [PATCH 3/5] [AMDGPU] needsReadlaneOnVALUConversion changed to return
operand index instead of boolean and renamed to
getReadlaneOperandOnVALUConversion
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 12 ++++++++----
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 14 ++++++--------
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 8 ++++----
3 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 6fc6022596500..caf68ac5696e5 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1027,19 +1027,23 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
if (TII->isSALU(*U))
Info.SChain.insert(U);
- if (TII->needsReadlaneOnVALUConversion(*U)) {
- unsigned SrcIdx = U->getOpcode() == AMDGPU::SI_INIT_M0 ? 0 : 1;
- MachineOperand &Src = U->getOperand(SrcIdx);
+ if (auto SrcIdx = TII->getReadlaneOperandOnVALUConversion(*U)) {
+ MachineOperand &Src = U->getOperand(*SrcIdx);
unsigned Width = Src.isReg() && Src.getReg().isVirtual()
? TRI->getRegSizeInBits(*MRI->getRegClass(Src.getReg()))
: 32;
unsigned RFLCount = Width / 32;
Info.NumUnavoidableReadfirstlanes += RFLCount;
+ // Each loop nesting level crossed by an unavoidable readfirstlane
+ // adds a heavy penalty because the instruction would execute every
+ // iteration instead of once in the preheader.
+ static constexpr unsigned LoopNestingPenaltyFactor = 10;
unsigned CopyDepth = MLI->getLoopDepth(Info.Copy->getParent());
unsigned UseDepth = MLI->getLoopDepth(U->getParent());
if (UseDepth > CopyDepth)
- Info.LoopDepthPenalty += RFLCount * (UseDepth - CopyDepth) * 10;
+ Info.LoopDepthPenalty +=
+ RFLCount * (UseDepth - CopyDepth) * LoopNestingPenaltyFactor;
}
AnalysisWorklist.push_back(U);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 6d9113f87111a..756f14c8e9477 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -6744,10 +6744,11 @@ void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
fixImplicitOperands(MI);
}
-bool SIInstrInfo::needsReadlaneOnVALUConversion(
+std::optional<unsigned> SIInstrInfo::getReadlaneOperandOnVALUConversion(
const MachineInstr &MI) const {
switch (MI.getOpcode()) {
case AMDGPU::SI_INIT_M0:
+ return 0;
case AMDGPU::S_BITREPLICATE_B64_B32:
case AMDGPU::S_QUADMASK_B32:
case AMDGPU::S_QUADMASK_B64:
@@ -6755,9 +6756,9 @@ bool SIInstrInfo::needsReadlaneOnVALUConversion(
case AMDGPU::S_WQM_B64:
case AMDGPU::S_INVERSE_BALLOT_U32:
case AMDGPU::S_INVERSE_BALLOT_U64:
- return true;
+ return 1;
default:
- return false;
+ return std::nullopt;
}
}
@@ -7461,11 +7462,8 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
return CreatedBB;
}
- // Legalize instructions with no VALU equivalent — insert readlane for
- // any VGPR source operand.
- if (needsReadlaneOnVALUConversion(MI)) {
- unsigned SrcIdx = MI.getOpcode() == AMDGPU::SI_INIT_M0 ? 0 : 1;
- MachineOperand &Src = MI.getOperand(SrcIdx);
+ if (auto SrcIdx = getReadlaneOperandOnVALUConversion(MI)) {
+ MachineOperand &Src = MI.getOperand(*SrcIdx);
if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
return CreatedBB;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index bbd01be1ec288..41cdb5a53aaec 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1457,10 +1457,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// Fix operands in \p MI to satisfy constant bus requirements.
void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
- /// Returns true if moveToVALU would fall through to legalizeOperands for
- /// this instruction, resulting in a readlane insertion at the use site
- /// rather than a genuine VALU conversion.
- bool needsReadlaneOnVALUConversion(const MachineInstr &MI) const;
+ /// Returns the source operand index that would need a readlane insertion
+ /// if this instruction were reached via VALU conversion, or std::nullopt
+ /// if the instruction has a proper VALU equivalent.
+ std::optional<unsigned> getReadlaneOperandOnVALUConversion(const MachineInstr &MI) const;
/// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
/// for the dst register (\p DstRC) can be optionally supplied. This function
>From f245b6019c843ffe079a165d2ec362ce0bccc349 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Tue, 21 Apr 2026 17:48:55 +0000
Subject: [PATCH 4/5] Update documentation for
getReadlaneOperandOnVALUConversion to reflect correct terminology and remove
unnecessary target triple from test file.
---
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 2 +-
.../CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll | 2 --
2 files changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 41cdb5a53aaec..1e54f91c880bd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1457,7 +1457,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
/// Fix operands in \p MI to satisfy constant bus requirements.
void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
- /// Returns the source operand index that would need a readlane insertion
+ /// Returns the source operand index that would need a readfirstlane insertion
/// if this instruction were reached via VALU conversion, or std::nullopt
/// if the instruction has a proper VALU equivalent.
std::optional<unsigned> getReadlaneOperandOnVALUConversion(const MachineInstr &MI) const;
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
index 5eaf8bd7a701d..5fb1edbee2477 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-v2s-readfirstlane-loop.ll
@@ -19,8 +19,6 @@
; CHECK-NOT: v_readfirstlane_b32
; CHECK: s_endpgm
-target triple = "amdgcn-amd-amdhsa"
-
@a = internal unnamed_addr addrspace(3) global [2048 x i8] poison, align 16
declare void @llvm.amdgcn.load.to.lds.p1(ptr addrspace(1), ptr addrspace(3), i32, i32, i32)
>From 2d25884aa481462f229eee1815b038e79e399919 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Thu, 23 Apr 2026 11:43:45 +0000
Subject: [PATCH 5/5] Refactor analyzeVGPRToSGPRCopy to ensure virtual register
checks are performed before calculating register width.
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index caf68ac5696e5..3f1c936feb54f 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1029,9 +1029,11 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
if (auto SrcIdx = TII->getReadlaneOperandOnVALUConversion(*U)) {
MachineOperand &Src = U->getOperand(*SrcIdx);
- unsigned Width = Src.isReg() && Src.getReg().isVirtual()
- ? TRI->getRegSizeInBits(*MRI->getRegClass(Src.getReg()))
- : 32;
+ if (!Src.isReg() || !Src.getReg().isVirtual())
+ continue;
+
+ unsigned Width =
+ TRI->getRegSizeInBits(*MRI->getRegClass(Src.getReg()));
unsigned RFLCount = Width / 32;
Info.NumUnavoidableReadfirstlanes += RFLCount;
More information about the llvm-commits
mailing list