[llvm] [AMDGPU] SelectionDAG divergence tracking should take into account Target divergency. (PR #144947)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 25 07:50:50 PDT 2025
https://github.com/alex-t updated https://github.com/llvm/llvm-project/pull/144947
>From 438cebe79a56c5cd2e0f523bfde64d0ec2774b8e Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Thu, 19 Jun 2025 20:03:28 +0000
Subject: [PATCH 1/8] [AMDGPU] SelectionDAG divergence tracking should take
into account Target divergency.
---
llvm/include/llvm/CodeGen/SelectionDAG.h | 10 ++++--
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 35 ++++++++++++-------
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 18 +++++-----
3 files changed, 37 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index a98e46c587273..66578bdaa2781 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -238,6 +238,8 @@ class SelectionDAG {
LLVMContext *Context;
CodeGenOptLevel OptLevel;
+ bool DivergentTarget = false;
+
UniformityInfo *UA = nullptr;
FunctionLoweringInfo * FLI = nullptr;
@@ -471,14 +473,16 @@ class SelectionDAG {
Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
UniformityInfo *UA, ProfileSummaryInfo *PSIin,
BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI,
- FunctionVarLocs const *FnVarLocs);
+ FunctionVarLocs const *FnVarLocs, bool HasDivergency);
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
MachineFunctionAnalysisManager &AM,
const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA,
ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin,
- MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs) {
- init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs);
+ MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs,
+ bool HasDivergency) {
+ init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs,
+ HasDivergency);
MFAM = &AM;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 5d8db8be9731f..a63d6561942ad 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1368,7 +1368,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
const TargetLibraryInfo *LibraryInfo,
UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
- FunctionVarLocs const *VarLocs) {
+ FunctionVarLocs const *VarLocs, bool HasDivergency) {
MF = &NewMF;
SDAGISelPass = PassPtr;
ORE = &NewORE;
@@ -1381,6 +1381,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
BFI = BFIin;
MMI = &MMIin;
FnVarLocs = VarLocs;
+ DivergentTarget = HasDivergency;
}
SelectionDAG::~SelectionDAG() {
@@ -2327,7 +2328,8 @@ SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
return SDValue(E, 0);
auto *N = newSDNode<RegisterSDNode>(Reg, VTs);
- N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
+ N->SDNodeBits.IsDivergent =
+ DivergentTarget ? TLI->isSDNodeSourceOfDivergence(N, FLI, UA) : false;
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -10929,7 +10931,8 @@ SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
// Now we update the operands.
N->OperandList[0].set(Op);
- updateDivergence(N);
+ if (DivergentTarget)
+ updateDivergence(N);
// If this gets put into a CSE map, add it.
if (InsertPos) CSEMap.InsertNode(N, InsertPos);
return N;
@@ -10958,7 +10961,8 @@ SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
if (N->OperandList[1] != Op2)
N->OperandList[1].set(Op2);
- updateDivergence(N);
+ if (DivergentTarget)
+ updateDivergence(N);
// If this gets put into a CSE map, add it.
if (InsertPos) CSEMap.InsertNode(N, InsertPos);
return N;
@@ -11009,7 +11013,8 @@ UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
if (N->OperandList[i] != Ops[i])
N->OperandList[i].set(Ops[i]);
- updateDivergence(N);
+ if (DivergentTarget)
+ updateDivergence(N);
// If this gets put into a CSE map, add it.
if (InsertPos) CSEMap.InsertNode(N, InsertPos);
return N;
@@ -11796,8 +11801,9 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
SDUse &Use = *UI;
++UI;
Use.set(To);
- if (To->isDivergent() != From->isDivergent())
- updateDivergence(User);
+ if (DivergentTarget)
+ if (To->isDivergent() != From->isDivergent())
+ updateDivergence(User);
} while (UI != UE && UI->getUser() == User);
// Now that we have modified User, add it back to the CSE maps. If it
// already exists there, recursively merge the results together.
@@ -11854,8 +11860,9 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
SDUse &Use = *UI;
++UI;
Use.setNode(To);
- if (To->isDivergent() != From->isDivergent())
- updateDivergence(User);
+ if (DivergentTarget)
+ if (To->isDivergent() != From->isDivergent())
+ updateDivergence(User);
} while (UI != UE && UI->getUser() == User);
// Now that we have modified User, add it back to the CSE maps. If it
@@ -11907,8 +11914,9 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
To_IsDivergent |= ToOp->isDivergent();
} while (UI != UE && UI->getUser() == User);
- if (To_IsDivergent != From->isDivergent())
- updateDivergence(User);
+ if (DivergentTarget)
+ if (To_IsDivergent != From->isDivergent())
+ updateDivergence(User);
// Now that we have modified User, add it back to the CSE maps. If it
// already exists there, recursively merge the results together.
@@ -11968,8 +11976,9 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
++UI;
Use.set(To);
- if (To->isDivergent() != From->isDivergent())
- updateDivergence(User);
+ if (DivergentTarget)
+ if (To->isDivergent() != From->isDivergent())
+ updateDivergence(User);
} while (UI != UE && UI->getUser() == User);
// We are iterating over all uses of the From node, so if a use
// doesn't use the specific value, no changes are made.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 4b98d87fcc63b..cca314fcda156 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -482,7 +482,10 @@ void SelectionDAGISel::initializeAnalysisResults(
MachineModuleInfo &MMI =
MAMP.getCachedResult<MachineModuleAnalysis>(*Fn.getParent())->getMMI();
- CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
+ TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
+
+ CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
+ TTI->hasBranchDivergence());
// Now get the optional analyzes if we want to.
// This is based on the possibly changed OptLevel (after optnone is taken
@@ -500,10 +503,6 @@ void SelectionDAGISel::initializeAnalysisResults(
BatchAA = std::nullopt;
SP = &FAM.getResult<SSPLayoutAnalysis>(Fn);
-
-#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
- TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
-#endif
}
void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
@@ -539,7 +538,10 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
MachineModuleInfo &MMI =
MFP.getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
- CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
+ TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
+
+ CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
+ TTI->hasBranchDivergence());
// Now get the optional analyzes if we want to.
// This is based on the possibly changed OptLevel (after optnone is taken
@@ -558,10 +560,6 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
BatchAA = std::nullopt;
SP = &MFP.getAnalysis<StackProtector>().getLayoutInfo();
-
-#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
- TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
-#endif
}
bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
>From 24cab01923cbf9c73de56194a7cac68aeb938e4b Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Fri, 20 Jun 2025 11:56:01 +0000
Subject: [PATCH 2/8] [AMDGPU] SelectionDAG divergence tracking. Reviewer
comments addressed.
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 33 ++++++++-----------
1 file changed, 14 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index a63d6561942ad..1d46445209272 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2329,7 +2329,7 @@ SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
auto *N = newSDNode<RegisterSDNode>(Reg, VTs);
N->SDNodeBits.IsDivergent =
- DivergentTarget ? TLI->isSDNodeSourceOfDivergence(N, FLI, UA) : false;
+ DivergentTarget && TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
CSEMap.InsertNode(N, IP);
InsertNode(N);
return SDValue(N, 0);
@@ -10931,8 +10931,7 @@ SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
// Now we update the operands.
N->OperandList[0].set(Op);
- if (DivergentTarget)
- updateDivergence(N);
+ updateDivergence(N);
// If this gets put into a CSE map, add it.
if (InsertPos) CSEMap.InsertNode(N, InsertPos);
return N;
@@ -10961,8 +10960,7 @@ SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
if (N->OperandList[1] != Op2)
N->OperandList[1].set(Op2);
- if (DivergentTarget)
- updateDivergence(N);
+ updateDivergence(N);
// If this gets put into a CSE map, add it.
if (InsertPos) CSEMap.InsertNode(N, InsertPos);
return N;
@@ -11013,8 +11011,7 @@ UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
if (N->OperandList[i] != Ops[i])
N->OperandList[i].set(Ops[i]);
- if (DivergentTarget)
- updateDivergence(N);
+ updateDivergence(N);
// If this gets put into a CSE map, add it.
if (InsertPos) CSEMap.InsertNode(N, InsertPos);
return N;
@@ -11801,9 +11798,8 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
SDUse &Use = *UI;
++UI;
Use.set(To);
- if (DivergentTarget)
- if (To->isDivergent() != From->isDivergent())
- updateDivergence(User);
+ if (To->isDivergent() != From->isDivergent())
+ updateDivergence(User);
} while (UI != UE && UI->getUser() == User);
// Now that we have modified User, add it back to the CSE maps. If it
// already exists there, recursively merge the results together.
@@ -11860,9 +11856,8 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
SDUse &Use = *UI;
++UI;
Use.setNode(To);
- if (DivergentTarget)
- if (To->isDivergent() != From->isDivergent())
- updateDivergence(User);
+ if (To->isDivergent() != From->isDivergent())
+ updateDivergence(User);
} while (UI != UE && UI->getUser() == User);
// Now that we have modified User, add it back to the CSE maps. If it
@@ -11914,9 +11909,8 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
To_IsDivergent |= ToOp->isDivergent();
} while (UI != UE && UI->getUser() == User);
- if (DivergentTarget)
- if (To_IsDivergent != From->isDivergent())
- updateDivergence(User);
+ if (To_IsDivergent != From->isDivergent())
+ updateDivergence(User);
// Now that we have modified User, add it back to the CSE maps. If it
// already exists there, recursively merge the results together.
@@ -11976,9 +11970,8 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
++UI;
Use.set(To);
- if (DivergentTarget)
- if (To->isDivergent() != From->isDivergent())
- updateDivergence(User);
+ if (To->isDivergent() != From->isDivergent())
+ updateDivergence(User);
} while (UI != UE && UI->getUser() == User);
// We are iterating over all uses of the From node, so if a use
// doesn't use the specific value, no changes are made.
@@ -12062,6 +12055,8 @@ bool SelectionDAG::calculateDivergence(SDNode *N) {
}
void SelectionDAG::updateDivergence(SDNode *N) {
+ if (!DivergentTarget)
+ return;
SmallVector<SDNode *, 16> Worklist(1, N);
do {
N = Worklist.pop_back_val();
>From ca23e1b3687dcc106d9e8c664435d99b678a8f97 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Mon, 23 Jun 2025 12:26:21 +0000
Subject: [PATCH 3/8] [AMDGPU] SelectionDAG divergence tracking. LIT test
added
---
.../CodeGen/AMDGPU/test_isel_single_lane.ll | 32 +++++++++++++++++++
1 file changed, 32 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
diff --git a/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
new file mode 100644
index 0000000000000..9d28f4a8e2790
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+
+define amdgpu_kernel void @test_isel_single_lane(ptr addrspace(1) %in, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test_isel_single_lane:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_load_dword s0, s[0:1], 0x58
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_lshl_b32 s0, s0, 4
+; GCN-NEXT: v_mov_b32_e32 v1, s0
+; GCN-NEXT: global_store_dword v0, v1, s[2:3]
+; GCN-NEXT: s_endpgm
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep = getelementptr i32, ptr addrspace(1) %in, i32 22
+ %val = load i32, ptr addrspace(1) %gep, align 4
+ %res0 = add i32 %val, %tid
+ %res1 = shl i32 %res0, 4
+ store i32 %res1, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = {
+ "amdgpu-flat-work-group-size"="1,1"
+ "amdgpu-waves-per-eu"="1,1"
+ "uniform-work-group-size"="true"
+}
>From 1c2b4ce6422280e589de0e54cce5074c278342cf Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Tue, 24 Jun 2025 12:42:01 +0000
Subject: [PATCH 4/8] [AMDGPU] SelectionDAG divergence tracking.
SelectionDAG::createOperands changed.
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 21 ++++++++++---------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 1d46445209272..56964ef44d2c2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -13598,24 +13598,25 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
SDUse *Ops = OperandRecycler.allocate(
ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
- bool IsDivergent = false;
+ Node->SDNodeBits.IsDivergent = false;
for (unsigned I = 0; I != Vals.size(); ++I) {
Ops[I].setUser(Node);
Ops[I].setInitial(Vals[I]);
EVT VT = Ops[I].getValueType();
- // Skip Chain. It does not carry divergence.
- if (VT != MVT::Other &&
- (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
- Ops[I].getNode()->isDivergent()) {
- IsDivergent = true;
- }
+ if (DivergentTarget)
+ // Skip Chain. It does not carry divergence.
+ if (VT != MVT::Other &&
+ (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
+ Ops[I].getNode()->isDivergent()) {
+ Node->SDNodeBits.IsDivergent = true;
+ }
}
Node->NumOperands = Vals.size();
Node->OperandList = Ops;
- if (!TLI->isSDNodeAlwaysUniform(Node)) {
- IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
- Node->SDNodeBits.IsDivergent = IsDivergent;
+ if (DivergentTarget && !TLI->isSDNodeAlwaysUniform(Node)) {
+ Node->SDNodeBits.IsDivergent |=
+ TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
}
checkForCycles(Node);
}
>From ae29a536501b3265d4ba07c44bef34a402034b78 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Tue, 24 Jun 2025 18:07:14 +0000
Subject: [PATCH 5/8] [AMDGPU] SelectionDAG divergence tracking. Bugfixing +
new test
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 2 +
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 4 +-
.../CodeGen/AMDGPU/test_isel_single_lane.ll | 47 ++++++++++++-------
3 files changed, 35 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 56964ef44d2c2..7aa4bf4a86f3a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -12036,6 +12036,8 @@ static bool gluePropagatesDivergence(const SDNode *Node) {
}
bool SelectionDAG::calculateDivergence(SDNode *N) {
+ if(!DivergentTarget)
+ return false;
if (TLI->isSDNodeAlwaysUniform(N)) {
assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
"Conflicting divergence information!");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index cca314fcda156..8ab241a170083 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -485,7 +485,7 @@ void SelectionDAGISel::initializeAnalysisResults(
TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
- TTI->hasBranchDivergence());
+ TTI->hasBranchDivergence(&Fn));
// Now get the optional analyzes if we want to.
// This is based on the possibly changed OptLevel (after optnone is taken
@@ -541,7 +541,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
- TTI->hasBranchDivergence());
+ TTI->hasBranchDivergence(&Fn));
// Now get the optional analyzes if we want to.
// This is based on the possibly changed OptLevel (after optnone is taken
diff --git a/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
index 9d28f4a8e2790..9075e371ae7ae 100644
--- a/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
+++ b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
@@ -1,30 +1,45 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.atomic.cond.sub.u32.p1(ptr addrspace(1), i32)
define amdgpu_kernel void @test_isel_single_lane(ptr addrspace(1) %in, ptr addrspace(1) %out) #0 {
; GCN-LABEL: test_isel_single_lane:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dword s0, s[0:1], 0x58
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s0, s0, 4
+; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: s_load_b32 s4, s[0:1], 0x58
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GCN-NEXT: global_atomic_cond_sub_u32 v1, v0, v1, s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GCN-NEXT: s_wait_loadcnt 0x0
+; GCN-NEXT: v_readfirstlane_b32 s0, v1
+; GCN-NEXT: s_addk_co_i32 s0, 0xf4
+; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GCN-NEXT: s_lshl_b32 s1, s0, 4
+; GCN-NEXT: s_mul_i32 s0, s0, s1
+; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GCN-NEXT: s_lshl_b32 s0, s0, 12
+; GCN-NEXT: s_sub_co_i32 s0, s1, s0
+; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: global_store_dword v0, v1, s[2:3]
+; GCN-NEXT: global_store_b32 v0, v1, s[2:3]
; GCN-NEXT: s_endpgm
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- %gep = getelementptr i32, ptr addrspace(1) %in, i32 22
- %val = load i32, ptr addrspace(1) %gep, align 4
- %res0 = add i32 %val, %tid
- %res1 = shl i32 %res0, 4
- store i32 %res1, ptr addrspace(1) %out
- ret void
+ %gep0 = getelementptr i32, ptr addrspace(1) %in, i32 22
+ %val0 = load i32, ptr addrspace(1) %gep0, align 4
+ %gep1 = getelementptr i32, ptr addrspace(1) %in, i32 4
+ %val1 = call i32 @llvm.amdgcn.atomic.cond.sub.u32.p0(ptr addrspace(1) %gep1, i32 %val0)
+ %res0 = add i32 %val1, 244
+ %res1 = shl i32 %res0, 4
+ %res2 = mul i32 %res0, %res1
+ %res3 = shl i32 %res2, 12
+ %res4 = sub i32 %res1, %res3
+ store i32 %res4, ptr addrspace(1) %out
+ ret void
}
+
attributes #0 = {
"amdgpu-flat-work-group-size"="1,1"
"amdgpu-waves-per-eu"="1,1"
>From 0603c0c10c5c42d6ad9da1d08da04b1b94981871 Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Wed, 25 Jun 2025 13:46:10 +0000
Subject: [PATCH 6/8] Revert "[AMDGPU] SelectionDAG divergence tracking.
SelectionDAG::createOperands changed."
This reverts commit 1c2b4ce6422280e589de0e54cce5074c278342cf.
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 21 +++++++++----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 7aa4bf4a86f3a..9edd0cdc3159b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -13600,25 +13600,24 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
SDUse *Ops = OperandRecycler.allocate(
ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
- Node->SDNodeBits.IsDivergent = false;
+ bool IsDivergent = false;
for (unsigned I = 0; I != Vals.size(); ++I) {
Ops[I].setUser(Node);
Ops[I].setInitial(Vals[I]);
EVT VT = Ops[I].getValueType();
- if (DivergentTarget)
- // Skip Chain. It does not carry divergence.
- if (VT != MVT::Other &&
- (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
- Ops[I].getNode()->isDivergent()) {
- Node->SDNodeBits.IsDivergent = true;
- }
+ // Skip Chain. It does not carry divergence.
+ if (VT != MVT::Other &&
+ (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
+ Ops[I].getNode()->isDivergent()) {
+ IsDivergent = true;
+ }
}
Node->NumOperands = Vals.size();
Node->OperandList = Ops;
- if (DivergentTarget && !TLI->isSDNodeAlwaysUniform(Node)) {
- Node->SDNodeBits.IsDivergent |=
- TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
+ if (!TLI->isSDNodeAlwaysUniform(Node)) {
+ IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
+ Node->SDNodeBits.IsDivergent = IsDivergent;
}
checkForCycles(Node);
}
>From d365862821611eb6a3316689917774ebcf144b1d Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Wed, 25 Jun 2025 14:46:24 +0000
Subject: [PATCH 7/8] [AMDGPU] SelectionDAG divergence tracking.
SelectionDAG::createOperands change. One more chance.
---
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9edd0cdc3159b..1ab9917433bf5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -13600,22 +13600,29 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
SDUse *Ops = OperandRecycler.allocate(
ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
+ // Init it to "false" even fro not divergent targets as there is no other init
+ // point for it
+ Node->SDNodeBits.IsDivergent = false;
bool IsDivergent = false;
for (unsigned I = 0; I != Vals.size(); ++I) {
Ops[I].setUser(Node);
Ops[I].setInitial(Vals[I]);
EVT VT = Ops[I].getValueType();
+ // Take care of the Node's operands iif target has divergence
// Skip Chain. It does not carry divergence.
- if (VT != MVT::Other &&
+ if (DivergentTarget && VT != MVT::Other &&
(VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
Ops[I].getNode()->isDivergent()) {
+ // Node is going to be divergent if at least one of its operand is
+ // divergent, unless it belongs to the "AlwaysUniform" exemptions.
IsDivergent = true;
}
}
Node->NumOperands = Vals.size();
Node->OperandList = Ops;
- if (!TLI->isSDNodeAlwaysUniform(Node)) {
+ // Check the divergence of the Node itself.
+ if (DivergentTarget && !TLI->isSDNodeAlwaysUniform(Node)) {
IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
Node->SDNodeBits.IsDivergent = IsDivergent;
}
>From 0616462303bf22c4f952f16ca80e38fa0d7dc74f Mon Sep 17 00:00:00 2001
From: alex-t <alexander.timofeev at amd.com>
Date: Wed, 25 Jun 2025 14:49:36 +0000
Subject: [PATCH 8/8] [AMDGPU] SelectionDAG divergence tracking. One
single-lane-execution LIT test updated.
---
.../CodeGen/AMDGPU/load-constant-always-uniform.ll | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll b/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
index 13ea8b08d2ade..91a8446915f66 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
@@ -18,9 +18,10 @@ define amdgpu_cs void @test_uniform_load_b96(ptr addrspace(1) %ptr, i32 %arg) "a
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x8
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v2, s2, v2, s0
+; GFX11-NEXT: s_or_b32 s1, s2, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
;
@@ -33,12 +34,14 @@ define amdgpu_cs void @test_uniform_load_b96(ptr addrspace(1) %ptr, i32 %arg) "a
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v3, vcc_lo
; GFX12-NEXT: v_readfirstlane_b32 s0, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: v_readfirstlane_b32 s1, v3
; GFX12-NEXT: s_load_b96 s[0:2], s[0:1], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_or_b32 s0, s0, s1
+; GFX12-NEXT: s_or_b32 s0, s2, s0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v2, s0
-; GFX12-NEXT: v_or3_b32 v2, v2, s1, s2
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_endpgm
bb:
More information about the llvm-commits
mailing list