[llvm] [AMDGPU] IGLP: Sanitize user schedule hints (PR #221256)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 00:46:06 PDT 2026
https://github.com/averne updated https://github.com/llvm/llvm-project/pull/221256
>From 899ba981db7c3bab07dfa0afe482e1d32dc3164d Mon Sep 17 00:00:00 2001
From: averne <averne381 at gmail.com>
Date: Fri, 4 Sep 2026 17:31:21 +0200
Subject: [PATCH 1/2] [AMDGPU] IGLP: Sanitize user schedule hints
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 8 +++++---
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll | 2 ++
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 7dc6d6c4d986c..35f4b1c612a12 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2771,7 +2771,8 @@ void IGroupLPDAGMutation::addSchedBarrierEdges(SUnit &SchedBarrier) {
LLVM_DEBUG(dbgs() << "Building SchedGroup for SchedBarrier with Mask: "
<< MI.getOperand(0).getImm() << "\n");
auto InvertedMask =
- invertSchedBarrierMask((SchedGroupMask)MI.getOperand(0).getImm());
+ invertSchedBarrierMask((SchedGroupMask)(MI.getOperand(0).getImm() &
+ (int32_t)SchedGroupMask::ALL));
SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
for (SUnit &SU : DAG->SUnits)
@@ -2825,8 +2826,9 @@ void IGroupLPDAGMutation::initSchedGroupBarrierPipelineStage(
int32_t SyncID = SGB.getOperand(2).getImm();
Size++; // Make room for the SCHED_GROUP_BARRIER instruction
- auto &SG = SyncedSchedGroups[SyncID].emplace_back((SchedGroupMask)SGMask,
- Size, SyncID, DAG, TII);
+ auto &SG = SyncedSchedGroups[SyncID].emplace_back(
+ (SchedGroupMask)(SGMask & (int32_t)SchedGroupMask::ALL), Size, SyncID,
+ DAG, TII);
SG.add(*RIter);
SG.findCandidateSUnits(RIter, SG.DAG->SUnits.rend(),
SyncedInstrs[SG.getSyncID()]);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
index 88396740a05e8..096e4c2f5eaa0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
@@ -9,12 +9,14 @@ define amdgpu_kernel void @test_sched_barrier() #0 {
; GCN-NEXT: ; sched_barrier mask(0x00000001)
; GCN-NEXT: ; sched_barrier mask(0x00000004)
; GCN-NEXT: ; sched_barrier mask(0x0000000F)
+; GCN-NEXT: ; sched_barrier mask(0xFFFFFFFFFFFFFFFF)
; GCN-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.sched.barrier(i32 0) #1
call void @llvm.amdgcn.sched.barrier(i32 1) #1
call void @llvm.amdgcn.sched.barrier(i32 4) #1
call void @llvm.amdgcn.sched.barrier(i32 15) #1
+ call void @llvm.amdgcn.sched.barrier(i32 -1) #1
ret void
}
>From 7acb6bdba4b2595d954950599d5040481a5e85a4 Mon Sep 17 00:00:00 2001
From: averne <averne381 at gmail.com>
Date: Sun, 6 Sep 2026 14:13:13 +0200
Subject: [PATCH 2/2] Move scheduling mask sanitization to instruction
selection
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 12 +++++++-----
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h | 2 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 ++++++
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 ++
.../test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll | 2 +-
5 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 35f4b1c612a12..9e5ebbf504d46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2771,8 +2771,7 @@ void IGroupLPDAGMutation::addSchedBarrierEdges(SUnit &SchedBarrier) {
LLVM_DEBUG(dbgs() << "Building SchedGroup for SchedBarrier with Mask: "
<< MI.getOperand(0).getImm() << "\n");
auto InvertedMask =
- invertSchedBarrierMask((SchedGroupMask)(MI.getOperand(0).getImm() &
- (int32_t)SchedGroupMask::ALL));
+ invertSchedBarrierMask((SchedGroupMask)MI.getOperand(0).getImm());
SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
for (SUnit &SU : DAG->SUnits)
@@ -2826,9 +2825,8 @@ void IGroupLPDAGMutation::initSchedGroupBarrierPipelineStage(
int32_t SyncID = SGB.getOperand(2).getImm();
Size++; // Make room for the SCHED_GROUP_BARRIER instruction
- auto &SG = SyncedSchedGroups[SyncID].emplace_back(
- (SchedGroupMask)(SGMask & (int32_t)SchedGroupMask::ALL), Size, SyncID,
- DAG, TII);
+ auto &SG = SyncedSchedGroups[SyncID].emplace_back((SchedGroupMask)SGMask,
+ Size, SyncID, DAG, TII);
SG.add(*RIter);
SG.findCandidateSUnits(RIter, SG.DAG->SUnits.rend(),
SyncedInstrs[SG.getSyncID()]);
@@ -2847,6 +2845,10 @@ bool IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) {
} // namespace
+unsigned llvm::AMDGPU::sanitizeSchedMask(unsigned Mask) {
+ return Mask & static_cast<unsigned>(SchedGroupMask::ALL);
+}
+
/// \p Phase specifes whether or not this is a reentry into the
/// IGroupLPDAGMutation. Since there may be multiple scheduling passes on the
/// same scheduling region (e.g. pre and post-RA scheduling / multiple
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
index 1cb45ebc315f5..74a9189536919 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
@@ -25,6 +25,8 @@ enum IGLPStrategyID : int {
MFMAExpInterleaveID = 2,
MFMAExpSimpleInterleaveID = 3,
};
+
+unsigned sanitizeSchedMask(unsigned Mask);
} // namespace AMDGPU
std::unique_ptr<ScheduleDAGMutation>
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5801d47656666..4c2f4112d8008 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13,6 +13,7 @@
#include "SIISelLowering.h"
#include "AMDGPU.h"
+#include "AMDGPUIGroupLP.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
#include "AMDGPUMemoryUtils.h"
@@ -7397,6 +7398,11 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MRI.setSimpleHint(MI.getOperand(0).getReg(), MI.getOperand(6).getReg());
return BB;
}
+ case AMDGPU::SCHED_BARRIER:
+ case AMDGPU::SCHED_GROUP_BARRIER:
+ MI.getOperand(0).setImm(
+ AMDGPU::sanitizeSchedMask(MI.getOperand(0).getImm()));
+ return BB;
default:
if (TII->isImage(MI) || TII->isMUBUF(MI)) {
if (!MI.mayStore())
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 554c2ba51bab8..a9fc135ee0df1 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -542,6 +542,7 @@ def WAVE_BARRIER : SPseudoInstSI<(outs), (ins),
def SCHED_BARRIER : SPseudoInstSI<(outs), (ins i32imm:$mask),
[(int_amdgcn_sched_barrier (i32 timm:$mask))]> {
+ let usesCustomInserter = 1;
let SchedRW = [];
let hasNoSchedulingInfo = 1;
let hasSideEffects = 1;
@@ -557,6 +558,7 @@ def SCHED_GROUP_BARRIER : SPseudoInstSI<
(outs),
(ins i32imm:$mask, i32imm:$size, i32imm:$syncid),
[(int_amdgcn_sched_group_barrier (i32 timm:$mask), (i32 timm:$size), (i32 timm:$syncid))]> {
+ let usesCustomInserter = 1;
let SchedRW = [];
let hasNoSchedulingInfo = 1;
let hasSideEffects = 1;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
index 096e4c2f5eaa0..6282c5fa6cc9e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @test_sched_barrier() #0 {
; GCN-NEXT: ; sched_barrier mask(0x00000001)
; GCN-NEXT: ; sched_barrier mask(0x00000004)
; GCN-NEXT: ; sched_barrier mask(0x0000000F)
-; GCN-NEXT: ; sched_barrier mask(0xFFFFFFFFFFFFFFFF)
+; GCN-NEXT: ; sched_barrier mask(0x00000FFF)
; GCN-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.sched.barrier(i32 0) #1
More information about the llvm-commits
mailing list