[llvm] [AMDGPU] IGLP: Sanitize user schedule hints (PR #221256)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 10:41:34 PDT 2026
https://github.com/averne updated https://github.com/llvm/llvm-project/pull/221256
>From 899ba981db7c3bab07dfa0afe482e1d32dc3164d Mon Sep 17 00:00:00 2001
From: averne <averne381 at gmail.com>
Date: Fri, 4 Sep 2026 17:31:21 +0200
Subject: [PATCH 1/3] [AMDGPU] IGLP: Sanitize user schedule hints
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 8 +++++---
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll | 2 ++
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 7dc6d6c4d986c2..35f4b1c612a12a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2771,7 +2771,8 @@ void IGroupLPDAGMutation::addSchedBarrierEdges(SUnit &SchedBarrier) {
LLVM_DEBUG(dbgs() << "Building SchedGroup for SchedBarrier with Mask: "
<< MI.getOperand(0).getImm() << "\n");
auto InvertedMask =
- invertSchedBarrierMask((SchedGroupMask)MI.getOperand(0).getImm());
+ invertSchedBarrierMask((SchedGroupMask)(MI.getOperand(0).getImm() &
+ (int32_t)SchedGroupMask::ALL));
SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
for (SUnit &SU : DAG->SUnits)
@@ -2825,8 +2826,9 @@ void IGroupLPDAGMutation::initSchedGroupBarrierPipelineStage(
int32_t SyncID = SGB.getOperand(2).getImm();
Size++; // Make room for the SCHED_GROUP_BARRIER instruction
- auto &SG = SyncedSchedGroups[SyncID].emplace_back((SchedGroupMask)SGMask,
- Size, SyncID, DAG, TII);
+ auto &SG = SyncedSchedGroups[SyncID].emplace_back(
+ (SchedGroupMask)(SGMask & (int32_t)SchedGroupMask::ALL), Size, SyncID,
+ DAG, TII);
SG.add(*RIter);
SG.findCandidateSUnits(RIter, SG.DAG->SUnits.rend(),
SyncedInstrs[SG.getSyncID()]);
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
index 88396740a05e8c..096e4c2f5eaa0f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
@@ -9,12 +9,14 @@ define amdgpu_kernel void @test_sched_barrier() #0 {
; GCN-NEXT: ; sched_barrier mask(0x00000001)
; GCN-NEXT: ; sched_barrier mask(0x00000004)
; GCN-NEXT: ; sched_barrier mask(0x0000000F)
+; GCN-NEXT: ; sched_barrier mask(0xFFFFFFFFFFFFFFFF)
; GCN-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.sched.barrier(i32 0) #1
call void @llvm.amdgcn.sched.barrier(i32 1) #1
call void @llvm.amdgcn.sched.barrier(i32 4) #1
call void @llvm.amdgcn.sched.barrier(i32 15) #1
+ call void @llvm.amdgcn.sched.barrier(i32 -1) #1
ret void
}
>From 7acb6bdba4b2595d954950599d5040481a5e85a4 Mon Sep 17 00:00:00 2001
From: averne <averne381 at gmail.com>
Date: Sun, 6 Sep 2026 14:13:13 +0200
Subject: [PATCH 2/3] Move scheduling mask sanitization to instruction
selection
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 12 +++++++-----
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h | 2 ++
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 ++++++
llvm/lib/Target/AMDGPU/SIInstructions.td | 2 ++
.../test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll | 2 +-
5 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 35f4b1c612a12a..9e5ebbf504d46f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -2771,8 +2771,7 @@ void IGroupLPDAGMutation::addSchedBarrierEdges(SUnit &SchedBarrier) {
LLVM_DEBUG(dbgs() << "Building SchedGroup for SchedBarrier with Mask: "
<< MI.getOperand(0).getImm() << "\n");
auto InvertedMask =
- invertSchedBarrierMask((SchedGroupMask)(MI.getOperand(0).getImm() &
- (int32_t)SchedGroupMask::ALL));
+ invertSchedBarrierMask((SchedGroupMask)MI.getOperand(0).getImm());
SchedGroup SG(InvertedMask, std::nullopt, DAG, TII);
for (SUnit &SU : DAG->SUnits)
@@ -2826,9 +2825,8 @@ void IGroupLPDAGMutation::initSchedGroupBarrierPipelineStage(
int32_t SyncID = SGB.getOperand(2).getImm();
Size++; // Make room for the SCHED_GROUP_BARRIER instruction
- auto &SG = SyncedSchedGroups[SyncID].emplace_back(
- (SchedGroupMask)(SGMask & (int32_t)SchedGroupMask::ALL), Size, SyncID,
- DAG, TII);
+ auto &SG = SyncedSchedGroups[SyncID].emplace_back((SchedGroupMask)SGMask,
+ Size, SyncID, DAG, TII);
SG.add(*RIter);
SG.findCandidateSUnits(RIter, SG.DAG->SUnits.rend(),
SyncedInstrs[SG.getSyncID()]);
@@ -2847,6 +2845,10 @@ bool IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) {
} // namespace
+unsigned llvm::AMDGPU::sanitizeSchedMask(unsigned Mask) {
+ return Mask & static_cast<unsigned>(SchedGroupMask::ALL);
+}
+
/// \p Phase specifes whether or not this is a reentry into the
/// IGroupLPDAGMutation. Since there may be multiple scheduling passes on the
/// same scheduling region (e.g. pre and post-RA scheduling / multiple
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
index 1cb45ebc315f54..74a9189536919b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
@@ -25,6 +25,8 @@ enum IGLPStrategyID : int {
MFMAExpInterleaveID = 2,
MFMAExpSimpleInterleaveID = 3,
};
+
+unsigned sanitizeSchedMask(unsigned Mask);
} // namespace AMDGPU
std::unique_ptr<ScheduleDAGMutation>
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5801d476566665..4c2f4112d80080 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -13,6 +13,7 @@
#include "SIISelLowering.h"
#include "AMDGPU.h"
+#include "AMDGPUIGroupLP.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
#include "AMDGPUMemoryUtils.h"
@@ -7397,6 +7398,11 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MRI.setSimpleHint(MI.getOperand(0).getReg(), MI.getOperand(6).getReg());
return BB;
}
+ case AMDGPU::SCHED_BARRIER:
+ case AMDGPU::SCHED_GROUP_BARRIER:
+ MI.getOperand(0).setImm(
+ AMDGPU::sanitizeSchedMask(MI.getOperand(0).getImm()));
+ return BB;
default:
if (TII->isImage(MI) || TII->isMUBUF(MI)) {
if (!MI.mayStore())
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 554c2ba51bab8f..a9fc135ee0df18 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -542,6 +542,7 @@ def WAVE_BARRIER : SPseudoInstSI<(outs), (ins),
def SCHED_BARRIER : SPseudoInstSI<(outs), (ins i32imm:$mask),
[(int_amdgcn_sched_barrier (i32 timm:$mask))]> {
+ let usesCustomInserter = 1;
let SchedRW = [];
let hasNoSchedulingInfo = 1;
let hasSideEffects = 1;
@@ -557,6 +558,7 @@ def SCHED_GROUP_BARRIER : SPseudoInstSI<
(outs),
(ins i32imm:$mask, i32imm:$size, i32imm:$syncid),
[(int_amdgcn_sched_group_barrier (i32 timm:$mask), (i32 timm:$size), (i32 timm:$syncid))]> {
+ let usesCustomInserter = 1;
let SchedRW = [];
let hasNoSchedulingInfo = 1;
let hasSideEffects = 1;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
index 096e4c2f5eaa0f..6282c5fa6cc9e4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.barrier.ll
@@ -9,7 +9,7 @@ define amdgpu_kernel void @test_sched_barrier() #0 {
; GCN-NEXT: ; sched_barrier mask(0x00000001)
; GCN-NEXT: ; sched_barrier mask(0x00000004)
; GCN-NEXT: ; sched_barrier mask(0x0000000F)
-; GCN-NEXT: ; sched_barrier mask(0xFFFFFFFFFFFFFFFF)
+; GCN-NEXT: ; sched_barrier mask(0x00000FFF)
; GCN-NEXT: s_endpgm
entry:
call void @llvm.amdgcn.sched.barrier(i32 0) #1
>From 107ad1526c8e25072cc6ade4092db521a6ad4891 Mon Sep 17 00:00:00 2001
From: averne <averne381 at gmail.com>
Date: Sun, 13 Sep 2026 19:35:42 +0200
Subject: [PATCH 3/3] Move scheduling mask enum to header, use it directly for
sanitization
---
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 26 -----------------------
llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h | 22 ++++++++++++++++++-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 ++--
3 files changed, 23 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
index 9e5ebbf504d46f..a7fa1f3e693be9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp
@@ -18,7 +18,6 @@
#include "AMDGPUIGroupLP.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
-#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/TargetOpcodes.h"
@@ -59,27 +58,6 @@ static cl::opt<bool> UseCostHeur(
"Experimentally, results are mixed, so this should be set on a "
"case-by-case basis."));
-// Components of the mask that determines which instruction types may be may be
-// classified into a SchedGroup.
-enum class SchedGroupMask {
- NONE = 0u,
- ALU = 1u << 0,
- VALU = 1u << 1,
- SALU = 1u << 2,
- MFMA = 1u << 3,
- VMEM = 1u << 4,
- VMEM_READ = 1u << 5,
- VMEM_WRITE = 1u << 6,
- DS = 1u << 7,
- DS_READ = 1u << 8,
- DS_WRITE = 1u << 9,
- TRANS = 1u << 10,
- LDSDMA = 1u << 11,
- ALL = ALU | VALU | SALU | MFMA | VMEM | VMEM_READ | VMEM_WRITE | DS |
- DS_READ | DS_WRITE | TRANS | LDSDMA,
- LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
-};
-
class SchedGroup;
// InstructionRule class is used to enact a filter which determines whether or
@@ -2845,10 +2823,6 @@ bool IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) {
} // namespace
-unsigned llvm::AMDGPU::sanitizeSchedMask(unsigned Mask) {
- return Mask & static_cast<unsigned>(SchedGroupMask::ALL);
-}
-
/// \p Phase specifes whether or not this is a reentry into the
/// IGroupLPDAGMutation. Since there may be multiple scheduling passes on the
/// same scheduling region (e.g. pre and post-RA scheduling / multiple
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
index 74a9189536919b..ad337422cdd355 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h
@@ -9,6 +9,7 @@
#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMFMAIGROUPLP_H
#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMFMAIGROUPLP_H
+#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/CodeGen/ScheduleDAGMutation.h"
#include <memory>
@@ -26,7 +27,26 @@ enum IGLPStrategyID : int {
MFMAExpSimpleInterleaveID = 3,
};
-unsigned sanitizeSchedMask(unsigned Mask);
+// Components of the mask that determines which instruction types may be may be
+// classified into a SchedGroup.
+enum class SchedGroupMask {
+ NONE = 0u,
+ ALU = 1u << 0,
+ VALU = 1u << 1,
+ SALU = 1u << 2,
+ MFMA = 1u << 3,
+ VMEM = 1u << 4,
+ VMEM_READ = 1u << 5,
+ VMEM_WRITE = 1u << 6,
+ DS = 1u << 7,
+ DS_READ = 1u << 8,
+ DS_WRITE = 1u << 9,
+ TRANS = 1u << 10,
+ LDSDMA = 1u << 11,
+ ALL = ALU | VALU | SALU | MFMA | VMEM | VMEM_READ | VMEM_WRITE | DS |
+ DS_READ | DS_WRITE | TRANS | LDSDMA,
+ LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ ALL)
+};
} // namespace AMDGPU
std::unique_ptr<ScheduleDAGMutation>
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4c2f4112d80080..152fc87a3a6d81 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7400,8 +7400,8 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
}
case AMDGPU::SCHED_BARRIER:
case AMDGPU::SCHED_GROUP_BARRIER:
- MI.getOperand(0).setImm(
- AMDGPU::sanitizeSchedMask(MI.getOperand(0).getImm()));
+ MI.getOperand(0).setImm(MI.getOperand(0).getImm() &
+ static_cast<unsigned>(AMDGPU::SchedGroupMask::ALL));
return BB;
default:
if (TII->isImage(MI) || TII->isMUBUF(MI)) {
More information about the llvm-commits
mailing list