[llvm] [AMDGPU] Don't apply gfx950 fetch-window loop align to the wrong block (PR #221821)
Akash Dutta via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 11:00:39 PDT 2026
https://github.com/akadutta updated https://github.com/llvm/llvm-project/pull/221821
>From 15982708519f06dfc0590292471bc2dd643d52c4 Mon Sep 17 00:00:00 2001
From: Akash Dutta <Akash.Dutta at amd.com>
Date: Mon, 7 Sep 2026 20:19:50 +0000
Subject: [PATCH 1/3] Don't apply gfx950 fetch-window loop align to the wrong
block
---
llvm/include/llvm/CodeGen/TargetLowering.h | 10 +
llvm/lib/CodeGen/MachineBlockPlacement.cpp | 4 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 21 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +-
.../AMDGPU/loop-header-align-gfx950.mir | 295 ++++++++++++++++--
5 files changed, 303 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 69f5e0e4e3011..ed15ec1fdb7eb 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2177,6 +2177,16 @@ class LLVM_ABI TargetLoweringBase {
/// Return the preferred loop alignment.
virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const;
+ /// Return the preferred alignment for MBB when it is being aligned as part
+ /// of ML. Callers that have no specific block should use the one-argument
+ /// hook. Targets with block-specific loop alignment requirements can
+ /// override this hook; AMDGPU requires MBB to be non-null. The default
+ /// preserves existing one-argument overrides.
+ virtual Align getPrefLoopAlignment(MachineLoop *ML,
+ const MachineBasicBlock *MBB) const {
+ return getPrefLoopAlignment(ML);
+ }
+
/// Return the maximum amount of bytes allowed to be emitted when padding for
/// alignment
virtual unsigned
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 1378aaa67ba39..a8bd36cfb22c5 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -3032,7 +3032,9 @@ void MachineBlockPlacement::alignBlocks() {
if (!L)
continue;
- const Align TLIAlign = TLI->getPrefLoopAlignment(L);
+ // Query the block being aligned rather than only the LoopInfo header.
+ // After loop rotation, ChainBB can be a different backedge destination.
+ const Align TLIAlign = TLI->getPrefLoopAlignment(L, ChainBB);
unsigned MDAlign = 1;
MDNode *LoopID = L->getLoopID();
if (LoopID) {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5fb904aa4970f..0a6dadafc9e03 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20436,21 +20436,24 @@ Align SITargetLowering::computeKnownAlignForTargetInstr(
return Align(1);
}
-Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
+Align SITargetLowering::getPrefLoopAlignment(
+ MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
const Align CacheLineAlign = Align(64);
- // GFX950: Prevent an 8-byte instruction at loop header from being split by
- // the 32-byte instruction fetch window boundary. This avoids a significant
- // fetch delay after backward branch. We use 32-byte alignment with max
- // padding of 4 bytes (one s_nop), see getMaxPermittedBytesForAlignment().
+ // GFX950: Prevent an 8-byte instruction at the block being aligned from being
+ // split by the 32-byte instruction fetch window boundary. This avoids a
+ // significant fetch delay after a backward branch. We use 32-byte alignment
+ // with max padding of 4 bytes (one s_nop), see
+ // getMaxPermittedBytesForAlignment().
if (ML && !DisableLoopAlignment &&
getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
- const MachineBasicBlock *Header = ML->getHeader();
+ assert(BlockToAlign &&
+ "gfx950 fetch-window alignment requires the block being aligned");
// Respect user-specified or previously set alignment.
- if (Header->getAlignment() != PrefAlign)
- return Header->getAlignment();
- if (needsFetchWindowAlignment(*Header))
+ if (BlockToAlign->getAlignment() != PrefAlign)
+ return BlockToAlign->getAlignment();
+ if (needsFetchWindowAlignment(*BlockToAlign))
return Align(32);
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 215107fcec712..324c1ab708e13 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -624,7 +624,9 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isDivergent) const override;
bool requiresUniformRegister(MachineFunction &MF,
const Value *V) const override;
- Align getPrefLoopAlignment(MachineLoop *ML) const override;
+ Align
+ getPrefLoopAlignment(MachineLoop *ML,
+ const MachineBasicBlock *BlockToAlign) const override;
unsigned
getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir b/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir
index a23fb912893b1..3acb7da6b3e96 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir
+++ b/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir
@@ -1,22 +1,39 @@
-# RUN: llc -mtriple=amdgpu9.50 -start-before=block-placement -o - %s | FileCheck %s
-
-# Test that loop headers are aligned to 32 bytes on GFX950 when the first
-# instruction is 8 bytes, to prevent the instruction from being split by the
-# 32-byte fetch window boundary.
-# The second test case verifies that 4-byte instructions do NOT trigger
-# alignment (CHECK-NEXT chain would break if .p2align were inserted).
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.50 -start-before=block-placement \
+# RUN: -stop-after=block-placement -o - %s | FileCheck %s
+#
+# GFX950 fetch-window loop alignment: 32-byte align with a 4-byte pad cap when
+# the block being aligned starts with an instruction wider than 4 bytes.
+# Rotated loops must use the backedge destination, not the LoopInfo header.
+# The extra preheaders keep function entry out of the rotated chain; the
+# diamonds give block placement a landing pad that is not the natural header.
---
name: loop_with_8byte_first_inst
tracksRegLiveness: true
body: |
- ; CHECK-LABEL: loop_with_8byte_first_inst:
- ; CHECK: ; %bb.0:
- ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
- ; CHECK-NEXT: s_mov_b64 s[0:1], 0
- ; CHECK-NEXT: .p2align 5, , 4
- ; CHECK-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
- ; CHECK-NEXT: v_lshrrev_b64 v[0:1], 1, v[0:1]
+ ; CHECK-LABEL: name: loop_with_8byte_first_inst
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_MOV_B64 0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1 (align 32):
+ ; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+ ; CHECK-NEXT: liveins: $sgpr0_sgpr1, $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, killed $vgpr0_vgpr1, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U64_e32 0, $vgpr0_vgpr1, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_OR_B64 killed renamable $vcc, killed renamable $sgpr0_sgpr1, implicit-def $scc
+ ; CHECK-NEXT: $exec = S_ANDN2_B64 $exec, renamable $sgpr0_sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.1, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: liveins: $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr0_sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31
bb.0:
successors: %bb.1(0x80000000)
liveins: $vgpr0_vgpr1
@@ -44,12 +61,28 @@ body: |
name: loop_with_4byte_first_inst
tracksRegLiveness: true
body: |
- ; CHECK-LABEL: loop_with_4byte_first_inst:
- ; CHECK: ; %bb.0:
- ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
- ; CHECK-NEXT: s_mov_b64 s[0:1], 0
- ; CHECK-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
- ; CHECK-NEXT: v_add_u32_e32 v0, 1, v0
+ ; CHECK-LABEL: name: loop_with_4byte_first_inst
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_MOV_B64 0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+ ; CHECK-NEXT: liveins: $sgpr0_sgpr1, $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, killed $vgpr0, implicit $exec
+ ; CHECK-NEXT: V_CMP_LT_U32_e32 10, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_OR_B64 killed renamable $vcc, killed renamable $sgpr0_sgpr1, implicit-def $scc
+ ; CHECK-NEXT: $exec = S_ANDN2_B64 $exec, renamable $sgpr0_sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_EXECNZ %bb.1, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: liveins: $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr0_sgpr1, implicit-def $scc
+ ; CHECK-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31
bb.0:
successors: %bb.1(0x80000000)
liveins: $vgpr0
@@ -72,3 +105,225 @@ body: |
$exec = S_OR_B64 $exec, killed renamable $sgpr0_sgpr1, implicit-def $scc
S_SETPC_B64_return undef $sgpr30_sgpr31
...
+
+---
+name: rotated_loop_with_4byte_landing_pad
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: rotated_loop_with_4byte_landing_pad
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6:
+ ; CHECK-NEXT: successors: %bb.7(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.7, implicit killed $vcc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, killed $vgpr0_vgpr1, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCZ %bb.4, implicit killed $vcc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCZ %bb.6, implicit killed $vcc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.4(0x40000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, killed renamable $vgpr0_vgpr1, implicit $exec
+ V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
+ S_BRANCH %bb.4
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ S_CBRANCH_VCCNZ %bb.5, implicit killed $vcc
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ S_BRANCH %bb.6
+
+ bb.6:
+ successors: %bb.7(0x04000000), %bb.2(0x7c000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ S_CBRANCH_VCCNZ %bb.7, implicit killed $vcc
+ S_BRANCH %bb.2
+
+ bb.7:
+ S_SETPC_B64_return undef $sgpr30_sgpr31
+...
+
+---
+name: rotated_loop_with_8byte_landing_pad
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: rotated_loop_with_8byte_landing_pad
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.6 (align 32):
+ ; CHECK-NEXT: successors: %bb.7(0x04000000), %bb.2(0x7c000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, $vgpr0_vgpr1, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.7, implicit killed $vcc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, $vgpr0_vgpr1, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCZ %bb.4, implicit killed $vcc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCZ %bb.6, implicit killed $vcc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.6(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: renamable $vgpr0 = V_ADD_U32_e32 1, $vgpr0, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.6
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.7:
+ ; CHECK-NEXT: S_SETPC_B64_return undef $sgpr30_sgpr31
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3(0x40000000), %bb.4(0x40000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, renamable $vgpr0_vgpr1, implicit $exec
+ V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ S_CBRANCH_VCCNZ %bb.3, implicit killed $vcc
+ S_BRANCH %bb.4
+
+ bb.3:
+ successors: %bb.4(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ S_BRANCH %bb.4
+
+ bb.4:
+ successors: %bb.5(0x40000000), %bb.6(0x40000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ S_CBRANCH_VCCNZ %bb.5, implicit killed $vcc
+ S_BRANCH %bb.6
+
+ bb.5:
+ successors: %bb.6(0x80000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0 = V_ADD_U32_e32 1, renamable $vgpr0, implicit $exec
+ S_BRANCH %bb.6
+
+ bb.6:
+ successors: %bb.7(0x04000000), %bb.2(0x7c000000)
+ liveins: $vgpr0_vgpr1
+
+ renamable $vgpr0_vgpr1 = V_LSHRREV_B64_e64 1, renamable $vgpr0_vgpr1, implicit $exec
+ V_CMP_EQ_U32_e32 0, $vgpr0, implicit-def $vcc, implicit $exec
+ S_CBRANCH_VCCNZ %bb.7, implicit killed $vcc
+ S_BRANCH %bb.2
+
+ bb.7:
+ S_SETPC_B64_return undef $sgpr30_sgpr31
+...
>From a2ea8433009624d07749046f5384383e2ea5a235 Mon Sep 17 00:00:00 2001
From: Akash Dutta <Akash.Dutta at amd.com>
Date: Tue, 8 Sep 2026 15:25:49 +0000
Subject: [PATCH 2/3] limit to one TL override;fix test
---
llvm/include/llvm/CodeGen/TargetLowering.h | 20 ++++++++-----------
llvm/lib/CodeGen/TargetLoweringBase.cpp | 3 ++-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 6 ++++--
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 3 ++-
llvm/lib/Target/PowerPC/PPCISelLowering.h | 4 +++-
llvm/lib/Target/X86/X86ISelLowering.cpp | 3 ++-
llvm/lib/Target/X86/X86ISelLowering.h | 4 +++-
.../AMDGPU/loop-header-align-gfx950.mir | 20 +++++++++++--------
8 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index ed15ec1fdb7eb..a5083819bd39f 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -2174,18 +2174,14 @@ class LLVM_ABI TargetLoweringBase {
/// Return the preferred function alignment.
Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
- /// Return the preferred loop alignment.
- virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const;
-
- /// Return the preferred alignment for MBB when it is being aligned as part
- /// of ML. Callers that have no specific block should use the one-argument
- /// hook. Targets with block-specific loop alignment requirements can
- /// override this hook; AMDGPU requires MBB to be non-null. The default
- /// preserves existing one-argument overrides.
- virtual Align getPrefLoopAlignment(MachineLoop *ML,
- const MachineBasicBlock *MBB) const {
- return getPrefLoopAlignment(ML);
- }
+ /// Return the preferred loop alignment. \p BlockToAlign, when non-null, is
+ /// the block that will actually be aligned; after loop rotation this need not
+ /// be the LoopInfo header. Targets whose alignment depends on the block
+ /// contents should use it. Callers that are not aligning a particular block,
+ /// such as llvm-exegesis and ARM constant islands, leave it null.
+ virtual Align
+ getPrefLoopAlignment(MachineLoop *ML = nullptr,
+ const MachineBasicBlock *BlockToAlign = nullptr) const;
/// Return the maximum amount of bytes allowed to be emitted when padding for
/// alignment
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 30738642e42c5..1100d6d5f2419 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -2277,7 +2277,8 @@ void TargetLoweringBase::setMinimumBitTestCmps(unsigned Val) {
MinimumBitTestCmps = Val;
}
-Align TargetLoweringBase::getPrefLoopAlignment(MachineLoop *ML) const {
+Align TargetLoweringBase::getPrefLoopAlignment(
+ MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
if (TM.Options.LoopAlignment)
return Align(TM.Options.LoopAlignment);
return PrefLoopAlignment;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 0a6dadafc9e03..2117395bdc7f8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -20448,8 +20448,10 @@ Align SITargetLowering::getPrefLoopAlignment(
// getMaxPermittedBytesForAlignment().
if (ML && !DisableLoopAlignment &&
getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
- assert(BlockToAlign &&
- "gfx950 fetch-window alignment requires the block being aligned");
+ // Loop rotation can make the backedge destination a block other than the
+ // LoopInfo header, so prefer the block the caller is actually aligning.
+ if (!BlockToAlign)
+ BlockToAlign = ML->getHeader();
// Respect user-specified or previously set alignment.
if (BlockToAlign->getAlignment() != PrefAlign)
return BlockToAlign->getAlignment();
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 8152d0350b018..f9a397b6a41b1 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -18752,7 +18752,8 @@ void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
}
}
-Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
+Align PPCTargetLowering::getPrefLoopAlignment(
+ MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
switch (Subtarget.getCPUDirective()) {
default: break;
case PPC::DIR_970:
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index ee1277eab14ee..6e6e0fd4c05f6 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -332,7 +332,9 @@ namespace llvm {
const SelectionDAG &DAG,
unsigned Depth = 0) const override;
- Align getPrefLoopAlignment(MachineLoop *ML) const override;
+ Align
+ getPrefLoopAlignment(MachineLoop *ML,
+ const MachineBasicBlock *BlockToAlign) const override;
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
return true;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e5028205b65e4..5938bede41b5d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -65466,7 +65466,8 @@ X86TargetLowering::getStackProbeSize(const MachineFunction &MF) const {
4096);
}
-Align X86TargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
+Align X86TargetLowering::getPrefLoopAlignment(
+ MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
if (ML && ML->isInnermost() &&
ExperimentalPrefInnermostLoopAlignment.getNumOccurrences())
return Align(1ULL << ExperimentalPrefInnermostLoopAlignment);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index b220c1bbe7bc7..be4321bf62bc3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -729,7 +729,9 @@ namespace llvm {
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,
int JTI, SelectionDAG &DAG) const override;
- Align getPrefLoopAlignment(MachineLoop *ML) const override;
+ Align
+ getPrefLoopAlignment(MachineLoop *ML,
+ const MachineBasicBlock *BlockToAlign) const override;
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {
if (VT == MVT::f80)
diff --git a/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir b/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir
index 3acb7da6b3e96..17fe786ee0f19 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir
+++ b/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.mir
@@ -1,12 +1,16 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu9.50 -start-before=block-placement \
-# RUN: -stop-after=block-placement -o - %s | FileCheck %s
-#
-# GFX950 fetch-window loop alignment: 32-byte align with a 4-byte pad cap when
-# the block being aligned starts with an instruction wider than 4 bytes.
-# Rotated loops must use the backedge destination, not the LoopInfo header.
-# The extra preheaders keep function entry out of the rotated chain; the
-# diamonds give block placement a landing pad that is not the natural header.
+# RUN: llc -mtriple=amdgpu9.50 -run-pass=block-placement -o - %s | FileCheck %s
+
+# Test that loop headers are aligned to 32 bytes on GFX950 when the first
+# instruction is 8 bytes, to prevent the instruction from being split by the
+# 32-byte fetch window boundary.
+# The second test case verifies that 4-byte instructions do NOT trigger
+# alignment.
+# The rotated cases verify that the decision is made for the block that block
+# placement actually aligns (the backedge destination), which after rotation is
+# not the LoopInfo header. The extra preheader keeps the function entry out of
+# the rotated chain, and the diamond gives placement a landing pad distinct
+# from the natural header.
---
name: loop_with_8byte_first_inst
>From 4d3f4771f1118cdd787f1ec7eaf695906d333a7c Mon Sep 17 00:00:00 2001
From: Akash Dutta <Akash.Dutta at amd.com>
Date: Thu, 24 Sep 2026 17:59:33 +0000
Subject: [PATCH 3/3] add IR test
---
.../AMDGPU/loop-header-align-gfx950.ll | 122 ++++++++++++++++++
1 file changed, 122 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.ll
diff --git a/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.ll b/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.ll
new file mode 100644
index 0000000000000..78a942ebe4874
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/loop-header-align-gfx950.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.50 < %s | FileCheck %s
+
+; Block placement rotates these loops so that %body, not the LoopInfo header,
+; is the backedge destination that gets aligned. The GFX950 fetch-window
+; alignment decision must be based on the first instruction of %body.
+
+; %body starts with an 8-byte v_mul_f64 while %header starts with a 4-byte
+; s_cmp, so %body must be aligned.
+define amdgpu_kernel void @rotated_loop_8byte_top_block(ptr addrspace(1) %out, i32 %n, double %x) {
+; CHECK-LABEL: rotated_loop_8byte_top_block:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
+; CHECK-NEXT: s_mov_b32 s3, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; CHECK-NEXT: s_branch .LBB0_3
+; CHECK-NEXT: .p2align 5, , 4
+; CHECK-NEXT: .LBB0_1: ; %body
+; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: v_mul_f64 v[2:3], v[0:1], v[0:1]
+; CHECK-NEXT: s_add_i32 s3, s3, 1
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: .LBB0_2: ; %Flow
+; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s0, 1
+; CHECK-NEXT: s_cbranch_scc0 .LBB0_5
+; CHECK-NEXT: .LBB0_3: ; %header
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_cmp_ge_i32 s3, s2
+; CHECK-NEXT: v_mov_b64_e32 v[0:1], v[2:3]
+; CHECK-NEXT: s_cbranch_scc0 .LBB0_1
+; CHECK-NEXT: ; %bb.4: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: ; implicit-def: $sgpr3
+; CHECK-NEXT: s_branch .LBB0_2
+; CHECK-NEXT: .LBB0_5: ; %exit
+; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; CHECK-NEXT: s_endpgm
+entry:
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+ %acc = phi double [ %x, %entry ], [ %acc.next, %body ]
+ %cmp = icmp slt i32 %i, %n
+ br i1 %cmp, label %body, label %exit
+
+body:
+ %acc.next = fmul double %acc, %acc
+ %i.next = add i32 %i, 1
+ br label %header
+
+exit:
+ store double %acc, ptr addrspace(1) %out
+ ret void
+}
+
+; %body starts with a 4-byte s_add while %header starts with an 8-byte s_cmp
+; with a literal operand, so %body must not be aligned.
+define amdgpu_kernel void @rotated_loop_4byte_top_block(ptr addrspace(1) %out, i32 %n, double %x) {
+; CHECK-LABEL: rotated_loop_4byte_top_block:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
+; CHECK-NEXT: s_mov_b32 s3, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; CHECK-NEXT: s_branch .LBB1_3
+; CHECK-NEXT: .LBB1_1: ; %body
+; CHECK-NEXT: ; in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT: s_add_i32 s3, s3, s2
+; CHECK-NEXT: v_add_f64 v[2:3], v[0:1], 1.0
+; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: .LBB1_2: ; %Flow
+; CHECK-NEXT: ; in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s0, 1
+; CHECK-NEXT: s_cbranch_scc0 .LBB1_5
+; CHECK-NEXT: .LBB1_3: ; %header
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_cmp_gt_i32 s3, 0xf423f
+; CHECK-NEXT: v_mul_f64 v[0:1], v[2:3], v[2:3]
+; CHECK-NEXT: s_cbranch_scc0 .LBB1_1
+; CHECK-NEXT: ; %bb.4: ; in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
+; CHECK-NEXT: ; implicit-def: $sgpr3
+; CHECK-NEXT: s_branch .LBB1_2
+; CHECK-NEXT: .LBB1_5: ; %exit
+; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; CHECK-NEXT: s_endpgm
+entry:
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+ %acc = phi double [ %x, %entry ], [ %acc.next, %body ]
+ %acc.sq = fmul double %acc, %acc
+ %cmp = icmp slt i32 %i, 1000000
+ br i1 %cmp, label %body, label %exit
+
+body:
+ %i.next = add i32 %i, %n
+ %acc.next = fadd double %acc.sq, 1.0
+ br label %header
+
+exit:
+ store double %acc.sq, ptr addrspace(1) %out
+ ret void
+}
More information about the llvm-commits
mailing list