[llvm-branch-commits] [llvm] [StructurizeCFG] Keep loop metadata (PR #243015)

Dmitry Sidorov via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 16 02:10:11 PST 2026


https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/243015

The pass replaces the latches of a loop with new backedges and dropped
their !llvm.loop metadata, so loop hints such as llvm.loop.align had no
effect on the restructured loops.

Record the loop IDs before the rewrite, recompute LoopInfo after it and
set them again with Loop::setLoopID. A loop that several old loops are
merged into gets none, and functions with irreducible control flow are
left as before.

---

<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>

>From 2e6e28bdf44bd4d334b8fee09f41e6e411f45e8c Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 16 Nov 2026 00:06:40 +0100
Subject: [PATCH] [StructurizeCFG] Keep loop metadata

The pass replaces the latches of a loop with new backedges and dropped
their !llvm.loop metadata, so loop hints such as llvm.loop.align had no
effect on the restructured loops.

Record the loop IDs before the rewrite, recompute LoopInfo after it and
set them again with Loop::setLoopID. A loop that several old loops are
merged into gets none, and functions with irreducible control flow are
left as before.
---
 llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 54 ++++++++++++++++---
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |  5 --
 .../CodeGen/AMDGPU/loop-align-metadata.ll     |  2 +
 .../StructurizeCFG/loop-metadata.ll           | 26 ++++++---
 4 files changed, 68 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index d94052d2837fb4..eeab34508d4a95 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -11,11 +11,14 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/RegionInfo.h"
 #include "llvm/Analysis/RegionIterator.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -377,10 +380,12 @@ class StructurizeCFGLegacyPass : public FunctionPass {
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<UniformityInfoWrapperPass>();
     AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addRequired<LoopInfoWrapperPass>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
     AU.addRequired<RegionInfoPass>();
 
     AU.addPreserved<DominatorTreeWrapperPass>();
+    AU.addPreserved<LoopInfoWrapperPass>();
   }
 };
 
@@ -392,6 +397,7 @@ INITIALIZE_PASS_BEGIN(StructurizeCFGLegacyPass, "structurizecfg",
                       "Structurize the CFG", false, false)
 INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(RegionInfoPass)
 INITIALIZE_PASS_END(StructurizeCFGLegacyPass, "structurizecfg",
@@ -1393,9 +1399,29 @@ void StructurizeCFGPass::printPipeline(
 
 /// Structurize every region in \p RI, processing inner regions before the
 /// regions that contain them. If \p UI is non-null, skip uniform regions.
-static bool runImpl(DominatorTree &DT, RegionInfo &RI,
-                    const TargetTransformInfo &TTI, const UniformityInfo *UI) {
+/// Recompute \p LI after a rewrite and carry a loop ID over to the loop that
+/// holds the old header, unless several old loops end up in that loop. Nothing
+/// is carried over for irreducible control flow, where a cycle that is not a
+/// natural loop may take over a loop header.
+static bool runImpl(Function &F, DominatorTree &DT, LoopInfo &LI,
+                    RegionInfo &RI, const TargetTransformInfo &TTI,
+                    const UniformityInfo *UI) {
+  SmallVector<std::pair<BasicBlock *, MDNode *>> OldLoops;
+  bool HasLoopID = false;
+  for (Loop *L : LI.getLoopsInPreorder()) {
+    MDNode *LoopID = L->getLoopID();
+    HasLoopID |= LoopID != nullptr;
+    OldLoops.push_back({L->getHeader(), LoopID});
+  }
+  if (HasLoopID) {
+    ReversePostOrderTraversal<const Function *> RPOT(&F);
+    HasLoopID = !containsIrreducibleCFG<const BasicBlock *>(RPOT, LI);
+  }
+  if (!HasLoopID)
+    OldLoops.clear();
+
   bool Changed = false;
+  bool Structurized = false;
   std::vector<Region *> Regions;
   addRegionIntoQueue(*RI.getTopLevelRegion(), Regions);
   while (!Regions.empty()) {
@@ -1410,9 +1436,22 @@ static bool runImpl(DominatorTree &DT, RegionInfo &RI,
       continue;
     }
 
-    Changed |= SCFG.run(R, &DT, &TTI);
+    Structurized |= SCFG.run(R, &DT, &TTI);
   }
-  return Changed;
+  if (!Structurized)
+    return Changed;
+
+  LI.releaseMemory();
+  LI.analyze(DT);
+  SmallDenseMap<Loop *, unsigned> NumOldLoops;
+  for (auto [Header, LoopID] : OldLoops)
+    if (Loop *L = LI.getLoopFor(Header))
+      ++NumOldLoops[L];
+  for (auto [Header, LoopID] : OldLoops)
+    if (Loop *L = LI.getLoopFor(Header);
+        L && LoopID && NumOldLoops.lookup(L) == 1)
+      L->setLoopID(LoopID);
+  return true;
 }
 
 bool StructurizeCFGLegacyPass::runOnFunction(Function &F) {
@@ -1421,15 +1460,17 @@ bool StructurizeCFGLegacyPass::runOnFunction(Function &F) {
   if (valueOr(Opts.structurizecfg_skip_uniform_regions, SkipUniformRegions))
     UI = &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
   DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   RegionInfo &RI = getAnalysis<RegionInfoPass>().getRegionInfo();
   const TargetTransformInfo &TTI =
       getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-  return runImpl(DT, RI, TTI, UI);
+  return runImpl(F, DT, LI, RI, TTI, UI);
 }
 
 PreservedAnalyses StructurizeCFGPass::run(Function &F,
                                           FunctionAnalysisManager &AM) {
   DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
   auto &RI = AM.getResult<RegionInfoAnalysis>(F);
   TargetTransformInfo &TTI = AM.getResult<TargetIRAnalysis>(F);
   const ScalarOptions &Opts = getScalarOptions(F);
@@ -1437,9 +1478,10 @@ PreservedAnalyses StructurizeCFGPass::run(Function &F,
   if (valueOr(Opts.structurizecfg_skip_uniform_regions, SkipUniformRegions))
     UI = &AM.getResult<UniformityInfoAnalysis>(F);
 
-  if (!runImpl(DT, RI, TTI, UI))
+  if (!runImpl(F, DT, LI, RI, TTI, UI))
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<LoopAnalysis>();
   return PA;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 757f7c9d48a43f..c441f222877ad7 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -77,7 +77,6 @@
 ; GCN-O0-NEXT:      Function Alias Analysis Results
 ; GCN-O0-NEXT:      Memory SSA
 ; GCN-O0-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O0-NEXT:      Natural Loop Information
 ; GCN-O0-NEXT:      SI annotate control flow
 ; GCN-O0-NEXT:      Cycle Info Analysis
 ; GCN-O0-NEXT:      Uniformity Analysis
@@ -282,7 +281,6 @@
 ; GCN-O1-NEXT:      Function Alias Analysis Results
 ; GCN-O1-NEXT:      Memory SSA
 ; GCN-O1-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O1-NEXT:      Natural Loop Information
 ; GCN-O1-NEXT:      SI annotate control flow
 ; GCN-O1-NEXT:      Cycle Info Analysis
 ; GCN-O1-NEXT:      Uniformity Analysis
@@ -596,7 +594,6 @@
 ; GCN-O1-OPTS-NEXT:      Function Alias Analysis Results
 ; GCN-O1-OPTS-NEXT:      Memory SSA
 ; GCN-O1-OPTS-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O1-OPTS-NEXT:      Natural Loop Information
 ; GCN-O1-OPTS-NEXT:      SI annotate control flow
 ; GCN-O1-OPTS-NEXT:      Cycle Info Analysis
 ; GCN-O1-OPTS-NEXT:      Uniformity Analysis
@@ -921,7 +918,6 @@
 ; GCN-O2-NEXT:      Function Alias Analysis Results
 ; GCN-O2-NEXT:      Memory SSA
 ; GCN-O2-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O2-NEXT:      Natural Loop Information
 ; GCN-O2-NEXT:      SI annotate control flow
 ; GCN-O2-NEXT:      Cycle Info Analysis
 ; GCN-O2-NEXT:      Uniformity Analysis
@@ -1263,7 +1259,6 @@
 ; GCN-O3-NEXT:      Function Alias Analysis Results
 ; GCN-O3-NEXT:      Memory SSA
 ; GCN-O3-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O3-NEXT:      Natural Loop Information
 ; GCN-O3-NEXT:      SI annotate control flow
 ; GCN-O3-NEXT:      Cycle Info Analysis
 ; GCN-O3-NEXT:      Uniformity Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll b/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll
index 0750368fb31d2a..909a3638da3c32 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll
@@ -47,6 +47,7 @@ define void @divergent_loop(ptr addrspace(1) %p, i32 %n) {
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b32 s2, 0
 ; GFX942-NEXT:    s_mov_b64 s[0:1], 0
+; GFX942-NEXT:    .p2align 6
 ; GFX942-NEXT:  .LBB1_1: ; %loop
 ; GFX942-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX942-NEXT:    s_ashr_i32 s3, s2, 31
@@ -88,6 +89,7 @@ define amdgpu_kernel void @uniform_loop_divergent_latches(ptr addrspace(1) %p, p
 ; GFX942-NEXT:    v_mov_b32_e32 v1, 2
 ; GFX942-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX942-NEXT:    s_branch .LBB2_2
+; GFX942-NEXT:    .p2align 6
 ; GFX942-NEXT:  .LBB2_1: ; %Flow4
 ; GFX942-NEXT:    ; in Loop: Header=BB2_2 Depth=1
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll b/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll
index fcd4e40e0008f5..c5ae252f6bd98a 100644
--- a/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll
+++ b/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll
@@ -12,7 +12,7 @@ define void @single_latch(i32 %n) {
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -58,7 +58,7 @@ define void @two_latches(i32 %n, i1 %c2) {
 ; CHECK:       [[FLOW2]]:
 ; CHECK-NEXT:    [[TMP3]] = phi i32 [ [[TMP0]], %[[FLOW1]] ], [ poison, %[[HEADER]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ false, %[[FLOW1]] ], [ true, %[[HEADER]] ]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -209,11 +209,11 @@ define void @nested_loops(i32 %n, i32 %m) {
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], %[[INNER]] ], [ 0, %[[OUTER]] ]
 ; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[C_INNER:%.*]] = icmp uge i32 [[J_NEXT]], [[M]]
-; CHECK-NEXT:    br i1 [[C_INNER]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK-NEXT:    br i1 [[C_INNER]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[OUTER_LATCH]]:
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[C_OUTER:%.*]] = icmp uge i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[C_OUTER]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK-NEXT:    br i1 [[C_OUTER]], label %[[EXIT:.*]], label %[[OUTER]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -262,7 +262,7 @@ define i32 @backedge_to_flow_block(i32 %s) {
 ; CHECK-NEXT:    [[TMP5]] = phi i32 [ [[I_NEXT]], %[[FLOW2]] ], [ [[TMP1]], %[[FLOW]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ [[TMP8:%.*]], %[[FLOW2]] ], [ true, %[[FLOW]] ]
 ; CHECK-NEXT:    [[TMP7]] = phi i1 [ [[TMP9:%.*]], %[[FLOW2]] ], [ [[TMP2]], %[[FLOW]] ]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[FLOW3:.*]], label %[[FLOW]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[FLOW3:.*]], label %[[FLOW]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[LATCH]]:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[I_NEXT]], 3
 ; CHECK-NEXT:    br label %[[FLOW2]]
@@ -314,14 +314,14 @@ define void @latches_in_nested_regions(i1 %c1, i1 %c2, i1 %c3, i1 %c4) {
 ; CHECK-NEXT:    br label %[[FLOW]]
 ; CHECK:       [[FLOW]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ [[C2_INV]], %[[A]] ], [ true, %[[HEADER]] ]
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[MID:.*]], label %[[HEADER]]
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[MID:.*]], label %[[HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MID]]:
 ; CHECK-NEXT:    br i1 [[C3]], label %[[B:.*]], label %[[FLOW1:.*]]
 ; CHECK:       [[B]]:
 ; CHECK-NEXT:    br label %[[FLOW1]]
 ; CHECK:       [[FLOW1]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[C4_INV]], %[[B]] ], [ true, %[[MID]] ]
-; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP6]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -394,7 +394,7 @@ define void @latches_below_top_level_header(i1 %c1, i1 %c2, i1 %c3) {
 ; CHECK-NEXT:    [[C3_INV:%.*]] = xor i1 [[C3]], true
 ; CHECK-NEXT:    br label %[[HEADER:.*]]
 ; CHECK:       [[FLOW1:.*]]:
-; CHECK-NEXT:    br label %[[HEADER]]
+; CHECK-NEXT:    br label %[[HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[HEADER]]:
 ; CHECK-NEXT:    br i1 [[C1]], label %[[BODY:.*]], label %[[EXIT1:.*]]
 ; CHECK:       [[BODY]]:
@@ -494,3 +494,13 @@ exit:
 !12 = distinct !{!12, !1}
 !13 = distinct !{!13, !14}
 !14 = !{!"llvm.loop.align", i32 32}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.align", i32 64}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
+;.



More information about the llvm-branch-commits mailing list