[llvm-branch-commits] [llvm] [StructurizeCFG] Keep loop metadata (PR #243015)

Dmitry Sidorov via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 18 12:06:58 PST 2026


https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/243015

>From 544f8fe9e53910609bc7fa6c739363d65ccfd7f3 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 16 Nov 2026 00:06:40 +0100
Subject: [PATCH 1/2] [StructurizeCFG] Keep loop metadata

The pass replaces the latches of a loop with new backedges and dropped
their !llvm.loop metadata, so loop hints such as llvm.loop.align had no
effect on the restructured loops.

Record the loop IDs before the rewrite, recompute LoopInfo after it and
set them again with Loop::setLoopID. A loop that several old loops are
merged into gets none, and functions with irreducible control flow are
left as before.
---
 llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 54 ++++++++++++++++---
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |  5 --
 .../CodeGen/AMDGPU/loop-align-metadata.ll     |  2 +
 .../StructurizeCFG/loop-metadata.ll           | 26 ++++++---
 4 files changed, 68 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index d94052d2837fb4..eeab34508d4a95 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -11,11 +11,14 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/RegionInfo.h"
 #include "llvm/Analysis/RegionIterator.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -377,10 +380,12 @@ class StructurizeCFGLegacyPass : public FunctionPass {
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<UniformityInfoWrapperPass>();
     AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addRequired<LoopInfoWrapperPass>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
     AU.addRequired<RegionInfoPass>();
 
     AU.addPreserved<DominatorTreeWrapperPass>();
+    AU.addPreserved<LoopInfoWrapperPass>();
   }
 };
 
@@ -392,6 +397,7 @@ INITIALIZE_PASS_BEGIN(StructurizeCFGLegacyPass, "structurizecfg",
                       "Structurize the CFG", false, false)
 INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(RegionInfoPass)
 INITIALIZE_PASS_END(StructurizeCFGLegacyPass, "structurizecfg",
@@ -1393,9 +1399,29 @@ void StructurizeCFGPass::printPipeline(
 
 /// Structurize every region in \p RI, processing inner regions before the
 /// regions that contain them. If \p UI is non-null, skip uniform regions.
-static bool runImpl(DominatorTree &DT, RegionInfo &RI,
-                    const TargetTransformInfo &TTI, const UniformityInfo *UI) {
+/// Recompute \p LI after a rewrite and carry a loop ID over to the loop that
+/// holds the old header, unless several old loops end up in that loop. Nothing
+/// is carried over for irreducible control flow, where a cycle that is not a
+/// natural loop may take over a loop header.
+static bool runImpl(Function &F, DominatorTree &DT, LoopInfo &LI,
+                    RegionInfo &RI, const TargetTransformInfo &TTI,
+                    const UniformityInfo *UI) {
+  SmallVector<std::pair<BasicBlock *, MDNode *>> OldLoops;
+  bool HasLoopID = false;
+  for (Loop *L : LI.getLoopsInPreorder()) {
+    MDNode *LoopID = L->getLoopID();
+    HasLoopID |= LoopID != nullptr;
+    OldLoops.push_back({L->getHeader(), LoopID});
+  }
+  if (HasLoopID) {
+    ReversePostOrderTraversal<const Function *> RPOT(&F);
+    HasLoopID = !containsIrreducibleCFG<const BasicBlock *>(RPOT, LI);
+  }
+  if (!HasLoopID)
+    OldLoops.clear();
+
   bool Changed = false;
+  bool Structurized = false;
   std::vector<Region *> Regions;
   addRegionIntoQueue(*RI.getTopLevelRegion(), Regions);
   while (!Regions.empty()) {
@@ -1410,9 +1436,22 @@ static bool runImpl(DominatorTree &DT, RegionInfo &RI,
       continue;
     }
 
-    Changed |= SCFG.run(R, &DT, &TTI);
+    Structurized |= SCFG.run(R, &DT, &TTI);
   }
-  return Changed;
+  if (!Structurized)
+    return Changed;
+
+  LI.releaseMemory();
+  LI.analyze(DT);
+  SmallDenseMap<Loop *, unsigned> NumOldLoops;
+  for (auto [Header, LoopID] : OldLoops)
+    if (Loop *L = LI.getLoopFor(Header))
+      ++NumOldLoops[L];
+  for (auto [Header, LoopID] : OldLoops)
+    if (Loop *L = LI.getLoopFor(Header);
+        L && LoopID && NumOldLoops.lookup(L) == 1)
+      L->setLoopID(LoopID);
+  return true;
 }
 
 bool StructurizeCFGLegacyPass::runOnFunction(Function &F) {
@@ -1421,15 +1460,17 @@ bool StructurizeCFGLegacyPass::runOnFunction(Function &F) {
   if (valueOr(Opts.structurizecfg_skip_uniform_regions, SkipUniformRegions))
     UI = &getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
   DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   RegionInfo &RI = getAnalysis<RegionInfoPass>().getRegionInfo();
   const TargetTransformInfo &TTI =
       getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-  return runImpl(DT, RI, TTI, UI);
+  return runImpl(F, DT, LI, RI, TTI, UI);
 }
 
 PreservedAnalyses StructurizeCFGPass::run(Function &F,
                                           FunctionAnalysisManager &AM) {
   DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
   auto &RI = AM.getResult<RegionInfoAnalysis>(F);
   TargetTransformInfo &TTI = AM.getResult<TargetIRAnalysis>(F);
   const ScalarOptions &Opts = getScalarOptions(F);
@@ -1437,9 +1478,10 @@ PreservedAnalyses StructurizeCFGPass::run(Function &F,
   if (valueOr(Opts.structurizecfg_skip_uniform_regions, SkipUniformRegions))
     UI = &AM.getResult<UniformityInfoAnalysis>(F);
 
-  if (!runImpl(DT, RI, TTI, UI))
+  if (!runImpl(F, DT, LI, RI, TTI, UI))
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<LoopAnalysis>();
   return PA;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 757f7c9d48a43f..c441f222877ad7 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -77,7 +77,6 @@
 ; GCN-O0-NEXT:      Function Alias Analysis Results
 ; GCN-O0-NEXT:      Memory SSA
 ; GCN-O0-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O0-NEXT:      Natural Loop Information
 ; GCN-O0-NEXT:      SI annotate control flow
 ; GCN-O0-NEXT:      Cycle Info Analysis
 ; GCN-O0-NEXT:      Uniformity Analysis
@@ -282,7 +281,6 @@
 ; GCN-O1-NEXT:      Function Alias Analysis Results
 ; GCN-O1-NEXT:      Memory SSA
 ; GCN-O1-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O1-NEXT:      Natural Loop Information
 ; GCN-O1-NEXT:      SI annotate control flow
 ; GCN-O1-NEXT:      Cycle Info Analysis
 ; GCN-O1-NEXT:      Uniformity Analysis
@@ -596,7 +594,6 @@
 ; GCN-O1-OPTS-NEXT:      Function Alias Analysis Results
 ; GCN-O1-OPTS-NEXT:      Memory SSA
 ; GCN-O1-OPTS-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O1-OPTS-NEXT:      Natural Loop Information
 ; GCN-O1-OPTS-NEXT:      SI annotate control flow
 ; GCN-O1-OPTS-NEXT:      Cycle Info Analysis
 ; GCN-O1-OPTS-NEXT:      Uniformity Analysis
@@ -921,7 +918,6 @@
 ; GCN-O2-NEXT:      Function Alias Analysis Results
 ; GCN-O2-NEXT:      Memory SSA
 ; GCN-O2-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O2-NEXT:      Natural Loop Information
 ; GCN-O2-NEXT:      SI annotate control flow
 ; GCN-O2-NEXT:      Cycle Info Analysis
 ; GCN-O2-NEXT:      Uniformity Analysis
@@ -1263,7 +1259,6 @@
 ; GCN-O3-NEXT:      Function Alias Analysis Results
 ; GCN-O3-NEXT:      Memory SSA
 ; GCN-O3-NEXT:      AMDGPU Annotate Uniform Values
-; GCN-O3-NEXT:      Natural Loop Information
 ; GCN-O3-NEXT:      SI annotate control flow
 ; GCN-O3-NEXT:      Cycle Info Analysis
 ; GCN-O3-NEXT:      Uniformity Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll b/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll
index 0750368fb31d2a..909a3638da3c32 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-align-metadata.ll
@@ -47,6 +47,7 @@ define void @divergent_loop(ptr addrspace(1) %p, i32 %n) {
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    s_mov_b32 s2, 0
 ; GFX942-NEXT:    s_mov_b64 s[0:1], 0
+; GFX942-NEXT:    .p2align 6
 ; GFX942-NEXT:  .LBB1_1: ; %loop
 ; GFX942-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX942-NEXT:    s_ashr_i32 s3, s2, 31
@@ -88,6 +89,7 @@ define amdgpu_kernel void @uniform_loop_divergent_latches(ptr addrspace(1) %p, p
 ; GFX942-NEXT:    v_mov_b32_e32 v1, 2
 ; GFX942-NEXT:    v_mov_b32_e32 v2, 1
 ; GFX942-NEXT:    s_branch .LBB2_2
+; GFX942-NEXT:    .p2align 6
 ; GFX942-NEXT:  .LBB2_1: ; %Flow4
 ; GFX942-NEXT:    ; in Loop: Header=BB2_2 Depth=1
 ; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
diff --git a/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll b/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll
index fcd4e40e0008f5..c5ae252f6bd98a 100644
--- a/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll
+++ b/llvm/test/Transforms/StructurizeCFG/loop-metadata.ll
@@ -12,7 +12,7 @@ define void @single_latch(i32 %n) {
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp uge i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -58,7 +58,7 @@ define void @two_latches(i32 %n, i1 %c2) {
 ; CHECK:       [[FLOW2]]:
 ; CHECK-NEXT:    [[TMP3]] = phi i32 [ [[TMP0]], %[[FLOW1]] ], [ poison, %[[HEADER]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i1 [ false, %[[FLOW1]] ], [ true, %[[HEADER]] ]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -209,11 +209,11 @@ define void @nested_loops(i32 %n, i32 %m) {
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], %[[INNER]] ], [ 0, %[[OUTER]] ]
 ; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[C_INNER:%.*]] = icmp uge i32 [[J_NEXT]], [[M]]
-; CHECK-NEXT:    br i1 [[C_INNER]], label %[[OUTER_LATCH]], label %[[INNER]]
+; CHECK-NEXT:    br i1 [[C_INNER]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[OUTER_LATCH]]:
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[C_OUTER:%.*]] = icmp uge i32 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[C_OUTER]], label %[[EXIT:.*]], label %[[OUTER]]
+; CHECK-NEXT:    br i1 [[C_OUTER]], label %[[EXIT:.*]], label %[[OUTER]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -262,7 +262,7 @@ define i32 @backedge_to_flow_block(i32 %s) {
 ; CHECK-NEXT:    [[TMP5]] = phi i32 [ [[I_NEXT]], %[[FLOW2]] ], [ [[TMP1]], %[[FLOW]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = phi i1 [ [[TMP8:%.*]], %[[FLOW2]] ], [ true, %[[FLOW]] ]
 ; CHECK-NEXT:    [[TMP7]] = phi i1 [ [[TMP9:%.*]], %[[FLOW2]] ], [ [[TMP2]], %[[FLOW]] ]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[FLOW3:.*]], label %[[FLOW]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[FLOW3:.*]], label %[[FLOW]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[LATCH]]:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[I_NEXT]], 3
 ; CHECK-NEXT:    br label %[[FLOW2]]
@@ -314,14 +314,14 @@ define void @latches_in_nested_regions(i1 %c1, i1 %c2, i1 %c3, i1 %c4) {
 ; CHECK-NEXT:    br label %[[FLOW]]
 ; CHECK:       [[FLOW]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i1 [ [[C2_INV]], %[[A]] ], [ true, %[[HEADER]] ]
-; CHECK-NEXT:    br i1 [[TMP0]], label %[[MID:.*]], label %[[HEADER]]
+; CHECK-NEXT:    br i1 [[TMP0]], label %[[MID:.*]], label %[[HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MID]]:
 ; CHECK-NEXT:    br i1 [[C3]], label %[[B:.*]], label %[[FLOW1:.*]]
 ; CHECK:       [[B]]:
 ; CHECK-NEXT:    br label %[[FLOW1]]
 ; CHECK:       [[FLOW1]]:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i1 [ [[C4_INV]], %[[B]] ], [ true, %[[MID]] ]
-; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[EXIT:.*]], label %[[HEADER]], !llvm.loop [[LOOP6]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -394,7 +394,7 @@ define void @latches_below_top_level_header(i1 %c1, i1 %c2, i1 %c3) {
 ; CHECK-NEXT:    [[C3_INV:%.*]] = xor i1 [[C3]], true
 ; CHECK-NEXT:    br label %[[HEADER:.*]]
 ; CHECK:       [[FLOW1:.*]]:
-; CHECK-NEXT:    br label %[[HEADER]]
+; CHECK-NEXT:    br label %[[HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[HEADER]]:
 ; CHECK-NEXT:    br i1 [[C1]], label %[[BODY:.*]], label %[[EXIT1:.*]]
 ; CHECK:       [[BODY]]:
@@ -494,3 +494,13 @@ exit:
 !12 = distinct !{!12, !1}
 !13 = distinct !{!13, !14}
 !14 = !{!"llvm.loop.align", i32 32}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.align", i32 64}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
+;.

>From 05f51d7a8bfee6ef0a517fef042faa0872a417c7 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Tue, 17 Nov 2026 16:40:13 +0100
Subject: [PATCH 2/2] apply comments

---
 llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index eeab34508d4a95..be558423b9db15 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -1443,14 +1443,17 @@ static bool runImpl(Function &F, DominatorTree &DT, LoopInfo &LI,
 
   LI.releaseMemory();
   LI.analyze(DT);
-  SmallDenseMap<Loop *, unsigned> NumOldLoops;
-  for (auto [Header, LoopID] : OldLoops)
-    if (Loop *L = LI.getLoopFor(Header))
-      ++NumOldLoops[L];
-  for (auto [Header, LoopID] : OldLoops)
-    if (Loop *L = LI.getLoopFor(Header);
-        L && LoopID && NumOldLoops.lookup(L) == 1)
+  SmallMapVector<Loop *, MDNode *, 8> LoopIDs;
+  for (auto [Header, LoopID] : OldLoops) {
+    if (Loop *L = LI.getLoopFor(Header)) {
+      if (auto [It, Inserted] = LoopIDs.try_emplace(L, LoopID); !Inserted)
+        It->second = nullptr;
+    }
+  }
+  for (auto [L, LoopID] : LoopIDs) {
+    if (LoopID)
       L->setLoopID(LoopID);
+  }
   return true;
 }
 



More information about the llvm-branch-commits mailing list