[llvm] [LoopInterchange] Add metadata to control loop-interchange (PR #127474)
Ryotaro Kasuga via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 17 03:31:57 PST 2025
https://github.com/kasuga-fj created https://github.com/llvm/llvm-project/pull/127474
This patch adds metadata to enable/disable the loop-interchange for a loop nest. This is a prelude to introduce a new pragma directive for loop-interchange, like other loop optimizations (unroll, vectorize, distribute, etc.) have.
>From 5d41708ecf44d05acf457b366a90759c39c66630 Mon Sep 17 00:00:00 2001
From: Ryotaro Kasuga <kasuga.ryotaro at fujitsu.com>
Date: Mon, 17 Feb 2025 11:23:46 +0000
Subject: [PATCH] [LoopInterchange] Add metadata to control loop-interchange
This patch adds metadata to enable/disable the loop-interchange for a
loop nest. This is a prelude to introduce a new pragma directive for
loop-interchange, like other loop optimizations (unroll, vectorize,
distribute, etc.) have.
---
.../lib/Transforms/Scalar/LoopInterchange.cpp | 168 ++++++---
.../Transforms/LoopInterchange/metadata.ll | 325 ++++++++++++++++++
2 files changed, 446 insertions(+), 47 deletions(-)
create mode 100644 llvm/test/Transforms/LoopInterchange/metadata.ll
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index f45d90ff13e14..d2f4ede6176ac 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -67,8 +67,6 @@ static cl::opt<unsigned int> MaxMemInstrCount(
namespace {
-using LoopVector = SmallVector<Loop *, 8>;
-
// TODO: Check if we can use a sparse matrix here.
using CharMatrix = std::vector<std::vector<char>>;
@@ -84,6 +82,14 @@ static cl::opt<unsigned int> MaxLoopNestDepth(
"loop-interchange-max-loop-nest-depth", cl::init(10), cl::Hidden,
cl::desc("Maximum depth of loop nest considered for the transform"));
+// Whether to apply by default.
+// TODO: Once this pass is enabled by default, remove this option and use the
+// value of PipelineTuningOptions.
+static cl::opt<bool> OnlyWhenForced(
+ "loop-interchange-only-when-forced", cl::init(false), cl::ReallyHidden,
+ cl::desc(
+ "Apply interchanges only when explicitly specified metadata exists"));
+
#ifndef NDEBUG
static void printDepMatrix(CharMatrix &DepMatrix) {
for (auto &Row : DepMatrix) {
@@ -233,7 +239,7 @@ static bool isLegalToInterChangeLoops(CharMatrix &DepMatrix,
return true;
}
-static void populateWorklist(Loop &L, LoopVector &LoopList) {
+static void populateWorklist(Loop &L, SmallVectorImpl<Loop *> &LoopList) {
LLVM_DEBUG(dbgs() << "Calling populateWorklist on Func: "
<< L.getHeader()->getParent()->getName() << " Loop: %"
<< L.getHeader()->getName() << '\n');
@@ -245,7 +251,7 @@ static void populateWorklist(Loop &L, LoopVector &LoopList) {
// nested.
// Discard all loops above it added into Worklist.
if (Vec->size() != 1) {
- LoopList = {};
+ LoopList.clear();
return;
}
@@ -256,27 +262,6 @@ static void populateWorklist(Loop &L, LoopVector &LoopList) {
LoopList.push_back(CurrentLoop);
}
-static bool hasSupportedLoopDepth(SmallVectorImpl<Loop *> &LoopList,
- OptimizationRemarkEmitter &ORE) {
- unsigned LoopNestDepth = LoopList.size();
- if (LoopNestDepth < MinLoopNestDepth || LoopNestDepth > MaxLoopNestDepth) {
- LLVM_DEBUG(dbgs() << "Unsupported depth of loop nest " << LoopNestDepth
- << ", the supported range is [" << MinLoopNestDepth
- << ", " << MaxLoopNestDepth << "].\n");
- Loop **OuterLoop = LoopList.begin();
- ORE.emit([&]() {
- return OptimizationRemarkMissed(DEBUG_TYPE, "UnsupportedLoopNestDepth",
- (*OuterLoop)->getStartLoc(),
- (*OuterLoop)->getHeader())
- << "Unsupported depth of loop nest, the supported range is ["
- << std::to_string(MinLoopNestDepth) << ", "
- << std::to_string(MaxLoopNestDepth) << "].\n";
- });
- return false;
- }
- return true;
-}
-
static bool isComputableLoopNest(ScalarEvolution *SE,
ArrayRef<Loop *> LoopList) {
for (Loop *L : LoopList) {
@@ -299,6 +284,26 @@ static bool isComputableLoopNest(ScalarEvolution *SE,
namespace {
+/// LoopInterchangeList manages the list of loops and the range to which the
+/// interchange may be applied.
+struct LoopInterchangeList {
+ SmallVector<Loop *, 8> LoopList;
+ unsigned ListBegin = 0;
+ unsigned ListEnd = 0;
+
+ LoopInterchangeList(LoopNest &LN)
+ : LoopList(LN.getLoops()), ListBegin(0), ListEnd(LoopList.size()) {}
+
+ LoopInterchangeList(Loop &L) {
+ populateWorklist(L, LoopList);
+ ListBegin = 0;
+ ListEnd = LoopList.size();
+ }
+
+ void checkMetadata(bool OnlyWhenForced);
+ bool hasSupportedLoopDepth(OptimizationRemarkEmitter &ORE);
+};
+
/// LoopInterchangeLegality checks if it is legal to interchange the loop.
class LoopInterchangeLegality {
public:
@@ -439,39 +444,38 @@ struct LoopInterchange {
bool run(Loop *L) {
if (L->getParentLoop())
return false;
- SmallVector<Loop *, 8> LoopList;
- populateWorklist(*L, LoopList);
- return processLoopList(LoopList);
+ LoopInterchangeList LIL(*L);
+ return processLoopList(LIL);
}
- bool run(LoopNest &LN) {
- SmallVector<Loop *, 8> LoopList(LN.getLoops());
+ bool run(LoopInterchangeList &LIL) {
+ const auto &LoopList = LIL.LoopList;
for (unsigned I = 1; I < LoopList.size(); ++I)
if (LoopList[I]->getParentLoop() != LoopList[I - 1])
return false;
- return processLoopList(LoopList);
+ return processLoopList(LIL);
}
- unsigned selectLoopForInterchange(ArrayRef<Loop *> LoopList) {
+ unsigned selectLoopForInterchange(LoopInterchangeList &LIL) {
// TODO: Add a better heuristic to select the loop to be interchanged based
// on the dependence matrix. Currently we select the innermost loop.
- return LoopList.size() - 1;
+ return LIL.ListEnd - 1;
}
- bool processLoopList(SmallVectorImpl<Loop *> &LoopList) {
+ bool processLoopList(LoopInterchangeList &LIL) {
bool Changed = false;
// Ensure proper loop nest depth.
- assert(hasSupportedLoopDepth(LoopList, *ORE) &&
+ assert(LIL.hasSupportedLoopDepth(*ORE) &&
"Unsupported depth of loop nest.");
- unsigned LoopNestDepth = LoopList.size();
+ unsigned LoopNestDepth = LIL.LoopList.size();
LLVM_DEBUG(dbgs() << "Processing LoopList of size = " << LoopNestDepth
<< "\n");
CharMatrix DependencyMatrix;
- Loop *OuterMostLoop = *(LoopList.begin());
+ Loop *OuterMostLoop = *(LIL.LoopList.begin());
if (!populateDependencyMatrix(DependencyMatrix, LoopNestDepth,
OuterMostLoop, DI, SE, ORE)) {
LLVM_DEBUG(dbgs() << "Populating dependency matrix failed\n");
@@ -488,7 +492,7 @@ struct LoopInterchange {
return false;
}
- unsigned SelecLoopId = selectLoopForInterchange(LoopList);
+ unsigned SelectLoopId = selectLoopForInterchange(LIL);
// Obtain the loop vector returned from loop cache analysis beforehand,
// and put each <Loop, index> pair into a map for constant time query
// later. Indices in loop vector reprsent the optimal order of the
@@ -504,19 +508,20 @@ struct LoopInterchange {
CostMap[LoopCosts[i].first] = i;
}
}
+
// We try to achieve the globally optimal memory access for the loopnest,
// and do interchange based on a bubble-sort fasion. We start from
// the innermost loop, move it outwards to the best possible position
// and repeat this process.
- for (unsigned j = SelecLoopId; j > 0; j--) {
+ for (unsigned j = LIL.ListEnd - LIL.ListBegin - 1; j > 0; j--) {
bool ChangedPerIter = false;
- for (unsigned i = SelecLoopId; i > SelecLoopId - j; i--) {
- bool Interchanged = processLoop(LoopList[i], LoopList[i - 1], i, i - 1,
- DependencyMatrix, CostMap);
+ for (unsigned i = SelectLoopId; i > SelectLoopId - j; i--) {
+ bool Interchanged = processLoop(LIL.LoopList[i], LIL.LoopList[i - 1], i,
+ i - 1, DependencyMatrix, CostMap);
if (!Interchanged)
continue;
// Loops interchanged, update LoopList accordingly.
- std::swap(LoopList[i - 1], LoopList[i]);
+ std::swap(LIL.LoopList[i - 1], LIL.LoopList[i]);
// Update the DependencyMatrix
interChangeDependencies(DependencyMatrix, i, i - 1);
@@ -526,6 +531,7 @@ struct LoopInterchange {
ChangedPerIter |= Interchanged;
Changed |= Interchanged;
}
+
// Early abort if there was no interchange during an entire round of
// moving loops outwards.
if (!ChangedPerIter)
@@ -572,6 +578,69 @@ struct LoopInterchange {
} // end anonymous namespace
+bool LoopInterchangeList::hasSupportedLoopDepth(
+ OptimizationRemarkEmitter &ORE) {
+ unsigned LoopNestDepth = ListEnd - ListBegin;
+ if (LoopNestDepth < MinLoopNestDepth || LoopNestDepth > MaxLoopNestDepth) {
+ LLVM_DEBUG(dbgs() << "Unsupported depth of loop nest " << LoopNestDepth
+ << ", the supported range is [" << MinLoopNestDepth
+ << ", " << MaxLoopNestDepth << "].\n");
+ Loop *OuterLoop = LoopList[ListBegin];
+ ORE.emit([&]() {
+ return OptimizationRemarkMissed(DEBUG_TYPE, "UnsupportedLoopNestDepth",
+ OuterLoop->getStartLoc(),
+ OuterLoop->getHeader())
+ << "Unsupported depth of loop nest, the supported range is ["
+ << std::to_string(MinLoopNestDepth) << ", "
+ << std::to_string(MaxLoopNestDepth) << "].\n";
+ });
+ return false;
+ }
+ return true;
+}
+
+// Check the metadata for interchange. The outermost one is taken into account
+// and nested ones are ignored. The metadata affects the entire loop nest such
+// that the outermost loop is the loop for which the metadata is specified. For
+// example, in the following example, the loop-interchange will be performed
+// only to the outermost two loops.
+//
+// for (...)
+// for (...)
+// #pragma clang loop interchange(disable)
+// for (...)
+// for (...)
+// for (...)
+// Stmt
+//
+void LoopInterchangeList::checkMetadata(bool OnlyWhenForced) {
+ ListBegin = 0;
+ ListEnd = LoopList.size();
+
+ for (unsigned I = 0; I != LoopList.size(); I++) {
+ Loop *L = LoopList[I];
+ auto Value = findStringMetadataForLoop(L, "llvm.loop.interchange.enable");
+ if (!Value)
+ continue;
+
+ const MDOperand *Op = *Value;
+ assert(Op && mdconst::hasa<ConstantInt>(*Op) && "invalid metadata");
+ bool Enabled = mdconst::extract<ConstantInt>(*Op)->getZExtValue();
+ if (Enabled && OnlyWhenForced) {
+ ListBegin = I;
+ } else if (!Enabled && !OnlyWhenForced) {
+ ListEnd = I;
+ } else if (OnlyWhenForced) {
+ ListEnd = 0;
+ }
+ break;
+ }
+
+ LLVM_DEBUG(
+ dbgs() << "LoopInterchange will be applied to the range: [from, to]=["
+ << ListBegin << ", " << ListEnd - 1 << "]\n";);
+}
+
bool LoopInterchangeLegality::containsUnsafeInstructions(BasicBlock *BB) {
return any_of(*BB, [](const Instruction &I) {
return I.mayHaveSideEffects() || I.mayReadFromMemory();
@@ -1748,7 +1817,7 @@ PreservedAnalyses LoopInterchangePass::run(LoopNest &LN,
LoopStandardAnalysisResults &AR,
LPMUpdater &U) {
Function &F = *LN.getParent();
- SmallVector<Loop *, 8> LoopList(LN.getLoops());
+ LoopInterchangeList LIL(LN);
if (MaxMemInstrCount < 1) {
LLVM_DEBUG(dbgs() << "MaxMemInstrCount should be at least 1");
@@ -1757,14 +1826,19 @@ PreservedAnalyses LoopInterchangePass::run(LoopNest &LN,
OptimizationRemarkEmitter ORE(&F);
// Ensure minimum depth of the loop nest to do the interchange.
- if (!hasSupportedLoopDepth(LoopList, ORE))
+ if (!LIL.hasSupportedLoopDepth(ORE))
return PreservedAnalyses::all();
// Ensure computable loop nest.
- if (!isComputableLoopNest(&AR.SE, LoopList)) {
+ if (!isComputableLoopNest(&AR.SE, LIL.LoopList)) {
LLVM_DEBUG(dbgs() << "Not valid loop candidate for interchange\n");
return PreservedAnalyses::all();
}
+ LIL.checkMetadata(OnlyWhenForced);
+ // Ensure the depth again.
+ if (!LIL.hasSupportedLoopDepth(ORE))
+ return PreservedAnalyses::all();
+
ORE.emit([&]() {
return OptimizationRemarkAnalysis(DEBUG_TYPE, "Dependence",
LN.getOutermostLoop().getStartLoc(),
@@ -1776,7 +1850,7 @@ PreservedAnalyses LoopInterchangePass::run(LoopNest &LN,
std::unique_ptr<CacheCost> CC =
CacheCost::getCacheCost(LN.getOutermostLoop(), AR, DI);
- if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, CC, &ORE).run(LN))
+ if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, CC, &ORE).run(LIL))
return PreservedAnalyses::all();
U.markLoopNestChanged(true);
return getLoopPassPreservedAnalyses();
diff --git a/llvm/test/Transforms/LoopInterchange/metadata.ll b/llvm/test/Transforms/LoopInterchange/metadata.ll
new file mode 100644
index 0000000000000..9838abb905a7e
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/metadata.ll
@@ -0,0 +1,325 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=loop-interchange -loop-interchange-only-when-forced=0 --cache-line-size=64 -S < %s | FileCheck %s --check-prefix=DEFAULT-ON
+; RUN: opt -passes=loop-interchange -loop-interchange-only-when-forced=1 --cache-line-size=64 -S < %s | FileCheck %s --check-prefix=DEFAULT-OFF
+
+; Test if the metadata works correctly. The code is as follows:
+;
+; #define N 4
+; int a[N][N][N][N];
+; int b[N][N][N][N];
+; void f() {
+; for (int i = 0; i < N; i++)
+; for (int j = 0; j < N; j++)
+; #pragma clang loop interchange(enable or disable)
+; for (int k = 0; k < N; k++)
+; for (int l = 0; l < N; l++)
+; a[l][k][j][i] += b[l][k][j][i];
+; }
+;
+; In the functions explicit_on and explicit_off, the values enable and disable
+; are specified in the pragma, respectively. If the
+; `loop-interchange-only-when-forced` is set to 0, the loop-interchange will be
+; performed to the loop nest unless it is explicitly disabled. If the value is
+; set to 1, the loop-interchange will be performed to the loop nest only when
+; it is explicitly enabled.
+
+ at a = dso_local local_unnamed_addr global [2 x [2 x [2 x [2 x i32]]]] zeroinitializer, align 4
+ at b = dso_local local_unnamed_addr global [2 x [2 x [2 x [2 x i32]]]] zeroinitializer, align 4
+
+define void @explicit_on() {
+; DEFAULT-ON-LABEL: define void @explicit_on() {
+; DEFAULT-ON-NEXT: [[ENTRY:.*:]]
+; DEFAULT-ON-NEXT: br label %[[FOR_BODY12_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND1_PREHEADER_PREHEADER:.*]]:
+; DEFAULT-ON-NEXT: br label %[[FOR_COND1_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND1_PREHEADER]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV61:%.*]] = phi i64 [ [[INDVARS_IV_NEXT62:%.*]], %[[FOR_COND_CLEANUP3:.*]] ], [ 0, %[[FOR_COND1_PREHEADER_PREHEADER]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_BODY12_SPLIT1:.*]]
+; DEFAULT-ON: [[FOR_COND5_PREHEADER_PREHEADER:.*]]:
+; DEFAULT-ON-NEXT: br label %[[FOR_COND5_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP3]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT62]] = add nuw nsw i64 [[INDVARS_IV61]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND64:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT62]], 2
+; DEFAULT-ON-NEXT: br i1 [[EXITCOND64]], label %[[FOR_COND1_PREHEADER]], label %[[FOR_COND_CLEANUP7_SPLIT:.*]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP7:.*]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT58:%.*]] = add nuw nsw i64 [[INDVARS_IV57:%.*]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND60:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT58]], 2
+; DEFAULT-ON-NEXT: br label %[[FOR_COND_CLEANUP3]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP7_SPLIT]]:
+; DEFAULT-ON-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[INDVARS_IV57]], 1
+; DEFAULT-ON-NEXT: [[TMP1:%.*]] = icmp ne i64 [[TMP0]], 2
+; DEFAULT-ON-NEXT: br i1 [[TMP1]], label %[[FOR_COND5_PREHEADER]], label %[[FOR_COND_CLEANUP11_SPLIT:.*]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP11:.*]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT54:%.*]] = add nuw nsw i64 [[INDVARS_IV53:%.*]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND56:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT54]], 2
+; DEFAULT-ON-NEXT: br label %[[FOR_COND_CLEANUP7]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP11_SPLIT]]:
+; DEFAULT-ON-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[INDVARS_IV53]], 1
+; DEFAULT-ON-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP2]], 2
+; DEFAULT-ON-NEXT: br i1 [[TMP3]], label %[[FOR_COND9_PREHEADER:.*]], label %[[FOR_BODY12_SPLIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; DEFAULT-ON: [[FOR_BODY12:.*]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP6:%.*]], %[[FOR_BODY12_SPLIT]] ], [ 0, %[[FOR_BODY12_PREHEADER]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_COND9_PREHEADER_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_BODY12_SPLIT1]]:
+; DEFAULT-ON-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @b, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-ON-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX18]], align 4
+; DEFAULT-ON-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @a, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-ON-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX26]], align 4
+; DEFAULT-ON-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP5]], [[TMP4]]
+; DEFAULT-ON-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX26]], align 4
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 2
+; DEFAULT-ON-NEXT: br label %[[FOR_COND_CLEANUP11]]
+; DEFAULT-ON: [[FOR_BODY12_SPLIT]]:
+; DEFAULT-ON-NEXT: [[TMP6]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DEFAULT-ON-NEXT: [[TMP7:%.*]] = icmp ne i64 [[TMP6]], 2
+; DEFAULT-ON-NEXT: br i1 [[TMP7]], label %[[FOR_BODY12]], label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT-ON: [[FOR_COND9_PREHEADER]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV53]] = phi i64 [ [[TMP2]], %[[FOR_COND_CLEANUP11_SPLIT]] ], [ 0, %[[FOR_COND9_PREHEADER_PREHEADER]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_COND5_PREHEADER_PREHEADER]]
+; DEFAULT-ON: [[FOR_BODY12_PREHEADER]]:
+; DEFAULT-ON-NEXT: br label %[[FOR_BODY12]]
+; DEFAULT-ON: [[FOR_COND5_PREHEADER]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV57]] = phi i64 [ [[TMP0]], %[[FOR_COND_CLEANUP7_SPLIT]] ], [ 0, %[[FOR_COND5_PREHEADER_PREHEADER]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_COND1_PREHEADER_PREHEADER]]
+; DEFAULT-ON: [[FOR_COND9_PREHEADER_PREHEADER]]:
+; DEFAULT-ON-NEXT: br label %[[FOR_COND9_PREHEADER]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP]]:
+; DEFAULT-ON-NEXT: ret void
+;
+; DEFAULT-OFF-LABEL: define void @explicit_on() {
+; DEFAULT-OFF-NEXT: [[ENTRY:.*]]:
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND1_PREHEADER:.*]]
+; DEFAULT-OFF: [[FOR_COND1_PREHEADER]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV61:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT62:%.*]], %[[FOR_COND_CLEANUP3:.*]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND5_PREHEADER:.*]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP3]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT62]] = add nuw nsw i64 [[INDVARS_IV61]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND64:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT62]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND64]], label %[[FOR_COND1_PREHEADER]], label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP7:.*]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT58:%.*]] = add nuw nsw i64 [[INDVARS_IV57:%.*]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND60:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT58]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND60]], label %[[FOR_COND5_PREHEADER]], label %[[FOR_COND_CLEANUP3]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP11:.*]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT54:%.*]] = add nuw nsw i64 [[INDVARS_IV53:%.*]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND56:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT54]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND56]], label %[[FOR_COND9_PREHEADER:.*]], label %[[FOR_BODY12_SPLIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; DEFAULT-OFF: [[FOR_BODY12:.*]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[TMP2:%.*]], %[[FOR_BODY12_SPLIT]] ], [ 0, %[[FOR_BODY12_PREHEADER:.*]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND9_PREHEADER_PREHEADER:.*]]
+; DEFAULT-OFF: [[FOR_BODY12_SPLIT1:.*]]:
+; DEFAULT-OFF-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @b, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-OFF-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX18]], align 4
+; DEFAULT-OFF-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @a, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-OFF-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX26]], align 4
+; DEFAULT-OFF-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; DEFAULT-OFF-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX26]], align 4
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 2
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND_CLEANUP11]]
+; DEFAULT-OFF: [[FOR_BODY12_SPLIT]]:
+; DEFAULT-OFF-NEXT: [[TMP2]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DEFAULT-OFF-NEXT: [[TMP3:%.*]] = icmp ne i64 [[TMP2]], 2
+; DEFAULT-OFF-NEXT: br i1 [[TMP3]], label %[[FOR_BODY12]], label %[[FOR_COND_CLEANUP7]]
+; DEFAULT-OFF: [[FOR_COND9_PREHEADER]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV53]] = phi i64 [ [[INDVARS_IV_NEXT54]], %[[FOR_COND_CLEANUP11]] ], [ 0, %[[FOR_COND9_PREHEADER_PREHEADER]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_BODY12_SPLIT1]]
+; DEFAULT-OFF: [[FOR_BODY12_PREHEADER]]:
+; DEFAULT-OFF-NEXT: br label %[[FOR_BODY12]]
+; DEFAULT-OFF: [[FOR_COND5_PREHEADER]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV57]] = phi i64 [ 0, %[[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT58]], %[[FOR_COND_CLEANUP7]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_BODY12_PREHEADER]]
+; DEFAULT-OFF: [[FOR_COND9_PREHEADER_PREHEADER]]:
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND9_PREHEADER]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP]]:
+; DEFAULT-OFF-NEXT: ret void
+;
+entry:
+ br label %for.cond1.preheader
+
+for.cond1.preheader:
+ %indvars.iv61 = phi i64 [ 0, %entry ], [ %indvars.iv.next62, %for.cond.cleanup3 ]
+ br label %for.cond5.preheader
+
+for.cond.cleanup3:
+ %indvars.iv.next62 = add nuw nsw i64 %indvars.iv61, 1
+ %exitcond64 = icmp ne i64 %indvars.iv.next62, 2
+ br i1 %exitcond64, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.cond.cleanup7:
+ %indvars.iv.next58 = add nuw nsw i64 %indvars.iv57, 1
+ %exitcond60 = icmp ne i64 %indvars.iv.next58, 2
+ br i1 %exitcond60, label %for.cond5.preheader, label %for.cond.cleanup3
+
+for.cond.cleanup11:
+ %indvars.iv.next54 = add nuw nsw i64 %indvars.iv53, 1
+ %exitcond56 = icmp ne i64 %indvars.iv.next54, 2
+ br i1 %exitcond56, label %for.cond9.preheader, label %for.cond.cleanup7, !llvm.loop !0
+
+for.body12:
+ %indvars.iv = phi i64 [ 0, %for.cond9.preheader ], [ %indvars.iv.next, %for.body12 ]
+ %arrayidx18 = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @b, i64 0, i64 %indvars.iv, i64 %indvars.iv53, i64 %indvars.iv57, i64 %indvars.iv61
+ %0 = load i32, ptr %arrayidx18, align 4
+ %arrayidx26 = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @a, i64 0, i64 %indvars.iv, i64 %indvars.iv53, i64 %indvars.iv57, i64 %indvars.iv61
+ %1 = load i32, ptr %arrayidx26, align 4
+ %add = add nsw i32 %1, %0
+ store i32 %add, ptr %arrayidx26, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, 2
+ br i1 %exitcond, label %for.body12, label %for.cond.cleanup11
+
+for.cond9.preheader:
+ %indvars.iv53 = phi i64 [ 0, %for.cond5.preheader ], [ %indvars.iv.next54, %for.cond.cleanup11 ]
+ br label %for.body12
+
+for.cond5.preheader:
+ %indvars.iv57 = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next58, %for.cond.cleanup7 ]
+ br label %for.cond9.preheader
+
+for.cond.cleanup:
+ ret void
+}
+
+define void @explicit_off() {
+; DEFAULT-ON-LABEL: define void @explicit_off() {
+; DEFAULT-ON-NEXT: [[ENTRY:.*:]]
+; DEFAULT-ON-NEXT: br label %[[FOR_COND5_PREHEADER_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND1_PREHEADER_PREHEADER:.*]]:
+; DEFAULT-ON-NEXT: br label %[[FOR_COND1_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND1_PREHEADER]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV61:%.*]] = phi i64 [ [[INDVARS_IV_NEXT62:%.*]], %[[FOR_COND_CLEANUP3:.*]] ], [ 0, %[[FOR_COND1_PREHEADER_PREHEADER]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_COND9_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND5_PREHEADER_PREHEADER]]:
+; DEFAULT-ON-NEXT: br label %[[FOR_COND5_PREHEADER:.*]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP3]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT62]] = add nuw nsw i64 [[INDVARS_IV61]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND64:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT62]], 2
+; DEFAULT-ON-NEXT: br i1 [[EXITCOND64]], label %[[FOR_COND1_PREHEADER]], label %[[FOR_COND_CLEANUP7:.*]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP7]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT58:%.*]] = add nuw nsw i64 [[INDVARS_IV57:%.*]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND60:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT58]], 2
+; DEFAULT-ON-NEXT: br i1 [[EXITCOND60]], label %[[FOR_COND5_PREHEADER]], label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP11:.*]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT54:%.*]] = add nuw nsw i64 [[INDVARS_IV53:%.*]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND56:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT54]], 2
+; DEFAULT-ON-NEXT: br i1 [[EXITCOND56]], label %[[FOR_COND9_PREHEADER]], label %[[FOR_COND_CLEANUP3]], !llvm.loop [[LOOP2:![0-9]+]]
+; DEFAULT-ON: [[FOR_BODY12:.*]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_COND9_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY12]] ]
+; DEFAULT-ON-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @b, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-ON-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX18]], align 4
+; DEFAULT-ON-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @a, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-ON-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX26]], align 4
+; DEFAULT-ON-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; DEFAULT-ON-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX26]], align 4
+; DEFAULT-ON-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DEFAULT-ON-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 2
+; DEFAULT-ON-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY12]], label %[[FOR_COND_CLEANUP11]]
+; DEFAULT-ON: [[FOR_COND9_PREHEADER]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV53]] = phi i64 [ 0, %[[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT54]], %[[FOR_COND_CLEANUP11]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_BODY12]]
+; DEFAULT-ON: [[FOR_COND5_PREHEADER]]:
+; DEFAULT-ON-NEXT: [[INDVARS_IV57]] = phi i64 [ [[INDVARS_IV_NEXT58]], %[[FOR_COND_CLEANUP7]] ], [ 0, %[[FOR_COND5_PREHEADER_PREHEADER]] ]
+; DEFAULT-ON-NEXT: br label %[[FOR_COND1_PREHEADER_PREHEADER]]
+; DEFAULT-ON: [[FOR_COND_CLEANUP]]:
+; DEFAULT-ON-NEXT: ret void
+;
+; DEFAULT-OFF-LABEL: define void @explicit_off() {
+; DEFAULT-OFF-NEXT: [[ENTRY:.*]]:
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND1_PREHEADER:.*]]
+; DEFAULT-OFF: [[FOR_COND1_PREHEADER]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV61:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT62:%.*]], %[[FOR_COND_CLEANUP3:.*]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND5_PREHEADER:.*]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP3]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT62]] = add nuw nsw i64 [[INDVARS_IV61]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND64:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT62]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND64]], label %[[FOR_COND1_PREHEADER]], label %[[FOR_COND_CLEANUP:.*]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP7:.*]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT58:%.*]] = add nuw nsw i64 [[INDVARS_IV57:%.*]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND60:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT58]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND60]], label %[[FOR_COND5_PREHEADER]], label %[[FOR_COND_CLEANUP3]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP11:.*]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT54:%.*]] = add nuw nsw i64 [[INDVARS_IV53:%.*]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND56:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT54]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND56]], label %[[FOR_COND9_PREHEADER:.*]], label %[[FOR_COND_CLEANUP7]], !llvm.loop [[LOOP2:![0-9]+]]
+; DEFAULT-OFF: [[FOR_BODY12:.*]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[FOR_COND9_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY12]] ]
+; DEFAULT-OFF-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @b, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-OFF-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX18]], align 4
+; DEFAULT-OFF-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @a, i64 0, i64 [[INDVARS_IV]], i64 [[INDVARS_IV53]], i64 [[INDVARS_IV57]], i64 [[INDVARS_IV61]]
+; DEFAULT-OFF-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX26]], align 4
+; DEFAULT-OFF-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; DEFAULT-OFF-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX26]], align 4
+; DEFAULT-OFF-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; DEFAULT-OFF-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 2
+; DEFAULT-OFF-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY12]], label %[[FOR_COND_CLEANUP11]]
+; DEFAULT-OFF: [[FOR_COND9_PREHEADER]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV53]] = phi i64 [ 0, %[[FOR_COND5_PREHEADER]] ], [ [[INDVARS_IV_NEXT54]], %[[FOR_COND_CLEANUP11]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_BODY12]]
+; DEFAULT-OFF: [[FOR_COND5_PREHEADER]]:
+; DEFAULT-OFF-NEXT: [[INDVARS_IV57]] = phi i64 [ 0, %[[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT58]], %[[FOR_COND_CLEANUP7]] ]
+; DEFAULT-OFF-NEXT: br label %[[FOR_COND9_PREHEADER]]
+; DEFAULT-OFF: [[FOR_COND_CLEANUP]]:
+; DEFAULT-OFF-NEXT: ret void
+;
+entry:
+ br label %for.cond1.preheader
+
+for.cond1.preheader:
+ %indvars.iv61 = phi i64 [ 0, %entry ], [ %indvars.iv.next62, %for.cond.cleanup3 ]
+ br label %for.cond5.preheader
+
+for.cond.cleanup3:
+ %indvars.iv.next62 = add nuw nsw i64 %indvars.iv61, 1
+ %exitcond64 = icmp ne i64 %indvars.iv.next62, 2
+ br i1 %exitcond64, label %for.cond1.preheader, label %for.cond.cleanup
+
+for.cond.cleanup7:
+ %indvars.iv.next58 = add nuw nsw i64 %indvars.iv57, 1
+ %exitcond60 = icmp ne i64 %indvars.iv.next58, 2
+ br i1 %exitcond60, label %for.cond5.preheader, label %for.cond.cleanup3
+
+for.cond.cleanup11:
+ %indvars.iv.next54 = add nuw nsw i64 %indvars.iv53, 1
+ %exitcond56 = icmp ne i64 %indvars.iv.next54, 2
+ br i1 %exitcond56, label %for.cond9.preheader, label %for.cond.cleanup7, !llvm.loop !2
+
+for.body12:
+ %indvars.iv = phi i64 [ 0, %for.cond9.preheader ], [ %indvars.iv.next, %for.body12 ]
+ %arrayidx18 = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @b, i64 0, i64 %indvars.iv, i64 %indvars.iv53, i64 %indvars.iv57, i64 %indvars.iv61
+ %0 = load i32, ptr %arrayidx18, align 4
+ %arrayidx26 = getelementptr inbounds nuw [2 x [2 x [2 x [2 x i32]]]], ptr @a, i64 0, i64 %indvars.iv, i64 %indvars.iv53, i64 %indvars.iv57, i64 %indvars.iv61
+ %1 = load i32, ptr %arrayidx26, align 4
+ %add = add nsw i32 %1, %0
+ store i32 %add, ptr %arrayidx26, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp ne i64 %indvars.iv.next, 2
+ br i1 %exitcond, label %for.body12, label %for.cond.cleanup11
+
+for.cond9.preheader:
+ %indvars.iv53 = phi i64 [ 0, %for.cond5.preheader ], [ %indvars.iv.next54, %for.cond.cleanup11 ]
+ br label %for.body12
+
+for.cond5.preheader:
+ %indvars.iv57 = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next58, %for.cond.cleanup7 ]
+ br label %for.cond9.preheader
+
+for.cond.cleanup:
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interchange.enable", i1 true}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.interchange.enable", i1 false}
+;.
+; DEFAULT-ON: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; DEFAULT-ON: [[META1]] = !{!"llvm.loop.interchange.enable", i1 true}
+; DEFAULT-ON: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+; DEFAULT-ON: [[META3]] = !{!"llvm.loop.interchange.enable", i1 false}
+;.
+; DEFAULT-OFF: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
+; DEFAULT-OFF: [[META1]] = !{!"llvm.loop.interchange.enable", i1 true}
+; DEFAULT-OFF: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+; DEFAULT-OFF: [[META3]] = !{!"llvm.loop.interchange.enable", i1 false}
+;.
More information about the llvm-commits
mailing list