[llvm] r328516 - [Pipeliner] Fix check for order dependences when finalizing instructions
Krzysztof Parzyszek via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 26 09:05:55 PDT 2018
Author: kparzysz
Date: Mon Mar 26 09:05:55 2018
New Revision: 328516
URL: http://llvm.org/viewvc/llvm-project?rev=328516&view=rev
Log:
[Pipeliner] Fix check for order dependences when finalizing instructions
The code in orderDepdences that looks at the order dependences between
instructions was processing all the successor and predecessor order
dependences. However, we really only want to check for an order dependence
for instructions scheduled in the same cycle.
Also, fixed how the pipeliner handles output dependences. An output
dependence is also a potential loop carried dependence. The pipeliner
didn't handle this case properly so an invalid schedule could be created
that allowed an output dependence to be scheduled in the next iteration
at the same cycle.
Patch by Brendon Cahoon.
Added:
llvm/trunk/test/CodeGen/Hexagon/swp-order-deps7.ll
Modified:
llvm/trunk/lib/CodeGen/MachinePipeliner.cpp
Modified: llvm/trunk/lib/CodeGen/MachinePipeliner.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/CodeGen/MachinePipeliner.cpp?rev=328516&r1=328515&r2=328516&view=diff
==============================================================================
--- llvm/trunk/lib/CodeGen/MachinePipeliner.cpp (original)
+++ llvm/trunk/lib/CodeGen/MachinePipeliner.cpp Mon Mar 26 09:05:55 2018
@@ -348,15 +348,7 @@ public:
return Source->getInstr()->isPHI() || Dep.getSUnit()->getInstr()->isPHI();
}
- /// Return true if the dependence is an order dependence between non-Phis.
- static bool isOrder(SUnit *Source, const SDep &Dep) {
- if (Dep.getKind() != SDep::Order)
- return false;
- return (!Source->getInstr()->isPHI() &&
- !Dep.getSUnit()->getInstr()->isPHI());
- }
-
- bool isLoopCarriedOrder(SUnit *Source, const SDep &Dep, bool isSucc = true);
+ bool isLoopCarriedDep(SUnit *Source, const SDep &Dep, bool isSucc = true);
/// The distance function, which indicates that operation V of iteration I
/// depends on operations U of iteration I-distance.
@@ -1486,10 +1478,23 @@ static void swapAntiDependences(std::vec
void SwingSchedulerDAG::Circuits::createAdjacencyStructure(
SwingSchedulerDAG *DAG) {
BitVector Added(SUnits.size());
+ DenseMap<int, int> OutputDeps;
for (int i = 0, e = SUnits.size(); i != e; ++i) {
Added.reset();
// Add any successor to the adjacency matrix and exclude duplicates.
for (auto &SI : SUnits[i].Succs) {
+ // Only create a back-edge on the first and last nodes of a dependence
+ // chain. This records any chains and adds them later.
+ if (SI.getKind() == SDep::Output) {
+ int N = SI.getSUnit()->NodeNum;
+ int BackEdge = i;
+ auto Dep = OutputDeps.find(BackEdge);
+ if (Dep != OutputDeps.end()) {
+ BackEdge = Dep->second;
+ OutputDeps.erase(Dep);
+ }
+ OutputDeps[N] = BackEdge;
+ }
// Do not process a boundary node and a back-edge is processed only
// if it goes to a Phi.
if (SI.getSUnit()->isBoundaryNode() ||
@@ -1505,7 +1510,7 @@ void SwingSchedulerDAG::Circuits::create
// adjacency matrix.
for (auto &PI : SUnits[i].Preds) {
if (!SUnits[i].getInstr()->mayStore() ||
- !DAG->isLoopCarriedOrder(&SUnits[i], PI, false))
+ !DAG->isLoopCarriedDep(&SUnits[i], PI, false))
continue;
if (PI.getKind() == SDep::Order && PI.getSUnit()->getInstr()->mayLoad()) {
int N = PI.getSUnit()->NodeNum;
@@ -1516,6 +1521,12 @@ void SwingSchedulerDAG::Circuits::create
}
}
}
+ // Add back-eges in the adjacency matrix for the output dependences.
+ for (auto &OD : OutputDeps)
+ if (!Added.test(OD.second)) {
+ AdjK[OD.first].push_back(OD.second);
+ Added.set(OD.second);
+ }
}
/// Identify an elementary circuit in the dependence graph starting at the
@@ -3493,17 +3504,21 @@ void SwingSchedulerDAG::applyInstrChange
}
}
-/// Return true for an order dependence that is loop carried potentially.
-/// An order dependence is loop carried if the destination defines a value
-/// that may be used by the source in a subsequent iteration.
-bool SwingSchedulerDAG::isLoopCarriedOrder(SUnit *Source, const SDep &Dep,
- bool isSucc) {
- if (!isOrder(Source, Dep) || Dep.isArtificial())
+/// Return true for an order or output dependence that is loop carried
+/// potentially. A dependence is loop carried if the destination defines a valu
+/// that may be used or defined by the source in a subsequent iteration.
+bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep,
+ bool isSucc) {
+ if ((Dep.getKind() != SDep::Order && Dep.getKind() != SDep::Output) ||
+ Dep.isArtificial())
return false;
if (!SwpPruneLoopCarried)
return true;
+ if (Dep.getKind() == SDep::Output)
+ return true;
+
MachineInstr *SI = Source->getInstr();
MachineInstr *DI = Dep.getSUnit()->getInstr();
if (!isSucc)
@@ -3621,7 +3636,7 @@ int SMSchedule::earliestCycleInChain(con
continue;
EarlyCycle = std::min(EarlyCycle, it->second);
for (const auto &PI : PrevSU->Preds)
- if (SwingSchedulerDAG::isOrder(PrevSU, PI))
+ if (PI.getKind() == SDep::Order || Dep.getKind() == SDep::Output)
Worklist.push_back(PI);
Visited.insert(PrevSU);
}
@@ -3644,7 +3659,7 @@ int SMSchedule::latestCycleInChain(const
continue;
LateCycle = std::max(LateCycle, it->second);
for (const auto &SI : SuccSU->Succs)
- if (SwingSchedulerDAG::isOrder(SuccSU, SI))
+ if (SI.getKind() == SDep::Order || Dep.getKind() == SDep::Output)
Worklist.push_back(SI);
Visited.insert(SuccSU);
}
@@ -3684,7 +3699,7 @@ void SMSchedule::computeStart(SUnit *SU,
int EarlyStart = cycle + Dep.getLatency() -
DAG->getDistance(Dep.getSUnit(), SU, Dep) * II;
*MaxEarlyStart = std::max(*MaxEarlyStart, EarlyStart);
- if (DAG->isLoopCarriedOrder(SU, Dep, false)) {
+ if (DAG->isLoopCarriedDep(SU, Dep, false)) {
int End = earliestCycleInChain(Dep) + (II - 1);
*MinEnd = std::min(*MinEnd, End);
}
@@ -3708,7 +3723,7 @@ void SMSchedule::computeStart(SUnit *SU,
int LateStart = cycle - Dep.getLatency() +
DAG->getDistance(SU, Dep.getSUnit(), Dep) * II;
*MinLateStart = std::min(*MinLateStart, LateStart);
- if (DAG->isLoopCarriedOrder(SU, Dep)) {
+ if (DAG->isLoopCarriedDep(SU, Dep)) {
int Start = latestCycleInChain(Dep) + 1 - II;
*MaxStart = std::max(*MaxStart, Start);
}
@@ -3788,40 +3803,23 @@ bool SMSchedule::orderDependence(SwingSc
}
// Check for order dependences between instructions. Make sure the source
// is ordered before the destination.
- for (auto &S : SU->Succs)
- if (S.getKind() == SDep::Order) {
- if (S.getSUnit() == *I && stageScheduled(*I) == StageInst1) {
- OrderBeforeUse = true;
- MoveUse = Pos;
- }
- } else if (TargetRegisterInfo::isPhysicalRegister(S.getReg())) {
- if (cycleScheduled(SU) != cycleScheduled(S.getSUnit())) {
- if (S.isAssignedRegDep()) {
- OrderAfterDef = true;
- MoveDef = Pos;
- }
- } else {
- OrderBeforeUse = true;
+ for (auto &S : SU->Succs) {
+ if (S.getSUnit() != *I)
+ continue;
+ if (S.getKind() == SDep::Order && stageScheduled(*I) == StageInst1) {
+ OrderBeforeUse = true;
+ if (Pos < MoveUse)
MoveUse = Pos;
- }
}
- for (auto &P : SU->Preds)
- if (P.getKind() == SDep::Order) {
- if (P.getSUnit() == *I && stageScheduled(*I) == StageInst1) {
- OrderAfterDef = true;
- MoveDef = Pos;
- }
- } else if (TargetRegisterInfo::isPhysicalRegister(P.getReg())) {
- if (cycleScheduled(SU) != cycleScheduled(P.getSUnit())) {
- if (P.isAssignedRegDep()) {
- OrderBeforeUse = true;
- MoveUse = Pos;
- }
- } else {
- OrderAfterDef = true;
- MoveDef = Pos;
- }
+ }
+ for (auto &P : SU->Preds) {
+ if (P.getSUnit() != *I)
+ continue;
+ if (P.getKind() == SDep::Order && stageScheduled(*I) == StageInst1) {
+ OrderAfterDef = true;
+ MoveDef = Pos;
}
+ }
}
// A circular dependence.
Added: llvm/trunk/test/CodeGen/Hexagon/swp-order-deps7.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/Hexagon/swp-order-deps7.ll?rev=328516&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/Hexagon/swp-order-deps7.ll (added)
+++ llvm/trunk/test/CodeGen/Hexagon/swp-order-deps7.ll Mon Mar 26 09:05:55 2018
@@ -0,0 +1,51 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Test that the pipeliner cause an assert and correctly pipelines the
+; loop.
+
+; CHECK: loop0(.LBB0_[[LOOP:.]],
+; CHECK: .LBB0_[[LOOP]]:
+; CHECK: [[REG0:r([0-9]+)]] = sath([[REG1:r([0-9]+)]])
+; CHECK: memh(r{{[0-9]+}}++#2) = [[REG0]].new
+; CHECK: [[REG1]] =
+; CHECK: endloop0
+
+define void @f0(i16* nocapture %a0, float* nocapture readonly %a1, float %a2, i32 %a3) {
+b0:
+ %v0 = icmp sgt i32 %a3, 0
+ br i1 %v0, label %b1, label %b2
+
+b1: ; preds = %b1, %b0
+ %v1 = phi i32 [ %v11, %b1 ], [ 0, %b0 ]
+ %v2 = phi i16* [ %v10, %b1 ], [ %a0, %b0 ]
+ %v3 = phi float* [ %v4, %b1 ], [ %a1, %b0 ]
+ %v4 = getelementptr inbounds float, float* %v3, i32 1
+ %v5 = load float, float* %v3, align 4, !tbaa !0
+ %v6 = fmul float %v5, %a2
+ %v7 = tail call i32 @llvm.hexagon.F2.conv.sf2w(float %v6)
+ %v8 = tail call i32 @llvm.hexagon.A2.sath(i32 %v7)
+ %v9 = trunc i32 %v8 to i16
+ %v10 = getelementptr inbounds i16, i16* %v2, i32 1
+ store i16 %v9, i16* %v2, align 2, !tbaa !4
+ %v11 = add nuw nsw i32 %v1, 1
+ %v12 = icmp eq i32 %v11, %a3
+ br i1 %v12, label %b2, label %b1
+
+b2: ; preds = %b1, %b0
+ ret void
+}
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.A2.sath(i32) #0
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.hexagon.F2.conv.sf2w(float) #0
+
+attributes #0 = { nounwind readnone }
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"float", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"short", !2, i64 0}
More information about the llvm-commits
mailing list