[polly] r316766 - [ForwardOpTree] Reload know values.
Michael Kruse via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 27 07:26:15 PDT 2017
Author: meinersbur
Date: Fri Oct 27 07:26:14 2017
New Revision: 316766
URL: http://llvm.org/viewvc/llvm-project?rev=316766&view=rev
Log:
[ForwardOpTree] Reload know values.
For scalar accesses, change the access target to an array element that
is known to contain the same value.
This may become an alternative to forwardKnownLoad which creates new
loads (and therefore closer to forwarding speculatives). Reloading does
not require the known value originating from a load, but can be a store
as well.
Differential Revision: https://reviews.llvm.org/D39325
Added:
polly/trunk/test/ForwardOpTree/forward_store.ll
Modified:
polly/trunk/lib/Transform/ForwardOpTree.cpp
Modified: polly/trunk/lib/Transform/ForwardOpTree.cpp
URL: http://llvm.org/viewvc/llvm-project/polly/trunk/lib/Transform/ForwardOpTree.cpp?rev=316766&r1=316765&r2=316766&view=diff
==============================================================================
--- polly/trunk/lib/Transform/ForwardOpTree.cpp (original)
+++ polly/trunk/lib/Transform/ForwardOpTree.cpp Fri Oct 27 07:26:14 2017
@@ -64,6 +64,7 @@ STATISTIC(KnownOutOfQuota,
STATISTIC(TotalInstructionsCopied, "Number of copied instructions");
STATISTIC(TotalKnownLoadsForwarded,
"Number of forwarded loads because their value was known");
+STATISTIC(TotalReloads, "Number of reloaded values");
STATISTIC(TotalReadOnlyCopied, "Number of copied read-only accesses");
STATISTIC(TotalForwardedTrees, "Number of forwarded operand trees");
STATISTIC(TotalModifiedStmts,
@@ -106,12 +107,21 @@ enum ForwardingDecision {
/// and can be used anywhere) into the same statement as %add would.
FD_CanForwardLeaf,
- /// The root instruction can be forwarded in a non-trivial way. This requires
- /// the operand tree root to be an instruction in some statement.
- FD_CanForwardTree,
-
- /// Used to indicate that a forwarding has be carried out successfully.
- FD_DidForward,
+ /// The root instruction can be forwarded and doing so avoids a scalar
+ /// dependency.
+ ///
+ /// This can be either because the operand tree can be moved to the target
+ /// statement, or a memory access is redirected to read from a different
+ /// location.
+ FD_CanForwardProfitably,
+
+ /// Used to indicate that a forwarding has be carried out successfully, and
+ /// the forwarded memory access can be deleted.
+ FD_DidForwardTree,
+
+ /// Used to indicate that a forwarding has be carried out successfully, and
+ /// the forwarded memory access is being reused.
+ FD_DidForwardLeaf,
/// A forwarding method cannot be applied to the operand tree.
/// The difference to FD_CannotForward is that there might be other methods
@@ -140,6 +150,9 @@ private:
/// Number of loads forwarded because their value was known.
int NumKnownLoadsForwarded = 0;
+ /// Number of values reloaded from known array elements.
+ int NumReloads = 0;
+
/// How many read-only accesses have been copied.
int NumReadOnlyCopied = 0;
@@ -293,6 +306,7 @@ public:
<< '\n';
OS.indent(Indent + 4) << "Known loads forwarded: " << NumKnownLoadsForwarded
<< '\n';
+ OS.indent(Indent + 4) << "Reloads: " << NumReloads<< '\n';
OS.indent(Indent + 4) << "Read-only accesses copied: " << NumReadOnlyCopied
<< '\n';
OS.indent(Indent + 4) << "Operand trees forwarded: " << NumForwardedTrees
@@ -391,12 +405,11 @@ public:
/// use DoIt==true if an operand tree is not known to be
/// forwardable.
///
- /// @return FD_NotApplicable if @p Inst is not a LoadInst.
- /// FD_CannotForward if no array element to load from was found.
- /// FD_CanForwardLeaf if the load is already in the target statement
- /// instance.
- /// FD_CanForwardTree if @p Inst is forwardable.
- /// FD_DidForward if @p DoIt was true.
+ /// @return FD_NotApplicable if @p Inst cannot be forwarded by creating a new
+ /// load.
+ /// FD_CannotForward if the pointer operand cannot be forwarded.
+ /// FD_CanForwardProfitably if @p Inst is forwardable.
+ /// FD_DidForwardTree if @p DoIt was true.
ForwardingDecision forwardKnownLoad(ScopStmt *TargetStmt, Instruction *Inst,
ScopStmt *UseStmt, Loop *UseLoop,
isl::map UseToTarget, ScopStmt *DefStmt,
@@ -421,10 +434,7 @@ public:
// do not add another MemoryAccess.
MemoryAccess *Access = TargetStmt->getArrayAccessOrNULLFor(LI);
if (Access && !DoIt)
- return FD_CanForwardTree;
-
- if (DoIt)
- TargetStmt->prependInstruction(LI);
+ return FD_CanForwardProfitably;
ForwardingDecision OpDecision =
forwardTree(TargetStmt, LI->getPointerOperand(), DefStmt, DefLoop,
@@ -435,11 +445,12 @@ public:
return OpDecision;
case FD_CanForwardLeaf:
- case FD_CanForwardTree:
+ case FD_CanForwardProfitably:
assert(!DoIt);
break;
- case FD_DidForward:
+ case FD_DidForwardLeaf:
+ case FD_DidForwardTree:
assert(DoIt);
break;
@@ -462,10 +473,13 @@ public:
isl::map SameVal = singleLocation(Candidates, getDomainFor(TargetStmt));
if (!SameVal)
- return FD_CannotForward;
+ return FD_NotApplicable;
+
+ if (DoIt)
+ TargetStmt->prependInstruction(LI);
if (!DoIt)
- return FD_CanForwardTree;
+ return FD_CanForwardProfitably;
if (Access) {
DEBUG(dbgs() << " forwarded known load with preexisting MemoryAccess"
@@ -519,7 +533,77 @@ public:
NumKnownLoadsForwarded++;
TotalKnownLoadsForwarded++;
- return FD_DidForward;
+ return FD_DidForwardTree;
+ }
+
+ /// Forward a scalar by redirecting the access to an array element that stores
+ /// the same value.
+ ///
+ /// @param TargetStmt The statement the operand tree will be copied to.
+ /// @param Inst The scalar to forward.
+ /// @param UseStmt The statement that uses @p Inst.
+ /// @param UseLoop The loop @p Inst is used in.
+ /// @param UseToTarget { DomainUse[] -> DomainTarget[] }
+ /// A mapping from the statement instance @p Inst is used
+ /// in, to the statement instance it is forwarded to.
+ /// @param DefStmt The statement @p Inst is defined in.
+ /// @param DefLoop The loop which contains @p Inst.
+ /// @param DefToTarget { DomainDef[] -> DomainTarget[] }
+ /// A mapping from the statement instance @p Inst is
+ /// defined in, to the statement instance it is forwarded
+ /// to.
+ /// @param DoIt If false, only determine whether an operand tree can be
+ /// forwarded. If true, carry out the forwarding. Do not
+ /// use DoIt==true if an operand tree is not known to be
+ /// forwardable.
+ ///
+ /// @return FD_NotApplicable if @p Inst cannot be reloaded.
+ /// FD_CanForwardLeaf if @p Inst can be reloaded.
+ /// FD_CanForwardProfitably if @p Inst has been reloaded.
+ /// FD_DidForwardLeaf if @p DoIt was true.
+ ForwardingDecision reloadKnownContent(ScopStmt *TargetStmt, Instruction *Inst,
+ ScopStmt *UseStmt, Loop *UseLoop,
+ isl::map UseToTarget, ScopStmt *DefStmt,
+ Loop *DefLoop, isl::map DefToTarget,
+ bool DoIt) {
+ // Cannot do anything without successful known analysis.
+ if (Known.is_null())
+ return FD_NotApplicable;
+
+ MemoryAccess *Access = TargetStmt->lookupInputAccessOf(Inst);
+ if (Access && Access->isLatestArrayKind()) {
+ if (DoIt)
+ return FD_DidForwardLeaf;
+ return FD_CanForwardLeaf;
+ }
+
+ // { DomainDef[] -> ValInst[] }
+ isl::union_map ExpectedVal = makeValInst(Inst, UseStmt, UseLoop);
+
+ // { DomainTarget[] -> ValInst[] }
+ isl::union_map TargetExpectedVal = ExpectedVal.apply_domain(UseToTarget);
+ isl::union_map TranslatedExpectedVal =
+ TargetExpectedVal.apply_range(Translator);
+
+ // { DomainTarget[] -> Element[] }
+ isl::union_map Candidates = findSameContentElements(TranslatedExpectedVal);
+
+ isl::map SameVal = singleLocation(Candidates, getDomainFor(TargetStmt));
+ if (!SameVal)
+ return FD_NotApplicable;
+
+ if (!DoIt)
+ return FD_CanForwardProfitably;
+
+ if (!Access)
+ Access = TargetStmt->ensureValueRead(Inst);
+
+ simplify(SameVal);
+ Access->setNewAccessRelation(SameVal);
+
+ TotalReloads++;
+ NumReloads++;
+ return FD_DidForwardLeaf;
}
/// Forwards a speculatively executable instruction.
@@ -584,11 +668,12 @@ public:
return FD_CannotForward;
case FD_CanForwardLeaf:
- case FD_CanForwardTree:
+ case FD_CanForwardProfitably:
assert(!DoIt);
break;
- case FD_DidForward:
+ case FD_DidForwardLeaf:
+ case FD_DidForwardTree:
assert(DoIt);
break;
@@ -598,8 +683,8 @@ public:
}
if (DoIt)
- return FD_DidForward;
- return FD_CanForwardTree;
+ return FD_DidForwardTree;
+ return FD_CanForwardProfitably;
}
/// Determines whether an operand tree can be forwarded or carries out a
@@ -636,14 +721,14 @@ public:
case VirtualUse::Hoisted:
// These can be used anywhere without special considerations.
if (DoIt)
- return FD_DidForward;
+ return FD_DidForwardTree;
return FD_CanForwardLeaf;
case VirtualUse::Synthesizable: {
// ScopExpander will take care for of generating the code at the new
// location.
if (DoIt)
- return FD_DidForward;
+ return FD_DidForwardTree;
// Check if the value is synthesizable at the new location as well. This
// might be possible when leaving a loop for which ScalarEvolution is
@@ -682,7 +767,7 @@ public:
NumReadOnlyCopied++;
TotalReadOnlyCopied++;
- return FD_DidForward;
+ return FD_DidForwardLeaf;
case VirtualUse::Intra:
// Knowing that UseStmt and DefStmt are the same statement instance, just
@@ -725,6 +810,12 @@ public:
if (KnownResult != FD_NotApplicable)
return KnownResult;
+ ForwardingDecision ReloadResult =
+ reloadKnownContent(TargetStmt, Inst, UseStmt, UseLoop, UseToTarget,
+ DefStmt, DefLoop, DefToTarget, DoIt);
+ if (ReloadResult != FD_NotApplicable)
+ return ReloadResult;
+
// When no method is found to forward the operand tree, we effectively
// cannot handle it.
DEBUG(dbgs() << " Cannot forward instruction: " << *Inst << "\n");
@@ -751,17 +842,18 @@ public:
ForwardingDecision Assessment = forwardTree(
Stmt, RA->getAccessValue(), Stmt, InLoop, TargetToUse, false);
- assert(Assessment != FD_DidForward);
- if (Assessment != FD_CanForwardTree)
+ assert(Assessment != FD_DidForwardTree && Assessment != FD_DidForwardLeaf);
+ if (Assessment != FD_CanForwardProfitably)
return false;
ForwardingDecision Execution = forwardTree(Stmt, RA->getAccessValue(), Stmt,
InLoop, TargetToUse, true);
- assert(Execution == FD_DidForward &&
+ assert(((Execution == FD_DidForwardTree) ||
+ (Execution == FD_DidForwardLeaf)) &&
"A previous positive assessment must also be executable");
- (void)Execution;
- Stmt->removeSingleMemoryAccess(RA);
+ if (Execution == FD_DidForwardTree)
+ Stmt->removeSingleMemoryAccess(RA);
return true;
}
Added: polly/trunk/test/ForwardOpTree/forward_store.ll
URL: http://llvm.org/viewvc/llvm-project/polly/trunk/test/ForwardOpTree/forward_store.ll?rev=316766&view=auto
==============================================================================
--- polly/trunk/test/ForwardOpTree/forward_store.ll (added)
+++ polly/trunk/test/ForwardOpTree/forward_store.ll Fri Oct 27 07:26:14 2017
@@ -0,0 +1,73 @@
+; RUN: opt %loadPolly -polly-optree -analyze < %s | FileCheck %s -match-full-lines
+;
+; Rematerialize a load.
+;
+; for (int j = 0; j < n; j += 1) {
+; bodyA:
+; double val = B[j];
+;
+; bodyB:
+; A[j] = val;
+; }
+;
+
+declare double @f() #0
+
+define void @func(i32 %n, double* noalias nonnull %A, double* noalias nonnull %B) {
+entry:
+ br label %for
+
+for:
+ %j = phi i32 [0, %entry], [%j.inc, %inc]
+ %j.cmp = icmp slt i32 %j, %n
+ br i1 %j.cmp, label %bodyA, label %exit
+
+ bodyA:
+ %val = call double @f()
+ %A_idx = getelementptr inbounds double, double* %A, i32 %j
+ store double %val, double* %A_idx
+ br label %bodyB
+
+ bodyB:
+ %B_idx = getelementptr inbounds double, double* %B, i32 %j
+ store double %val, double* %B_idx
+ br label %inc
+
+inc:
+ %j.inc = add nuw nsw i32 %j, 1
+ br label %for
+
+exit:
+ br label %return
+
+return:
+ ret void
+}
+
+attributes #0 = { nounwind readnone }
+
+
+; CHECK: Statistics {
+; CHECK: Reloads: 1
+; CHECK: }
+
+; CHECK: After statements {
+; CHECK-NEXT: Stmt_bodyA
+; CHECK-NEXT: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; CHECK-NEXT: [n] -> { Stmt_bodyA[i0] -> MemRef_A[i0] };
+; CHECK-NEXT: MustWriteAccess := [Reduction Type: NONE] [Scalar: 1]
+; CHECK-NEXT: [n] -> { Stmt_bodyA[i0] -> MemRef_val[] };
+; CHECK-NEXT: Instructions {
+; CHECK-NEXT: %val = call double @f()
+; CHECK-NEXT: store double %val, double* %A_idx
+; CHECK-NEXT: }
+; CHECK-NEXT: Stmt_bodyB
+; CHECK-NEXT: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; CHECK-NEXT: [n] -> { Stmt_bodyB[i0] -> MemRef_B[i0] };
+; CHECK-NEXT: ReadAccess := [Reduction Type: NONE] [Scalar: 1]
+; CHECK-NEXT: [n] -> { Stmt_bodyB[i0] -> MemRef_val[] };
+; CHECK-NEXT: new: [n] -> { Stmt_bodyB[i0] -> MemRef_A[i0] };
+; CHECK-NEXT: Instructions {
+; CHECK-NEXT: store double %val, double* %B_idx
+; CHECK-NEXT: }
+; CHECK-NEXT: }
More information about the llvm-commits
mailing list