[llvm-commits] [llvm] r165285 - in /llvm/trunk: lib/Transforms/Scalar/SROA.cpp test/Transforms/SROA/basictest.ll
Chandler Carruth
chandlerc at gmail.com
Thu Oct 4 18:29:09 PDT 2012
Author: chandlerc
Date: Thu Oct 4 20:29:09 2012
New Revision: 165285
URL: http://llvm.org/viewvc/llvm-project?rev=165285&view=rev
Log:
Teach the new SROA a new trick. Now we zap any memcpy or memmoves which
are in fact identity operations. We detect these and kill their
partitions so that even splitting is unaffected by them. This is
particularly important because Clang relies on emitting identity memcpy
operations for struct copies, and these fold away to constants very
often after inlining.
Fixes the last big performance FIXME I have on my plate.
Modified:
llvm/trunk/lib/Transforms/Scalar/SROA.cpp
llvm/trunk/test/Transforms/SROA/basictest.ll
Modified: llvm/trunk/lib/Transforms/Scalar/SROA.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Scalar/SROA.cpp?rev=165285&r1=165284&r2=165285&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Scalar/SROA.cpp (original)
+++ llvm/trunk/lib/Transforms/Scalar/SROA.cpp Thu Oct 4 20:29:09 2012
@@ -137,6 +137,23 @@
/// splittable and eagerly split them into scalar values.
bool IsSplittable;
+ /// \brief Test whether a partition has been marked as dead.
+ bool isDead() const {
+ if (BeginOffset == UINT64_MAX) {
+ assert(EndOffset == UINT64_MAX);
+ return true;
+ }
+ return false;
+ }
+
+ /// \brief Kill a partition.
+ /// This is accomplished by setting both its beginning and end offset to
+ /// the maximum possible value.
+ void kill() {
+ assert(!isDead() && "He's Dead, Jim!");
+ BeginOffset = EndOffset = UINT64_MAX;
+ }
+
Partition() : ByteRange(), IsSplittable() {}
Partition(uint64_t BeginOffset, uint64_t EndOffset, bool IsSplittable)
: ByteRange(BeginOffset, EndOffset), IsSplittable(IsSplittable) {}
@@ -255,8 +272,16 @@
/// correctly represent. We stash extra data to help us untangle this
/// after the partitioning is complete.
struct MemTransferOffsets {
+ /// The destination begin and end offsets when the destination is within
+ /// this alloca. If the end offset is zero the destination is not within
+ /// this alloca.
uint64_t DestBegin, DestEnd;
+
+ /// The source begin and end offsets when the source is within this alloca.
+ /// If the end offset is zero, the source is not within this alloca.
uint64_t SourceBegin, SourceEnd;
+
+ /// Flag for whether an alloca is splittable.
bool IsSplittable;
};
MemTransferOffsets getMemTransferOffsets(MemTransferInst &II) const {
@@ -555,14 +580,6 @@
EndOffset = AllocSize;
}
- // See if we can just add a user onto the last slot currently occupied.
- if (!P.Partitions.empty() &&
- P.Partitions.back().BeginOffset == BeginOffset &&
- P.Partitions.back().EndOffset == EndOffset) {
- P.Partitions.back().IsSplittable &= IsSplittable;
- return;
- }
-
Partition New(BeginOffset, EndOffset, IsSplittable);
P.Partitions.push_back(New);
}
@@ -643,33 +660,56 @@
// Only intrinsics with a constant length can be split.
Offsets.IsSplittable = Length;
- if (*U != II.getRawDest()) {
- assert(*U == II.getRawSource());
- Offsets.SourceBegin = Offset;
- Offsets.SourceEnd = Offset + Size;
- } else {
+ if (*U == II.getRawDest()) {
Offsets.DestBegin = Offset;
Offsets.DestEnd = Offset + Size;
}
+ if (*U == II.getRawSource()) {
+ Offsets.SourceBegin = Offset;
+ Offsets.SourceEnd = Offset + Size;
+ }
- insertUse(II, Offset, Size, Offsets.IsSplittable);
- unsigned NewIdx = P.Partitions.size() - 1;
+ // If we have set up end offsets for both the source and the destination,
+ // we have found both sides of this transfer pointing at the same alloca.
+ bool SeenBothEnds = Offsets.SourceEnd && Offsets.DestEnd;
+ if (SeenBothEnds && II.getRawDest() != II.getRawSource()) {
+ unsigned PrevIdx = MemTransferPartitionMap[&II];
+
+ // Check if the begin offsets match and this is a non-volatile transfer.
+ // In that case, we can completely elide the transfer.
+ if (!II.isVolatile() && Offsets.SourceBegin == Offsets.DestBegin) {
+ P.Partitions[PrevIdx].kill();
+ return true;
+ }
+
+ // Otherwise we have an offset transfer within the same alloca. We can't
+ // split those.
+ P.Partitions[PrevIdx].IsSplittable = Offsets.IsSplittable = false;
+ } else if (SeenBothEnds) {
+ // Handle the case where this exact use provides both ends of the
+ // operation.
+ assert(II.getRawDest() == II.getRawSource());
+
+ // For non-volatile transfers this is a no-op.
+ if (!II.isVolatile())
+ return true;
- SmallDenseMap<Instruction *, unsigned>::const_iterator PMI;
- bool Inserted = false;
- llvm::tie(PMI, Inserted)
- = MemTransferPartitionMap.insert(std::make_pair(&II, NewIdx));
- if (Offsets.IsSplittable &&
- (!Inserted || II.getRawSource() == II.getRawDest())) {
- // We've found a memory transfer intrinsic which refers to the alloca as
- // both a source and dest. This is detected either by direct equality of
- // the operand values, or when we visit the intrinsic twice due to two
- // different chains of values leading to it. We refuse to split these to
- // simplify splitting logic. If possible, SROA will still split them into
- // separate allocas and then re-analyze.
+ // Otherwise just suppress splitting.
Offsets.IsSplittable = false;
- P.Partitions[PMI->second].IsSplittable = false;
- P.Partitions[NewIdx].IsSplittable = false;
+ }
+
+
+ // Insert the use now that we've fixed up the splittable nature.
+ insertUse(II, Offset, Size, Offsets.IsSplittable);
+
+ // Setup the mapping from intrinsic to partition of we've not seen both
+ // ends of this transfer.
+ if (!SeenBothEnds) {
+ unsigned NewIdx = P.Partitions.size() - 1;
+ bool Inserted
+ = MemTransferPartitionMap.insert(std::make_pair(&II, NewIdx)).second;
+ assert(Inserted &&
+ "Already have intrinsic in map but haven't seen both ends");
}
return true;
@@ -913,6 +953,14 @@
void visitMemTransferInst(MemTransferInst &II) {
ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
uint64_t Size = Length ? Length->getZExtValue() : AllocSize - Offset;
+ if (!Size)
+ return markAsDead(II);
+
+ MemTransferOffsets &Offsets = P.MemTransferInstData[&II];
+ if (!II.isVolatile() && Offsets.DestEnd && Offsets.SourceEnd &&
+ Offsets.DestBegin == Offsets.SourceBegin)
+ return markAsDead(II); // Skip identity transfers without side-effects.
+
insertUse(II, Offset, Size);
}
@@ -1011,7 +1059,7 @@
SplitEndOffset = std::max(SplitEndOffset, Partitions[j].EndOffset);
}
- Partitions[j].BeginOffset = Partitions[j].EndOffset = UINT64_MAX;
+ Partitions[j].kill();
++NumDeadPartitions;
++j;
}
@@ -1032,7 +1080,7 @@
if (New.BeginOffset != New.EndOffset)
Partitions.push_back(New);
// Mark the old one for removal.
- Partitions[i].BeginOffset = Partitions[i].EndOffset = UINT64_MAX;
+ Partitions[i].kill();
++NumDeadPartitions;
}
@@ -1059,8 +1107,7 @@
// replaced in the process.
std::sort(Partitions.begin(), Partitions.end());
if (NumDeadPartitions) {
- assert(Partitions.back().BeginOffset == UINT64_MAX);
- assert(Partitions.back().EndOffset == UINT64_MAX);
+ assert(Partitions.back().isDead());
assert((ptrdiff_t)NumDeadPartitions ==
std::count(Partitions.begin(), Partitions.end(), Partitions.back()));
}
@@ -1077,11 +1124,15 @@
if (!PB())
return;
- if (Partitions.size() > 1) {
- // Sort the uses. This arranges for the offsets to be in ascending order,
- // and the sizes to be in descending order.
- std::sort(Partitions.begin(), Partitions.end());
+ // Sort the uses. This arranges for the offsets to be in ascending order,
+ // and the sizes to be in descending order.
+ std::sort(Partitions.begin(), Partitions.end());
+
+ // Remove any partitions from the back which are marked as dead.
+ while (!Partitions.empty() && Partitions.back().isDead())
+ Partitions.pop_back();
+ if (Partitions.size() > 1) {
// Intersect splittability for all partitions with equal offsets and sizes.
// Then remove all but the first so that we have a sequence of non-equal but
// potentially overlapping partitions.
@@ -3208,10 +3259,6 @@
if (P.isEscaped())
return Changed;
- // No partitions to split. Leave the dead alloca for a later pass to clean up.
- if (P.begin() == P.end())
- return Changed;
-
// Delete all the dead users of this alloca before splitting and rewriting it.
for (AllocaPartitioning::dead_user_iterator DI = P.dead_user_begin(),
DE = P.dead_user_end();
@@ -3233,6 +3280,10 @@
}
}
+ // No partitions to split. Leave the dead alloca for a later pass to clean up.
+ if (P.begin() == P.end())
+ return Changed;
+
return splitAlloca(AI, P) || Changed;
}
Modified: llvm/trunk/test/Transforms/SROA/basictest.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/SROA/basictest.ll?rev=165285&r1=165284&r2=165285&view=diff
==============================================================================
--- llvm/trunk/test/Transforms/SROA/basictest.ll (original)
+++ llvm/trunk/test/Transforms/SROA/basictest.ll Thu Oct 4 20:29:09 2012
@@ -862,8 +862,7 @@
; Ensure that we handle overlapping memcpy intrinsics correctly, especially in
; the case where there is a directly identical value for both source and dest.
; CHECK: @PR13916.1
-; FIXME: We shouldn't leave this alloca around.
-; CHECK: alloca
+; CHECK-NOT: alloca
; CHECK: ret void
entry:
@@ -878,8 +877,7 @@
; different pointer value chains, but during rewriting we coalesce them into the
; same value.
; CHECK: @PR13916.2
-; FIXME: We shouldn't leave this alloca around.
-; CHECK: alloca
+; CHECK-NOT: alloca
; CHECK: ret void
entry:
More information about the llvm-commits
mailing list