[llvm] 8228153 - [Attributor][NFC] Encode IRPositions in the bits of a single pointer
Johannes Doerfert via llvm-commits
llvm-commits at lists.llvm.org
Sun May 3 10:27:05 PDT 2020
Author: Johannes Doerfert
Date: 2020-05-03T12:15:19-05:00
New Revision: 8228153f87707e9760a648608806cdba5e58141c
URL: https://github.com/llvm/llvm-project/commit/8228153f87707e9760a648608806cdba5e58141c
DIFF: https://github.com/llvm/llvm-project/commit/8228153f87707e9760a648608806cdba5e58141c.diff
LOG: [Attributor][NFC] Encode IRPositions in the bits of a single pointer
This reduces memory consumption for IRPositions by eliminating the
vtable pointer and the `KindOrArgNo` integer. Since each abstract
attribute has an associated IRPosition, the 12-16 bytes we save add up
quickly.
No functional change is intended.
---
Single run of the Attributor module and then CGSCC pass (oldPM)
for SPASS/clause.c (~10k LLVM-IR loc):
Before:
```
calls to allocation functions: 469545 (260135/s)
temporary memory allocations: 77137 (42735/s)
peak heap memory consumption: 30.50MB
peak RSS (including heaptrack overhead): 119.50MB
total memory leaked: 269.07KB
```
After:
```
calls to allocation functions: 468999 (274108/s)
temporary memory allocations: 77002 (45004/s)
peak heap memory consumption: 28.83MB
peak RSS (including heaptrack overhead): 118.05MB
total memory leaked: 269.07KB
```
Difference:
```
calls to allocation functions: -546 (5808/s)
temporary memory allocations: -135 (1436/s)
peak heap memory consumption: -1.67MB
peak RSS (including heaptrack overhead): 0B
total memory leaked: 0B
```
---
CTMark 15 runs
Metric: compile_time
Program lhs rhs diff
test-suite...:: CTMark/sqlite3/sqlite3.test 25.07 24.09 -3.9%
test-suite...Mark/mafft/pairlocalalign.test 14.58 14.14 -3.0%
test-suite...-typeset/consumer-typeset.test 21.78 21.58 -0.9%
test-suite :: CTMark/SPASS/SPASS.test 21.95 22.03 0.4%
test-suite :: CTMark/lencod/lencod.test 25.43 25.50 0.3%
test-suite...ark/tramp3d-v4/tramp3d-v4.test 23.88 23.83 -0.2%
test-suite...TMark/7zip/7zip-benchmark.test 60.24 60.11 -0.2%
test-suite :: CTMark/kimwitu++/kc.test 15.69 15.69 -0.0%
test-suite...:: CTMark/ClamAV/clamscan.test 25.43 25.42 -0.0%
test-suite :: CTMark/Bullet/bullet.test 37.63 37.62 -0.0%
Geomean difference -0.8%
---
Reviewed By: lebedev.ri
Differential Revision: https://reviews.llvm.org/D78722
Added:
Modified:
llvm/include/llvm/Transforms/IPO/Attributor.h
llvm/lib/Transforms/IPO/Attributor.cpp
llvm/lib/Transforms/IPO/AttributorAttributes.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index c4110bed2793..f1deb298745d 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -153,29 +153,24 @@ enum class DepClassTy {
/// are floating values that do not have a corresponding attribute list
/// position.
struct IRPosition {
- virtual ~IRPosition() {}
/// The positions we distinguish in the IR.
- ///
- /// The values are chosen such that the KindOrArgNo member has a value >= 0
- /// if it is an argument or call site argument while a value < 0 indicates the
- /// respective kind of that value.
- enum Kind : int {
- IRP_INVALID = -6, ///< An invalid position.
- IRP_FLOAT = -5, ///< A position that is not associated with a spot suitable
- ///< for attributes. This could be any value or instruction.
- IRP_RETURNED = -4, ///< An attribute for the function return value.
- IRP_CALL_SITE_RETURNED = -3, ///< An attribute for a call site return value.
- IRP_FUNCTION = -2, ///< An attribute for a function (scope).
- IRP_CALL_SITE = -1, ///< An attribute for a call site (function scope).
- IRP_ARGUMENT = 0, ///< An attribute for a function argument.
- IRP_CALL_SITE_ARGUMENT = 1, ///< An attribute for a call site argument.
+ enum Kind : char {
+ IRP_INVALID, ///< An invalid position.
+ IRP_FLOAT, ///< A position that is not associated with a spot suitable
+ ///< for attributes. This could be any value or instruction.
+ IRP_RETURNED, ///< An attribute for the function return value.
+ IRP_CALL_SITE_RETURNED, ///< An attribute for a call site return value.
+ IRP_FUNCTION, ///< An attribute for a function (scope).
+ IRP_CALL_SITE, ///< An attribute for a call site (function scope).
+ IRP_ARGUMENT, ///< An attribute for a function argument.
+ IRP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument.
};
/// Default constructor available to create invalid positions implicitly. All
/// other positions need to be created explicitly through the appropriate
/// static member function.
- IRPosition() : AnchorVal(nullptr), KindOrArgNo(IRP_INVALID) { verify(); }
+ IRPosition() : Enc(nullptr, ENC_VALUE) { verify(); }
/// Create a position describing the value of \p V.
static const IRPosition value(const Value &V) {
@@ -198,7 +193,7 @@ struct IRPosition {
/// Create a position describing the argument \p Arg.
static const IRPosition argument(const Argument &Arg) {
- return IRPosition(const_cast<Argument &>(Arg), Kind(Arg.getArgNo()));
+ return IRPosition(const_cast<Argument &>(Arg), IRP_ARGUMENT);
}
/// Create a position describing the function scope of \p CB.
@@ -214,7 +209,8 @@ struct IRPosition {
/// Create a position describing the argument of \p CB at position \p ArgNo.
static const IRPosition callsite_argument(const CallBase &CB,
unsigned ArgNo) {
- return IRPosition(const_cast<CallBase &>(CB), Kind(ArgNo));
+ return IRPosition(const_cast<Use &>(CB.getArgOperandUse(ArgNo)),
+ IRP_CALL_SITE_ARGUMENT);
}
/// Create a position describing the argument of \p ACS at position \p ArgNo.
@@ -242,9 +238,7 @@ struct IRPosition {
return IRPosition::function(*IRP.getAssociatedFunction());
}
- bool operator==(const IRPosition &RHS) const {
- return (AnchorVal == RHS.AnchorVal) && (KindOrArgNo == RHS.KindOrArgNo);
- }
+ bool operator==(const IRPosition &RHS) const { return Enc == RHS.Enc; }
bool operator!=(const IRPosition &RHS) const { return !(*this == RHS); }
/// Return the value this abstract attribute is anchored with.
@@ -254,16 +248,21 @@ struct IRPosition {
/// far, only the case for call site arguments as the value is not sufficient
/// to pinpoint them. Instead, we can use the call site as an anchor.
Value &getAnchorValue() const {
- assert(KindOrArgNo != IRP_INVALID &&
- "Invalid position does not have an anchor value!");
- return *AnchorVal;
+ switch (getEncodingBits()) {
+ case ENC_VALUE:
+ case ENC_RETURNED_VALUE:
+ case ENC_FLOATING_FUNCTION:
+ return *getAsValuePtr();
+ case ENC_CALL_SITE_ARGUMENT_USE:
+ return *(getAsUsePtr()->getUser());
+ default:
+ llvm_unreachable("Unkown encoding!");
+ };
}
/// Return the associated function, if any.
Function *getAssociatedFunction() const {
- assert(KindOrArgNo != IRP_INVALID &&
- "Invalid position does not have an anchor scope!");
- if (auto *CB = dyn_cast<CallBase>(AnchorVal))
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
return CB->getCalledFunction();
return getAnchorScope();
}
@@ -312,18 +311,14 @@ struct IRPosition {
/// Return the value this abstract attribute is associated with.
Value &getAssociatedValue() const {
- assert(KindOrArgNo != IRP_INVALID &&
- "Invalid position does not have an associated value!");
- if (getArgNo() < 0 || isa<Argument>(AnchorVal))
- return *AnchorVal;
- assert(isa<CallBase>(AnchorVal) && "Expected a call base!");
- return *cast<CallBase>(AnchorVal)->getArgOperand(getArgNo());
+ if (getArgNo() < 0 || isa<Argument>(&getAnchorValue()))
+ return getAnchorValue();
+ assert(isa<CallBase>(&getAnchorValue()) && "Expected a call base!");
+ return *cast<CallBase>(&getAnchorValue())->getArgOperand(getArgNo());
}
/// Return the type this abstract attribute is associated with.
Type *getAssociatedType() const {
- assert(KindOrArgNo != IRP_INVALID &&
- "Invalid position does not have an associated type!");
if (getPositionKind() == IRPosition::IRP_RETURNED)
return getAssociatedFunction()->getReturnType();
return getAssociatedValue().getType();
@@ -331,7 +326,18 @@ struct IRPosition {
/// Return the argument number of the associated value if it is an argument or
/// call site argument, otherwise a negative value.
- int getArgNo() const { return KindOrArgNo; }
+ int getArgNo() const {
+ switch (getPositionKind()) {
+ case IRPosition::IRP_ARGUMENT:
+ return cast<Argument>(getAsValuePtr())->getArgNo();
+ case IRPosition::IRP_CALL_SITE_ARGUMENT: {
+ Use &U = *getAsUsePtr();
+ return cast<CallBase>(U.getUser())->getArgOperandNo(&U);
+ }
+ default:
+ return -1;
+ }
+ }
/// Return the index in the attribute list for this position.
unsigned getAttrIdx() const {
@@ -347,7 +353,7 @@ struct IRPosition {
return AttributeList::ReturnIndex;
case IRPosition::IRP_ARGUMENT:
case IRPosition::IRP_CALL_SITE_ARGUMENT:
- return KindOrArgNo + AttributeList::FirstArgIndex;
+ return getArgNo() + AttributeList::FirstArgIndex;
}
llvm_unreachable(
"There is no attribute index for a floating or invalid position!");
@@ -355,19 +361,23 @@ struct IRPosition {
/// Return the associated position kind.
Kind getPositionKind() const {
- if (getArgNo() >= 0) {
- assert(((isa<Argument>(getAnchorValue()) &&
- isa<Argument>(getAssociatedValue())) ||
- isa<CallBase>(getAnchorValue())) &&
- "Expected argument or call base due to argument number!");
- if (isa<CallBase>(getAnchorValue()))
- return IRP_CALL_SITE_ARGUMENT;
+ char EncodingBits = getEncodingBits();
+ if (EncodingBits == ENC_CALL_SITE_ARGUMENT_USE)
+ return IRP_CALL_SITE_ARGUMENT;
+ if (EncodingBits == ENC_FLOATING_FUNCTION)
+ return IRP_FLOAT;
+
+ Value *V = getAsValuePtr();
+ if (!V)
+ return IRP_INVALID;
+ if (isa<Argument>(V))
return IRP_ARGUMENT;
- }
-
- assert(KindOrArgNo < 0 &&
- "Expected (call site) arguments to never reach this point!");
- return Kind(KindOrArgNo);
+ if (isa<Function>(V))
+ return isReturnPosition(EncodingBits) ? IRP_RETURNED : IRP_FUNCTION;
+ if (isa<CallBase>(V))
+ return isReturnPosition(EncodingBits) ? IRP_CALL_SITE_RETURNED
+ : IRP_CALL_SITE;
+ return IRP_FLOAT;
}
/// TODO: Figure out if the attribute related helper functions should live
@@ -435,14 +445,52 @@ struct IRPosition {
static const IRPosition TombstoneKey;
///}
+ /// Conversion into a void * to allow reuse of pointer hashing.
+ operator void *() const { return Enc.getOpaqueValue(); }
+
private:
/// Private constructor for special values only!
- explicit IRPosition(int KindOrArgNo)
- : AnchorVal(0), KindOrArgNo(KindOrArgNo) {}
+ explicit IRPosition(void *Ptr) { Enc.setFromOpaqueValue(Ptr); }
/// IRPosition anchored at \p AnchorVal with kind/argument numbet \p PK.
- explicit IRPosition(Value &AnchorVal, Kind PK)
- : AnchorVal(&AnchorVal), KindOrArgNo(PK) {
+ explicit IRPosition(Value &AnchorVal, Kind PK) {
+ switch (PK) {
+ case IRPosition::IRP_INVALID:
+ llvm_unreachable("Cannot create invalid IRP with an anchor value!");
+ break;
+ case IRPosition::IRP_FLOAT:
+ // Special case for floating functions.
+ if (isa<Function>(AnchorVal))
+ Enc = {&AnchorVal, ENC_FLOATING_FUNCTION};
+ else
+ Enc = {&AnchorVal, ENC_VALUE};
+ break;
+ case IRPosition::IRP_FUNCTION:
+ case IRPosition::IRP_CALL_SITE:
+ Enc = {&AnchorVal, ENC_VALUE};
+ break;
+ case IRPosition::IRP_RETURNED:
+ case IRPosition::IRP_CALL_SITE_RETURNED:
+ Enc = {&AnchorVal, ENC_RETURNED_VALUE};
+ break;
+ case IRPosition::IRP_ARGUMENT:
+ Enc = {&AnchorVal, ENC_VALUE};
+ break;
+ case IRPosition::IRP_CALL_SITE_ARGUMENT:
+ llvm_unreachable(
+ "Cannot create call site argument IRP with an anchor value!");
+ break;
+ }
+ verify();
+ }
+
+ /// IRPosition for the use \p U. The position kind \p PK needs to be
+ /// IRP_CALL_SITE_ARGUMENT, the anchor value is the user, the associated value
+ /// the used value.
+ explicit IRPosition(Use &U, Kind PK) {
+ assert(PK == IRP_CALL_SITE_ARGUMENT &&
+ "Use constructor is for call site arguments only!");
+ Enc = {&U, ENC_CALL_SITE_ARGUMENT_USE};
verify();
}
@@ -459,30 +507,65 @@ struct IRPosition {
SmallVectorImpl<Attribute> &Attrs,
Attributor &A) const;
-protected:
- /// The value this position is anchored at.
- Value *AnchorVal;
-
- /// If AnchorVal is Argument or CallBase then this number should be
- /// non-negative and it denotes the argument or call site argument index
- /// respectively. Otherwise, it denotes the kind of this IRPosition according
- /// to Kind above.
- int KindOrArgNo;
+ /// Return the underlying pointer as Value *, valid for all positions but
+ /// IRP_CALL_SITE_ARGUMENT.
+ Value *getAsValuePtr() const {
+ assert(getEncodingBits() != ENC_CALL_SITE_ARGUMENT_USE &&
+ "Not a value pointer!");
+ return reinterpret_cast<Value *>(Enc.getPointer());
+ }
+
+ /// Return the underlying pointer as Use *, valid only for
+ /// IRP_CALL_SITE_ARGUMENT positions.
+ Use *getAsUsePtr() const {
+ assert(getEncodingBits() == ENC_CALL_SITE_ARGUMENT_USE &&
+ "Not a value pointer!");
+ return reinterpret_cast<Use *>(Enc.getPointer());
+ }
+
+ /// Return true if \p EncodingBits describe a returned or call site returned
+ /// position.
+ static bool isReturnPosition(char EncodingBits) {
+ return EncodingBits == ENC_RETURNED_VALUE;
+ }
+
+ /// Return true if the encoding bits describe a returned or call site returned
+ /// position.
+ bool isReturnPosition() const { return isReturnPosition(getEncodingBits()); }
+
+ /// The encoding of the IRPosition is a combination of a pointer and two
+ /// encoding bits. The values of the encoding bits are defined in the enum
+ /// below. The pointer is either a Value* (for the first three encoding bit
+ /// combinations) or Use* (for ENC_CALL_SITE_ARGUMENT_USE).
+ ///
+ ///{
+ enum {
+ ENC_VALUE = 0b00,
+ ENC_RETURNED_VALUE = 0b01,
+ ENC_FLOATING_FUNCTION = 0b10,
+ ENC_CALL_SITE_ARGUMENT_USE = 0b11,
+ };
+
+ // Reserve the maximal amount of bits so there is no need to mask out the
+ // remaining ones. We will not encode anything else in the pointer anyway.
+ static constexpr int NumEncodingBits =
+ PointerLikeTypeTraits<void *>::NumLowBitsAvailable;
+ static_assert(NumEncodingBits >= 2, "At least two bits are required!");
+
+ /// The pointer with the encoding bits.
+ PointerIntPair<void *, NumEncodingBits, char> Enc;
+ ///}
+
+ /// Return the encoding bits.
+ char getEncodingBits() const { return Enc.getInt(); }
};
/// Helper that allows IRPosition as a key in a DenseMap.
-template <> struct DenseMapInfo<IRPosition> {
+template <> struct DenseMapInfo<IRPosition> : DenseMapInfo<void *> {
static inline IRPosition getEmptyKey() { return IRPosition::EmptyKey; }
static inline IRPosition getTombstoneKey() {
return IRPosition::TombstoneKey;
}
- static unsigned getHashValue(const IRPosition &IRP) {
- return (DenseMapInfo<Value *>::getHashValue(&IRP.getAnchorValue()) << 4) ^
- (unsigned(IRP.getArgNo()));
- }
- static bool isEqual(const IRPosition &LHS, const IRPosition &RHS) {
- return LHS == RHS;
- }
};
/// A visitor class for IR positions.
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 04fc4108d8d6..bf2cbed0b492 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -258,8 +258,9 @@ IRAttributeManifest::manifestAttrs(Attributor &A, const IRPosition &IRP,
return HasChanged;
}
-const IRPosition IRPosition::EmptyKey(255);
-const IRPosition IRPosition::TombstoneKey(256);
+const IRPosition IRPosition::EmptyKey(DenseMapInfo<void *>::getEmptyKey());
+const IRPosition
+ IRPosition::TombstoneKey(DenseMapInfo<void *>::getTombstoneKey());
SubsumingPositionIterator::SubsumingPositionIterator(const IRPosition &IRP) {
IRPositions.emplace_back(IRP);
@@ -401,52 +402,60 @@ bool IRPosition::getAttrsFromAssumes(Attribute::AttrKind AK,
void IRPosition::verify() {
#ifdef EXPENSIVE_CHECKS
- switch (KindOrArgNo) {
- default:
- assert(KindOrArgNo >= 0 && "Expected argument or call site argument!");
- assert((isa<CallBase>(AnchorVal) || isa<Argument>(AnchorVal)) &&
- "Expected call base or argument for positive attribute index!");
- if (isa<Argument>(AnchorVal)) {
- assert(cast<Argument>(AnchorVal)->getArgNo() == unsigned(getArgNo()) &&
- "Argument number mismatch!");
- assert(cast<Argument>(AnchorVal) == &getAssociatedValue() &&
- "Associated value mismatch!");
- } else {
- assert(cast<CallBase>(*AnchorVal).arg_size() > unsigned(getArgNo()) &&
- "Call site argument number mismatch!");
- assert(cast<CallBase>(*AnchorVal).getArgOperand(getArgNo()) ==
- &getAssociatedValue() &&
- "Associated value mismatch!");
- }
- break;
+ switch (getPositionKind()) {
case IRP_INVALID:
- assert(!AnchorVal && "Expected no value for an invalid position!");
- break;
+ assert(!Enc.getOpaqueValue() &&
+ "Expected a nullptr for an invalid position!");
+ return;
case IRP_FLOAT:
assert((!isa<CallBase>(&getAssociatedValue()) &&
!isa<Argument>(&getAssociatedValue())) &&
"Expected specialized kind for call base and argument values!");
- break;
+ return;
case IRP_RETURNED:
- assert(isa<Function>(AnchorVal) &&
+ assert(isa<Function>(getAsValuePtr()) &&
"Expected function for a 'returned' position!");
- assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
- break;
+ assert(getAsValuePtr() == &getAssociatedValue() &&
+ "Associated value mismatch!");
+ return;
case IRP_CALL_SITE_RETURNED:
- assert((isa<CallBase>(AnchorVal)) &&
+ assert((isa<CallBase>(getAsValuePtr())) &&
"Expected call base for 'call site returned' position!");
- assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
- break;
+ assert(getAsValuePtr() == &getAssociatedValue() &&
+ "Associated value mismatch!");
+ return;
case IRP_CALL_SITE:
- assert((isa<CallBase>(AnchorVal)) &&
+ assert((isa<CallBase>(getAsValuePtr())) &&
"Expected call base for 'call site function' position!");
- assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
- break;
+ assert(getAsValuePtr() == &getAssociatedValue() &&
+ "Associated value mismatch!");
+ return;
case IRP_FUNCTION:
- assert(isa<Function>(AnchorVal) &&
+ assert(isa<Function>(getAsValuePtr()) &&
"Expected function for a 'function' position!");
- assert(AnchorVal == &getAssociatedValue() && "Associated value mismatch!");
- break;
+ assert(getAsValuePtr() == &getAssociatedValue() &&
+ "Associated value mismatch!");
+ return;
+ case IRP_ARGUMENT:
+ assert(isa<Argument>(getAsValuePtr()) &&
+ "Expected argument for a 'argument' position!");
+ assert(getAsValuePtr() == &getAssociatedValue() &&
+ "Associated value mismatch!");
+ return;
+ case IRP_CALL_SITE_ARGUMENT: {
+ Use *U = getAsUsePtr();
+ assert(U && "Expected use for a 'call site argument' position!");
+ assert(isa<CallBase>(U->getUser()) &&
+ "Expected call base user for a 'call site argument' position!");
+ assert(cast<CallBase>(U->getUser())->isArgOperand(U) &&
+ "Expected call base argument operand for a 'call site argument' "
+ "position");
+ assert(cast<CallBase>(U->getUser())->getArgOperandNo(U) ==
+ unsigned(getArgNo()) &&
+ "Argument number mismatch!");
+ assert(U->get() == &getAssociatedValue() && "Associated value mismatch!");
+ return;
+ }
}
#endif
}
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index c354e5be13c7..6f14ad13027b 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -908,9 +908,7 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) {
if (auto *UniqueRVArg = dyn_cast<Argument>(UniqueRV.getValue())) {
if (UniqueRVArg->getType()->canLosslesslyBitCastTo(
getAssociatedFunction()->getReturnType())) {
- // TODO: This should be handled
diff erently!
- this->AnchorVal = UniqueRVArg;
- this->KindOrArgNo = UniqueRVArg->getArgNo();
+ getIRPosition() = IRPosition::argument(*UniqueRVArg);
Changed = IRAttribute::manifest(A);
}
} else if (auto *RVC = dyn_cast<Constant>(UniqueRV.getValue())) {
More information about the llvm-commits
mailing list