[llvm] DynamicAPInt: optimize size of structure (PR #97831)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 5 12:50:16 PDT 2024
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/97831
>From 59f29d8aaea51f42d7c41c387fb099f224afb0cd Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Wed, 3 Jul 2024 11:39:57 +0100
Subject: [PATCH 1/3] DynamicAPInt: optimize size of structure
Reuse the APInt::BitWidth to eliminate DynamicAPInt::HoldsLarge, cutting
the size of DynamicAPInt by four bytes. This is implemented by making
DynamicAPInt a friend of SlowDynamicAPInt and APInt, so it can directly
access SlowDynamicAPInt::Val and APInt::BitWidth.
---
llvm/include/llvm/ADT/APInt.h | 4 ++++
llvm/include/llvm/ADT/DynamicAPInt.h | 23 ++++++++++++-----------
llvm/include/llvm/ADT/SlowDynamicAPInt.h | 7 +++++++
3 files changed, 23 insertions(+), 11 deletions(-)
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 6cfa6ec665084..108df7e0eaeaa 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -30,6 +30,7 @@ class StringRef;
class hash_code;
class raw_ostream;
struct Align;
+class DynamicAPInt;
template <typename T> class SmallVectorImpl;
template <typename T> class ArrayRef;
@@ -1895,6 +1896,9 @@ class [[nodiscard]] APInt {
friend struct DenseMapInfo<APInt, void>;
friend class APSInt;
+ // Make DynamicAPInt a friend so it can access BitWidth directly.
+ friend DynamicAPInt;
+
/// This constructor is used only internally for speed of construction of
/// temporaries. It is unsafe since it takes ownership of the pointer, so it
/// is not public.
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index f312f776df971..03325d3313b5a 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -35,21 +35,20 @@ namespace llvm {
/// We always_inline all operations; removing these results in a 1.5x
/// performance slowdown.
///
-/// When HoldsLarge is true, a SlowMPInt is held in the union. If it is false,
-/// the int64_t is held. Using std::variant instead would lead to significantly
-/// worse performance.
+/// When isLarge returns true, a SlowMPInt is held in the union. If isSmall
+/// returns true, the int64_t is held. Using std::variant instead would lead to
+/// significantly worse performance.
class DynamicAPInt {
union {
int64_t ValSmall;
detail::SlowDynamicAPInt ValLarge;
};
- unsigned HoldsLarge;
LLVM_ATTRIBUTE_ALWAYS_INLINE void initSmall(int64_t O) {
if (LLVM_UNLIKELY(isLarge()))
ValLarge.detail::SlowDynamicAPInt::~SlowDynamicAPInt();
ValSmall = O;
- HoldsLarge = false;
+ ValLarge.Val.BitWidth = 0;
}
LLVM_ATTRIBUTE_ALWAYS_INLINE void
initLarge(const detail::SlowDynamicAPInt &O) {
@@ -66,14 +65,13 @@ class DynamicAPInt {
// and leak it.
ValLarge = O;
}
- HoldsLarge = true;
}
LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(
const detail::SlowDynamicAPInt &Val)
- : ValLarge(Val), HoldsLarge(true) {}
- LLVM_ATTRIBUTE_ALWAYS_INLINE bool isSmall() const { return !HoldsLarge; }
- LLVM_ATTRIBUTE_ALWAYS_INLINE bool isLarge() const { return HoldsLarge; }
+ : ValLarge(Val) {}
+ constexpr bool isSmall() const { return ValLarge.Val.BitWidth == 0; }
+ constexpr bool isLarge() const { return !isSmall(); }
/// Get the stored value. For getSmall/Large,
/// the stored value should be small/large.
LLVM_ATTRIBUTE_ALWAYS_INLINE int64_t getSmall() const {
@@ -105,14 +103,17 @@ class DynamicAPInt {
public:
LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(int64_t Val)
- : ValSmall(Val), HoldsLarge(false) {}
+ : ValSmall(Val) {
+ ValLarge.Val.BitWidth = 0;
+ }
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt() : DynamicAPInt(0) {}
LLVM_ATTRIBUTE_ALWAYS_INLINE ~DynamicAPInt() {
if (LLVM_UNLIKELY(isLarge()))
ValLarge.detail::SlowDynamicAPInt::~SlowDynamicAPInt();
}
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt(const DynamicAPInt &O)
- : ValSmall(O.ValSmall), HoldsLarge(false) {
+ : ValSmall(O.ValSmall) {
+ ValLarge.Val.BitWidth = 0;
if (LLVM_UNLIKELY(O.isLarge()))
initLarge(O.ValLarge);
}
diff --git a/llvm/include/llvm/ADT/SlowDynamicAPInt.h b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
index 1678bda046fec..cda5f39eb98c3 100644
--- a/llvm/include/llvm/ADT/SlowDynamicAPInt.h
+++ b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
@@ -21,6 +21,10 @@
#include "llvm/ADT/APInt.h"
#include "llvm/Support/raw_ostream.h"
+namespace llvm {
+class DynamicAPInt;
+} // namespace llvm
+
namespace llvm::detail {
/// A simple class providing dynamic arbitrary-precision arithmetic. Internally,
/// it stores an APInt, whose width is doubled whenever an overflow occurs at a
@@ -69,6 +73,9 @@ class SlowDynamicAPInt {
/// Overload to compute a hash_code for a SlowDynamicAPInt value.
friend hash_code hash_value(const SlowDynamicAPInt &X); // NOLINT
+ // Make DynamicAPInt a friend so it can access Val directly.
+ friend DynamicAPInt;
+
unsigned getBitWidth() const { return Val.getBitWidth(); }
void print(raw_ostream &OS) const;
>From 0ed017abfe1006217039ae9e3ec5be46c8caec80 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Fri, 5 Jul 2024 20:10:15 +0100
Subject: [PATCH 2/3] DynamicAPInt: fix inline issue; address review
---
llvm/include/llvm/ADT/DynamicAPInt.h | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index 03325d3313b5a..aaf476ed0f248 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -36,8 +36,11 @@ namespace llvm {
/// performance slowdown.
///
/// When isLarge returns true, a SlowMPInt is held in the union. If isSmall
-/// returns true, the int64_t is held. Using std::variant instead would lead to
-/// significantly worse performance.
+/// returns true, the int64_t is held. We don't have a separate field for
+/// indicating this, and instead "steal" memory from ValLarge when it is not in
+/// use because we know that the memory layout of APInt is such that BitWidth
+/// doesn't overlap with ValSmall (see static_assert_layout). Using std::variant
+/// instead would lead to significantly worse performance.
class DynamicAPInt {
union {
int64_t ValSmall;
@@ -70,8 +73,12 @@ class DynamicAPInt {
LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(
const detail::SlowDynamicAPInt &Val)
: ValLarge(Val) {}
- constexpr bool isSmall() const { return ValLarge.Val.BitWidth == 0; }
- constexpr bool isLarge() const { return !isSmall(); }
+ LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr bool isSmall() const {
+ return ValLarge.Val.BitWidth == 0;
+ }
+ LLVM_ATTRIBUTE_ALWAYS_INLINE constexpr bool isLarge() const {
+ return !isSmall();
+ }
/// Get the stored value. For getSmall/Large,
/// the stored value should be small/large.
LLVM_ATTRIBUTE_ALWAYS_INLINE int64_t getSmall() const {
@@ -204,6 +211,11 @@ class DynamicAPInt {
friend hash_code hash_value(const DynamicAPInt &x); // NOLINT
+ void static_assert_layout() { // NOLINT
+ static_assert(offsetof(DynamicAPInt, ValSmall) !=
+ offsetof(DynamicAPInt, ValLarge.Val.BitWidth));
+ }
+
raw_ostream &print(raw_ostream &OS) const;
LLVM_DUMP_METHOD void dump() const;
};
>From 71fcd1e21266c38cfef974142b1f07328626cd43 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Fri, 5 Jul 2024 20:25:33 +0100
Subject: [PATCH 3/3] DynamicAPInt: strengthen static_assert_layout
---
llvm/include/llvm/ADT/DynamicAPInt.h | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index aaf476ed0f248..b2e2d53b8280c 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -212,8 +212,11 @@ class DynamicAPInt {
friend hash_code hash_value(const DynamicAPInt &x); // NOLINT
void static_assert_layout() { // NOLINT
- static_assert(offsetof(DynamicAPInt, ValSmall) !=
- offsetof(DynamicAPInt, ValLarge.Val.BitWidth));
+ constexpr size_t ValLargeOff =
+ offsetof(DynamicAPInt, ValLarge.Val.BitWidth);
+ constexpr size_t ValSmallOff = offsetof(DynamicAPInt, ValSmall);
+ constexpr size_t ValSmallSz = sizeof(ValSmall);
+ static_assert(ValLargeOff >= ValSmallOff + ValSmallSz);
}
raw_ostream &print(raw_ostream &OS) const;
More information about the llvm-commits
mailing list