[llvm] [FoldingSet] Switch to linear probing and Algorithm R deletion (PR #218190)

[Fangrui Song via llvm-commits]

================
@@ -207,145 +175,110 @@ FoldingSetBase::~FoldingSetBase() { free(Buckets); }
 
 void FoldingSetBase::clear() {
   incrementEpoch();
-  // Set all but the last bucket to null pointers.
-  memset(Buckets, 0, NumBuckets * sizeof(void *));
-
-  // Set the very last bucket to be a non-null "pointer".
-  Buckets[NumBuckets] = reinterpret_cast<void *>(-1);
-
-  // Reset the node count to zero.
+  // Stale hashes are unreachable, so only the occupancy needs resetting.
+  if (NumBuckets)
+    memset(Buckets, 0, NumBuckets * sizeof(void *));
   NumNodes = 0;
 }
 
-void FoldingSetBase::GrowBucketCount(unsigned NewBucketCount,
-                                     const FoldingSetInfo &Info) {
-  assert((NewBucketCount > NumBuckets) &&
-         "Can't shrink a folding set with GrowBucketCount");
-  assert(isPowerOf2_32(NewBucketCount) && "Bad bucket count!");
+void FoldingSetBase::placeNode(Node *N, uint32_t Hash) {
+  unsigned Mask = NumBuckets - 1;
+  unsigned I = Hash & Mask;
+  while (Buckets[I]) {
+    // A second copy of N hashes here too, so the probe always passes it.
+    assert(Buckets[I] != N && "Node already in the folding set");
+    I = (I + 1) & Mask;
+  }
+  Buckets[I] = N;
+  ++NumNodes;
+}
+
+void FoldingSetBase::grow(unsigned MinNumBuckets) {
+  // The floor is the smallest size the constructor accepts.
+  unsigned NewBucketCount = std::max(64u, llvm::bit_ceil(MinNumBuckets));
+  assert(NewBucketCount > NumBuckets && "Can't shrink a folding set");
 
   FoldingSetBase Tmp(llvm::Log2_32(NewBucketCount));
-  FoldingSetNodeID TempID;
-  for (unsigned i = 0; i != NumBuckets; ++i) {
-    void *Probe = Buckets[i];
-    if (!Probe)
-      continue;
-    while (Node *NodeInBucket = GetNextPtr(Probe)) {
-      // Figure out the next link, remove NodeInBucket from the old link.
-      Probe = NodeInBucket->getNextInBucket();
-      NodeInBucket->SetNextInBucket(nullptr);
-
-      // Insert the node into the new bucket, after recomputing the hash.
-      Tmp.InsertNode(
-          NodeInBucket,
-          GetBucketFor(Info.ComputeNodeHash(this, NodeInBucket, TempID),
-                       Tmp.Buckets, Tmp.NumBuckets),
-          Info);
-      TempID.clear();
-    }
-  }
+  for (unsigned I = 0; I != NumBuckets; ++I)
+    if (void *N = Buckets[I])
+      Tmp.placeNode(static_cast<Node *>(N),
+                    static_cast<Node *>(N)->getFoldingSetHash());
 
   *this = std::move(Tmp);
 }
 
-void FoldingSetBase::reserve(unsigned EltCount, const FoldingSetInfo &Info) {
-  // This will give us somewhere between EltCount / 2 and
-  // EltCount buckets.  This puts us in the load factor
-  // range of 1.0 - 2.0.
-  if (EltCount <= capacity())
+void FoldingSetBase::reserve(unsigned N) {
+  if (N * 4 <= NumBuckets * 3)
     return;
-  GrowBucketCount(llvm::bit_floor(EltCount), Info);
+  // N + (N + 2) / 3 is ceil(4N/3).
+  grow(N + (N + 2) / 3);
+}
+
+LLVM_ATTRIBUTE_NOINLINE bool
+FoldingSetBase::nodeEquals(const FoldingSetInfo &Info,
+                           const FoldingSetBase *Self, Node *N,
+                           const FoldingSetNodeID &ID, unsigned IDHash) {
+  FoldingSetNodeID TempID;
+  return Info.NodeEquals(Self, N, ID, IDHash, TempID);
 }
 
 FoldingSetBase::Node *FoldingSetBase::FindNodeOrInsertPos(
     const FoldingSetNodeID &ID, void *&InsertPos, const FoldingSetInfo &Info) {
   unsigned IDHash = ID.ComputeHash();
-  void **Bucket = GetBucketFor(IDHash, Buckets, NumBuckets);
-  void *Probe = *Bucket;
-
-  InsertPos = nullptr;
-
-  FoldingSetNodeID TempID;
-  while (Node *NodeInBucket = GetNextPtr(Probe)) {
-    if (Info.NodeEquals(this, NodeInBucket, ID, IDHash, TempID))
-      return NodeInBucket;
-    TempID.clear();
-
-    Probe = NodeInBucket->getNextInBucket();
+  unsigned Mask = NumBuckets - 1;
+  for (unsigned I = IDHash & Mask; Buckets[I]; I = (I + 1) & Mask) {
+    // Reject on the hash first, so a probe step touches no node.
+    Node *N = static_cast<Node *>(Buckets[I]);
+    if (N->getFoldingSetHash() != IDHash)
+      continue;
+    if (nodeEquals(Info, this, N, ID, IDHash)) {
+      InsertPos = nullptr;
+      return N;
+    }
   }
 
-  // Didn't find the node, return null with the bucket as the InsertPos.
-  InsertPos = Bucket;
+  InsertPos = encodeHash(IDHash);
   return nullptr;
 }
 
-void FoldingSetBase::InsertNode(Node *N, void *InsertPos,
-                                const FoldingSetInfo &Info) {
-  assert(!N->getNextInBucket());
+void FoldingSetBase::InsertNode(Node *N, void *InsertPos) {
+  assert(InsertPos && "Invalid InsertPos!");
   incrementEpoch();
-  // Do we need to grow the hashtable?
-  if (NumNodes + 1 > capacity()) {
-    GrowBucketCount(NumBuckets * 2, Info);
-    FoldingSetNodeID TempID;
-    InsertPos = GetBucketFor(Info.ComputeNodeHash(this, N, TempID), Buckets,
-                             NumBuckets);
-  }
-
-  ++NumNodes;
-
-  /// The insert position is actually a bucket pointer.
-  void **Bucket = static_cast<void **>(InsertPos);
-
-  void *Next = *Bucket;
-
-  // If this is the first insertion into this bucket, its next pointer will be
-  // null.  Pretend as if it pointed to itself, setting the low bit to indicate
-  // that it is a pointer to the bucket.
-  if (!Next)
-    Next = reinterpret_cast<void *>(reinterpret_cast<intptr_t>(Bucket) | 1);
-
-  // Set the node's next pointer, and make the bucket point to the node.
-  N->SetNextInBucket(Next);
-  *Bucket = N;
+  if (LLVM_UNLIKELY((NumNodes + 1) * 4 > NumBuckets * 3))
+    grow(NumBuckets * 2);
+  uint32_t Hash = decodeHash(InsertPos);
+  placeNode(N, Hash);
+  N->setFoldingSetHash(Hash);
 }
 
 bool FoldingSetBase::RemoveNode(Node *N) {
-  // Because each bucket is a circular list, we don't need to compute N's hash
-  // to remove it.
-  void *Ptr = N->getNextInBucket();
-  if (!Ptr)
-    return false; // Not in folding set.
+  uint32_t Hash = N->getFoldingSetHash();
+  if (Hash == FoldingSetNodeIDRef::NotAHash)
+    return false; // Never inserted.
+
+  unsigned Mask = NumBuckets - 1;
+  unsigned I = Hash & Mask;
+  while (Buckets[I] != N) {
+    if (!Buckets[I])
----------------
MaskRay wrote:

Done

https://github.com/llvm/llvm-project/pull/218190