[llvm] [FoldingSet] Switch to linear probing and Algorithm R deletion (PR #218190)

[Fangrui Song via llvm-commits]

================
@@ -207,145 +175,110 @@ FoldingSetBase::~FoldingSetBase() { free(Buckets); }
 
 void FoldingSetBase::clear() {
   incrementEpoch();
-  // Set all but the last bucket to null pointers.
-  memset(Buckets, 0, NumBuckets * sizeof(void *));
-
-  // Set the very last bucket to be a non-null "pointer".
-  Buckets[NumBuckets] = reinterpret_cast<void *>(-1);
-
-  // Reset the node count to zero.
+  // Stale hashes are unreachable, so only the occupancy needs resetting.
+  if (NumBuckets)
+    memset(Buckets, 0, NumBuckets * sizeof(void *));
   NumNodes = 0;
 }
 
-void FoldingSetBase::GrowBucketCount(unsigned NewBucketCount,
-                                     const FoldingSetInfo &Info) {
-  assert((NewBucketCount > NumBuckets) &&
-         "Can't shrink a folding set with GrowBucketCount");
-  assert(isPowerOf2_32(NewBucketCount) && "Bad bucket count!");
+void FoldingSetBase::placeNode(Node *N, uint32_t Hash) {
+  unsigned Mask = NumBuckets - 1;
+  unsigned I = Hash & Mask;
+  while (Buckets[I]) {
+    // A second copy of N hashes here too, so the probe always passes it.
+    assert(Buckets[I] != N && "Node already in the folding set");
+    I = (I + 1) & Mask;
+  }
+  Buckets[I] = N;
+  ++NumNodes;
+}
+
+void FoldingSetBase::grow(unsigned MinNumBuckets) {
+  // The floor is the smallest size the constructor accepts.
+  unsigned NewBucketCount = std::max(64u, llvm::bit_ceil(MinNumBuckets));
+  assert(NewBucketCount > NumBuckets && "Can't shrink a folding set");
 
   FoldingSetBase Tmp(llvm::Log2_32(NewBucketCount));
-  FoldingSetNodeID TempID;
-  for (unsigned i = 0; i != NumBuckets; ++i) {
-    void *Probe = Buckets[i];
-    if (!Probe)
-      continue;
-    while (Node *NodeInBucket = GetNextPtr(Probe)) {
-      // Figure out the next link, remove NodeInBucket from the old link.
-      Probe = NodeInBucket->getNextInBucket();
-      NodeInBucket->SetNextInBucket(nullptr);
-
-      // Insert the node into the new bucket, after recomputing the hash.
-      Tmp.InsertNode(
-          NodeInBucket,
-          GetBucketFor(Info.ComputeNodeHash(this, NodeInBucket, TempID),
-                       Tmp.Buckets, Tmp.NumBuckets),
-          Info);
-      TempID.clear();
-    }
-  }
+  for (unsigned I = 0; I != NumBuckets; ++I)
+    if (void *N = Buckets[I])
+      Tmp.placeNode(static_cast<Node *>(N),
+                    static_cast<Node *>(N)->getFoldingSetHash());
 
   *this = std::move(Tmp);
 }
 
-void FoldingSetBase::reserve(unsigned EltCount, const FoldingSetInfo &Info) {
-  // This will give us somewhere between EltCount / 2 and
-  // EltCount buckets.  This puts us in the load factor
-  // range of 1.0 - 2.0.
-  if (EltCount <= capacity())
+void FoldingSetBase::reserve(unsigned N) {
+  if (N * 4 <= NumBuckets * 3)
     return;
-  GrowBucketCount(llvm::bit_floor(EltCount), Info);
+  // N + (N + 2) / 3 is ceil(4N/3).
+  grow(N + (N + 2) / 3);
+}
+
+LLVM_ATTRIBUTE_NOINLINE bool
+FoldingSetBase::nodeEquals(const FoldingSetInfo &Info,
+                           const FoldingSetBase *Self, Node *N,
+                           const FoldingSetNodeID &ID, unsigned IDHash) {
+  FoldingSetNodeID TempID;
+  return Info.NodeEquals(Self, N, ID, IDHash, TempID);
 }
 
 FoldingSetBase::Node *FoldingSetBase::FindNodeOrInsertPos(
     const FoldingSetNodeID &ID, void *&InsertPos, const FoldingSetInfo &Info) {
   unsigned IDHash = ID.ComputeHash();
-  void **Bucket = GetBucketFor(IDHash, Buckets, NumBuckets);
-  void *Probe = *Bucket;
-
-  InsertPos = nullptr;
-
-  FoldingSetNodeID TempID;
-  while (Node *NodeInBucket = GetNextPtr(Probe)) {
-    if (Info.NodeEquals(this, NodeInBucket, ID, IDHash, TempID))
-      return NodeInBucket;
-    TempID.clear();
-
-    Probe = NodeInBucket->getNextInBucket();
+  unsigned Mask = NumBuckets - 1;
+  for (unsigned I = IDHash & Mask; Buckets[I]; I = (I + 1) & Mask) {
+    // Reject on the hash first, so a probe step touches no node.
+    Node *N = static_cast<Node *>(Buckets[I]);
+    if (N->getFoldingSetHash() != IDHash)
+      continue;
+    if (nodeEquals(Info, this, N, ID, IDHash)) {
----------------
MaskRay wrote:

Applied!

https://github.com/llvm/llvm-project/pull/218190