[PATCH] D74039: Make llvm::crc32() work also for input sizes larger than 32 bits.

[Hans Wennborg via Phabricator via llvm-commits]

hans created this revision.
hans added reviewers: manojgupta, thakis, rnk.
Herald added a subscriber: hiraditya.
Herald added a reviewer: alexshap.
Herald added a project: LLVM.

The problem was noticed by the Chrome OS toolchain folks
(crbug.com/1048445) because llvm-objcopy --add-gnu-debuglink would
insert the wrong checksum when processing a binary clarger than 4 GB.
That use case regressed in 1e1e3ba2526 when we started using
llvm::crc32() in more places.


https://reviews.llvm.org/D74039

Files:
  llvm/lib/Support/CRC.cpp
  llvm/unittests/Support/CRCTest.cpp


Index: llvm/unittests/Support/CRCTest.cpp
===================================================================
--- llvm/unittests/Support/CRCTest.cpp
+++ llvm/unittests/Support/CRCTest.cpp
@@ -13,6 +13,7 @@
 #include "llvm/Support/CRC.h"
 #include "llvm/ADT/StringExtras.h"
 #include "gtest/gtest.h"
+#include <stdlib.h>
 
 using namespace llvm;
 
@@ -39,6 +40,25 @@
     uint8_t byte = i;
     EXPECT_EQ(crc, ~llvm::crc32(0xFFFFFFFFU, byte));
   }
+
+  EXPECT_EQ(0x00000000U, llvm::crc32(arrayRefFromStringRef("")));
+}
+
+#if (SIZE_MAX > UINT32_MAX) && defined(EXPENSIVE_CHECKS)
+TEST(CRCTest, LargeCRC32) {
+  // Check that crc32 can handle inputs with sizes larger than 32 bits.
+  size_t TestSize = (size_t)UINT32_MAX + 42;
+  uint8_t *TestData = (uint8_t*)calloc(TestSize, 1);
+  if (!TestData)
+    return;
+
+  // Test expectation generated with:
+  // $ truncate --size=`echo 2^32-1+42 | bc` /tmp/foo
+  // $ crc32 /tmp/foo
+  EXPECT_EQ(0xE46F28FBU, llvm::crc32(makeArrayRef(TestData, TestSize)));
+
+  free(TestData);
 }
+#endif
 
 } // end anonymous namespace
Index: llvm/lib/Support/CRC.cpp
===================================================================
--- llvm/lib/Support/CRC.cpp
+++ llvm/lib/Support/CRC.cpp
@@ -85,7 +85,15 @@
 
 #include <zlib.h>
 uint32_t llvm::crc32(uint32_t CRC, ArrayRef<uint8_t> Data) {
-  return ::crc32(CRC, (const Bytef *)Data.data(), Data.size());
+  // Zlib's crc32() only takes a 32-bit length, so we have to iterate for larger
+  // sizes. One could use crc32_z() instead, but that's a recent (2017) addition
+  // and may not be available on all systems.
+  do {
+    ArrayRef<uint8_t> Slice = Data.take_front(UINT32_MAX);
+    CRC = ::crc32(CRC, (const Bytef *)Slice.data(), (uInt)Slice.size());
+    Data = Data.drop_front(Slice.size());
+  } while (Data.size() > 0);
+  return CRC;
 }
 
 #endif


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D74039.242555.patch
Type: text/x-patch
Size: 1849 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20200205/f45e069f/attachment.bin>