[llvm-branch-commits] [llvm] d0104a5 - Make llvm::crc32() work also for input sizes larger than 32 bits.

[Hans Wennborg via llvm-branch-commits]

Author: Hans Wennborg
Date: 2020-02-06T09:02:41+01:00
New Revision: d0104a596199a41963dbba70338d9ff3c36b185a

URL: https://github.com/llvm/llvm-project/commit/d0104a596199a41963dbba70338d9ff3c36b185a
DIFF: https://github.com/llvm/llvm-project/commit/d0104a596199a41963dbba70338d9ff3c36b185a.diff

LOG: Make llvm::crc32() work also for input sizes larger than 32 bits.

The problem was noticed by the Chrome OS toolchain folks
(crbug.com/1048445) because llvm-objcopy --add-gnu-debuglink would
insert the wrong checksum when processing a binary larger than 4 GB.
That use case regressed in 1e1e3ba2526 when we started using
llvm::crc32() in more places.

Differential revision: https://reviews.llvm.org/D74039

(cherry picked from commit 6c4a8bc0a9f6a466d90d542bef66d69550c1b041)

Added: 
    

Modified: 
    llvm/lib/Support/CRC.cpp
    llvm/unittests/Support/CRCTest.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Support/CRC.cpp b/llvm/lib/Support/CRC.cpp
index a3dba1a3aa10..2bc668beed32 100644
--- a/llvm/lib/Support/CRC.cpp
+++ b/llvm/lib/Support/CRC.cpp
@@ -85,7 +85,15 @@ uint32_t llvm::crc32(uint32_t CRC, ArrayRef<uint8_t> Data) {
 
 #include <zlib.h>
 uint32_t llvm::crc32(uint32_t CRC, ArrayRef<uint8_t> Data) {
-  return ::crc32(CRC, (const Bytef *)Data.data(), Data.size());
+  // Zlib's crc32() only takes a 32-bit length, so we have to iterate for larger
+  // sizes. One could use crc32_z() instead, but that's a recent (2017) addition
+  // and may not be available on all systems.
+  do {
+    ArrayRef<uint8_t> Slice = Data.take_front(UINT32_MAX);
+    CRC = ::crc32(CRC, (const Bytef *)Slice.data(), (uInt)Slice.size());
+    Data = Data.drop_front(Slice.size());
+  } while (Data.size() > 0);
+  return CRC;
 }
 
 #endif

diff  --git a/llvm/unittests/Support/CRCTest.cpp b/llvm/unittests/Support/CRCTest.cpp
index 8e2cb2c5e782..32d2cf719399 100644
--- a/llvm/unittests/Support/CRCTest.cpp
+++ b/llvm/unittests/Support/CRCTest.cpp
@@ -13,6 +13,7 @@
 #include "llvm/Support/CRC.h"
 #include "llvm/ADT/StringExtras.h"
 #include "gtest/gtest.h"
+#include <stdlib.h>
 
 using namespace llvm;
 
@@ -39,6 +40,25 @@ TEST(CRCTest, CRC32) {
     uint8_t byte = i;
     EXPECT_EQ(crc, ~llvm::crc32(0xFFFFFFFFU, byte));
   }
+
+  EXPECT_EQ(0x00000000U, llvm::crc32(arrayRefFromStringRef("")));
+}
+
+#if (SIZE_MAX > UINT32_MAX) && defined(EXPENSIVE_CHECKS)
+TEST(CRCTest, LargeCRC32) {
+  // Check that crc32 can handle inputs with sizes larger than 32 bits.
+  size_t TestSize = (size_t)UINT32_MAX + 42;
+  uint8_t *TestData = (uint8_t*)calloc(TestSize, 1);
+  if (!TestData)
+    return;
+
+  // Test expectation generated with:
+  // $ truncate --size=`echo 2^32-1+42 | bc` /tmp/foo
+  // $ crc32 /tmp/foo
+  EXPECT_EQ(0xE46F28FBU, llvm::crc32(makeArrayRef(TestData, TestSize)));
+
+  free(TestData);
 }
+#endif
 
 } // end anonymous namespace