[llvm] [LoopIdiomRecognize] Enable clmul optimization for CRC loops (PR #203405)

[Ramkumar Ramachandra via llvm-commits]

================
@@ -1665,24 +1649,105 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
     Indexer = Info.ByteOrderSwapped ? HiIdx(Builder, Indexer, "indexer.hi")
                                     : LoByte(Builder, Indexer, "indexer.lo");
 
-    // Always index into a GEP using the index type.
-    Indexer = Builder.CreateZExt(
-        Indexer, SE->getDataLayout().getIndexType(GV->getType()),
-        "indexer.ext");
-
-    // CRCTableLd = CRCTable[(iv'th byte of data) ^ (top|bottom) byte of CRC].
-    Value *CRCTableGEP =
-        Builder.CreateInBoundsGEP(CRCTy, GV, Indexer, "tbl.ptradd");
-    Value *CRCTableLd = Builder.CreateLoad(CRCTy, CRCTableGEP, "tbl.ld");
-
-    // CRCNext = (CRC (<<|>>) 8) ^ CRCTableLd, or simply CRCTableLd in case of
-    // CRC-8.
-    Value *CRCNext = CRCTableLd;
-    if (CRCBW > 8) {
-      Value *CRCShift = Info.ByteOrderSwapped
-                            ? Builder.CreateShl(CRC, 8, "crc.be.shift")
-                            : Builder.CreateLShr(CRC, 8, "crc.le.shift");
-      CRCNext = Builder.CreateXor(CRCShift, CRCTableLd, "crc.next");
+    // Compute the next value of the CRC register based on Indexer. This could
+    // either be a table load or an on-the-fly computation, depending on whether
+    // llvm.clmul is fast on the target.
+    Value *CRCNext;
+    Type *ClmulTy = IntegerType::get(CRCTy->getContext(), 2 * CRCBW);
+    if (TTI->haveFastClmul(ClmulTy)) {
+      // Use a Barrett-style reduction to calculate CRCNext with two clmuls.
+      // Little-endian:
+      //   q = ((iv'th byte of data) ^ (bottom byte of crc)) clmul (reciprocal)
+      //   entry = (bottom byte of q) clmul (genpoly)
+      //   crc = (crc >> 8) ^ (entry >> 7)
+      //   Equivalently, to save one shift instruction:
+      //   (genBarrettConstants gives us genpoly << 1)
+      //   entry = (bottom byte of q) clmul (genpoly << 1)
+      //   crc = (crc ^ entry) >> 8
+      // Big-Endian:
+      //   q = ((iv'th byte of data) ^ (top byte of crc)) clmul (reciprocal)
+      //   entry = (top byte of q) clmul (genpoly)
+      //   crc = (crc << 8) ^ (bottom CRCBW bits of entry)
+
+      // First, generate the two constants required for a Barrett-style
+      // reduction.
+      CRCBarrettConstants Constants =
+          HashRecognize::genBarrettConstants(Info.RHS, Info.ByteOrderSwapped);
+      Value *Reciprocal = ConstantInt::get(ClmulTy, Constants.Reciprocal);
+      Value *Generator = ConstantInt::get(ClmulTy, Constants.Generator);
+
+      // Approximate floor(Indexer / GenPoly) using Indexer * Reciprocal.
+      Indexer = Builder.CreateZExt(Indexer, ClmulTy, "indexer.ext");
+      Value *Quotient = Builder.CreateBinaryIntrinsic(Intrinsic::clmul, Indexer,
+                                                      Reciprocal, {}, "quot");
+      Quotient = Info.ByteOrderSwapped
+                     ? Builder.CreateLShr(Quotient, CRCBW, "quot.be.shift")
+                     : Builder.CreateAnd(Quotient, 0xFF, "quot.le.mask");
+
+      // floor(Indexer / GenPoly) * GenPoly should give us the equivalent of the
+      // Sarwate table entry.
+      Value *ComputedByte = Builder.CreateBinaryIntrinsic(
+          Intrinsic::clmul, Quotient, Generator, {}, "entry");
----------------
artagnon wrote:

Can you kindly check your testing environment, because this is incorrect: we should never be using the Generator, once we have the Reciprocal.

https://github.com/llvm/llvm-project/pull/203405