[llvm] [LoopIdiomRecognize] Enable clmul optimization for CRC loops (PR #203405)

[Piotr Fusik via llvm-commits]

================
@@ -1582,6 +1590,164 @@ bool LoopIdiomRecognize::optimizeCRCLoop(const PolynomialInfo &Info) {
   if (TT.getArch() == Triple::hexagon)
     return false;
 
+  // In the clmul optimization, the first clmul uses 2*TC bits, and the second
+  // clmul uses CRCBW+TC bits. For simplicity, have both clmuls operate on the
+  // same bit width.
+  unsigned CRCBW = Info.LHS->getType()->getIntegerBitWidth();
+  unsigned ClmulBW = std::max(2 * Info.TripCount, CRCBW + Info.TripCount);
+  auto *ClmulTy = IntegerType::get(Info.LHS->getContext(), ClmulBW);
+
+  // The force-crc-clmul flag should cause the clmul optimization to run
+  // unconditionally.
+  if (ForceCRCClmul) {
+    optimizeCRCLoopUsingClmul(Info, ClmulTy);
+    return true;
+  }
+
+  // FIXME: Once intrinsic cost modeling is more reliable for clmul, that should
+  // be used to determine which optimization to use. Until then, only apply the
+  // clmul optimization when optimizing for size, since a lookup table is not
+  // viable in that case.
+  if (!ApplyCodeSizeHeuristics) {
+    optimizeCRCLoopUsingTableLookup(Info);
+    return true;
+  }
+
+  // The clmul optimization should only be applied if clmul with the required
+  // bit width is a fast operation on the target.
+  // TODO: If clmul exists on the target but not for the required width, it
+  // might be possible to split into multiple iterations of reduction.
+  if (TTI->haveFastClmul(ClmulTy)) {
+    optimizeCRCLoopUsingClmul(Info, ClmulTy);
+    return true;
+  }
+
+  return false;
+}
+
+// The algorithm used in this optimization is a Polynomial (GF(2)) Barrett
+// Reduction based on Intel's "Fast CRC Computation for Generic Polynomials
+// Using PCLMULQDQ Instruction" white paper (December 2009).
+void LoopIdiomRecognize::optimizeCRCLoopUsingClmul(const PolynomialInfo &Info,
+                                                   IntegerType *ClmulTy) {
+  Type *CRCTy = Info.LHS->getType();
+  LLVMContext &Ctx = CRCTy->getContext();
+  unsigned CRCBW = CRCTy->getIntegerBitWidth();
+  // The loop's TripCount determines how many bits of the data are processed,
+  // regardless of whether the actual data bit width matches (if auxiliary data
+  // is even used at all).
+  unsigned TC = Info.TripCount;
+  unsigned ClmulBW = ClmulTy->getBitWidth();
+
+  // First, generate the constants required for GF(2) Barrett reduction.
+  auto [Mu, FullGenPoly] = HashRecognize::genBarrettConstants(Info);
+  Value *MuConst = ConstantInt::get(Ctx, Mu.zext(ClmulBW));
+  Value *GenPolyConst = ConstantInt::get(Ctx, FullGenPoly.zext(ClmulBW));
+
+  IRBuilder<> Builder(CurLoop->getLoopPreheader()->getTerminator());
+
+  auto ShlOrLShr = [&Builder](Value *Op, int ShlAmt, const Twine &Name) {
+    if (ShlAmt > 0)
+      return Builder.CreateShl(Op, ShlAmt, Name);
+    if (ShlAmt < 0)
+      return Builder.CreateLShr(Op, -ShlAmt, Name);
+    return Op;
+  };
+
+  auto LoTCBits = [TC, &Builder, &Ctx](Value *Op, const Twine &Name) {
+    unsigned OpBW = Op->getType()->getIntegerBitWidth();
+    assert(OpBW >= TC && "Bit width should be at least TripCount");
+    auto *Mask = ConstantInt::get(Ctx, APInt::getLowBitsSet(OpBW, TC));
+    return Builder.CreateAnd(Op, Mask, Name);
+  };
+
+  Value *LHS = Builder.CreateZExt(Info.LHS, ClmulTy, "crc.cast");
+
+  // Based on the Intel white paper, in our case, we have
+  // R(x) = (LHS*x^TC) xor (LHSAux ? getTCBits(LHSAux)*x^CRCBW : 0)
+  // since the CRC loop multiplies LHS by x each iteration, and the x^CRCBW term
+  // of getTCBits(LHSAux) is XORed in for the significant bit check.
+  // Rather than compute the full R(x), we can split it in two: a quotient for
+  // step 1 (floor(R(x)/x^CRCBW)) and a remainder for step 3 (R(x) mod x^CRCBW).
+  //
+  // ClmulMuInput is an evolving variable that will eventually become the part
+  // used in step 1, which can be simplified to
+  // (LHS*x^(TC-CRCBW)) xor (LHSAux ? getTCBits(LHSAux) : 0).
+  // Thanks to restrictions imposed by HashRecognize for big-endian CRC loops,
+  // getTCBits(LHSAux) = LHSAux*x^(TC-CRCBW), so this can be further simplified
+  // to (LHS xor (LHSAux ? LHSAux : 0))*x^(TC-CRCBW).
+  Value *ClmulMuInput = LHS;
+
+  // If auxiliary data is present, XOR it in with the CRC.
+  if (Value *Data = Info.LHSAux) {
+    // This is usually a zext, but DataBW may exceed ClmulBW if both CRCBW and
+    // TC are small enough.
+    Data = Builder.CreateZExtOrTrunc(Data, ClmulTy, "data.cast");
+
+    ClmulMuInput = Builder.CreateXor(ClmulMuInput, Data, "xor.crc.data");
+  }
+
+  // Align the current CRC with TripCount (multiply or divide by x^(TC-CRCBW)).
+  ClmulMuInput = Info.IsBigEndian
+                     ? ShlOrLShr(ClmulMuInput, int(TC - CRCBW), "crc.align.tc")
+                     : ClmulMuInput;
----------------
pfusik wrote:

```suggestion
  if (Info.IsBigEndian)
    ClmulMuInput = ShlOrLShr(ClmulMuInput, int(TC - CRCBW), "crc.align.tc");
```

Even better: drop the `ShlOrLShr` lambda which is used only here:
```cpp
  if (Info.IsBigEndian && TC != CRCBW) {
    ClmulMuInput = TC > CRCBW
	    ? Builder.CreateShl(ClmulMuInput, TC - CRCBW, "crc.align.tc")
        : Builder.CreateLShr(ClmulMuInput, CRCBW - TC, "crc.align.tc");
  }
```

https://github.com/llvm/llvm-project/pull/203405