[llvm] use multiplication with holes for 32-bit/64-bit clmul fallback (PR #203727)

[Ramkumar Ramachandra via llvm-commits]

================
@@ -8899,6 +8899,57 @@ SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
     // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
     // Intrinsic::clmul.
 
+    // Strategy 4: multiplication with holes.
+    //
+    // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
+    // do occur, they wind up in a "hole" and are subsequently masked out of the
+    // result.
+    //
+    // A hole of 3 bits is optimal for 32-bit and 64-bit inputs. 128-bit
+    // integers need a larger hole, and for smaller integers the fallback below
+    // is more efficient.
+    //
+    // Based on bmul64 in bearssl and bmul in the rust polyval crate.
+    if (BW >= 32 && BW <= 64 &&
+        isOperationLegalOrCustom(ISD::MUL, getTypeToTransformTo(Ctx, VT))) {
+
+      // Set every fourth bit of each nibble, equivalent to 0b100010001...0001.
+      APInt MaskVal(BW, 0);
+      for (unsigned i = 0; i < BW; i += 4)
+        MaskVal.setBit(i);
+
+      // Create versions of X and Y that keep only the first, second, third, or
+      // fourth bit of every nibble.
+      SDValue M[4], Xp[4], Yp[4];
+      for (unsigned i = 0; i < 4; ++i) {
+        M[i] = DAG.getConstant(MaskVal.shl(i), DL, VT);
+        Xp[i] = DAG.getNode(ISD::AND, DL, VT, X, M[i]);
+        Yp[i] = DAG.getNode(ISD::AND, DL, VT, Y, M[i]);
+      }
+
+      // Codegens these expressions (16 multiplications):
+      //
+      // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
+      // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
+      // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
+      // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
+      SDValue Res;
+      for (unsigned i = 0; i < 4; ++i) {
+        SDValue Zi;
+        for (unsigned A = 0; A < 4; ++A) {
+          unsigned B = (i + 4 - A) % 4;
+          SDValue P = DAG.getNode(ISD::MUL, DL, VT, Xp[A], Yp[B]);
+          Zi = Zi ? DAG.getNode(ISD::XOR, DL, VT, Zi, P) : P;
+        }
+
+        // Keep only the bits belonging to this iteration, and bitwise or it all
+        // together.
+        Zi = DAG.getNode(ISD::AND, DL, VT, Zi, M[i]);
+        Res = Res ? DAG.getNode(ISD::OR, DL, VT, Res, Zi) : Zi;
----------------
artagnon wrote:

Not sure I understand how this produces a clmul? We're or'ing 4 bits, yes? This loop runs four times?

https://github.com/llvm/llvm-project/pull/203727