[llvm] [LoopIdiomRecognize] Enable clmul optimization for CRC loops (PR #203405)
Piotr Fusik via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 05:03:00 PDT 2026
================
@@ -440,30 +637,249 @@ exit: ; preds = %loop
ret i16 %crc.next
}
+define i16 @crc16.be.tc16(i16 %msg, i16 %checksum) {
+; TABLE-LABEL: define i16 @crc16.be.tc16(
+; TABLE-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) {
+; TABLE-NEXT: [[ENTRY:.*]]:
+; TABLE-NEXT: br label %[[LOOP:.*]]
+; TABLE: [[LOOP]]:
+; TABLE-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[CRC2:%.*]] = phi i16 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[CRC_NEXT3:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[IV_BITS:%.*]] = shl i8 [[IV]], 3
+; TABLE-NEXT: [[IV_INDEXER:%.*]] = zext i8 [[IV_BITS]] to i16
+; TABLE-NEXT: [[DATA_INDEXER:%.*]] = shl i16 [[MSG]], [[IV_INDEXER]]
+; TABLE-NEXT: [[CRC_DATA_INDEXER:%.*]] = xor i16 [[DATA_INDEXER]], [[CRC2]]
+; TABLE-NEXT: [[INDEXER_HI:%.*]] = lshr i16 [[CRC_DATA_INDEXER]], 8
+; TABLE-NEXT: [[INDEXER_HI_LO_BYTE:%.*]] = trunc i16 [[INDEXER_HI]] to i8
+; TABLE-NEXT: [[INDEXER_EXT:%.*]] = zext i8 [[INDEXER_HI_LO_BYTE]] to i64
+; TABLE-NEXT: [[TBL_PTRADD:%.*]] = getelementptr inbounds i16, ptr @.crctable.9, i64 [[INDEXER_EXT]]
+; TABLE-NEXT: [[TBL_LD:%.*]] = load i16, ptr [[TBL_PTRADD]], align 2
+; TABLE-NEXT: [[CRC_BE_SHIFT:%.*]] = shl i16 [[CRC2]], 8
+; TABLE-NEXT: [[CRC_NEXT3]] = xor i16 [[CRC_BE_SHIFT]], [[TBL_LD]]
+; TABLE-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; TABLE-NEXT: [[EXIT_COND1:%.*]] = icmp ne i8 [[IV]], 1
+; TABLE-NEXT: br i1 [[EXIT_COND1]], label %[[LOOP]], label %[[EXIT:.*]]
+; TABLE: [[EXIT]]:
+; TABLE-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i16 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; TABLE-NEXT: ret i16 [[CRC_NEXT_LCSSA]]
+;
+; CLMUL-LABEL: define i16 @crc16.be.tc16(
+; CLMUL-SAME: i16 [[MSG:%.*]], i16 [[CHECKSUM:%.*]]) {
+; CLMUL-NEXT: [[ENTRY:.*:]]
+; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i16 [[CHECKSUM]] to i32
+; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32
+; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]]
+; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[XOR_CRC_DATA1]], 65535
+; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 69936)
+; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16
+; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 69665)
+; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16
+; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]]
+; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i16
+; CLMUL-NEXT: br label %[[LOOP:.*]]
+; CLMUL: [[LOOP]]:
+; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CLMUL: [[EXIT]]:
+; CLMUL-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i16 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; CLMUL-NEXT: ret i16 [[CRC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+ %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+ %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+ %xor.crc.data = xor i16 %crc, %data
+ %data.next = shl i16 %data, 1
+ %check.sb = icmp sge i16 %xor.crc.data, 0
+ %crc.shl = shl i16 %crc, 1
+ %crc.xor = xor i16 %crc.shl, 4129
+ %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+ %iv.next = add nuw nsw i8 %iv, 1
+ %exit.cond = icmp samesign ult i8 %iv, 15
+ br i1 %exit.cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret i16 %crc.next
+}
+
+define i8 @crc8.be.tc16.misalign(i16 %msg, i8 %checksum) {
+; TABLE-LABEL: define i8 @crc8.be.tc16.misalign(
+; TABLE-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) {
+; TABLE-NEXT: [[ENTRY:.*]]:
+; TABLE-NEXT: br label %[[LOOP:.*]]
+; TABLE: [[LOOP]]:
+; TABLE-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[CRC2:%.*]] = phi i8 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[TBL_LD:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[IV_BITS:%.*]] = shl i8 [[IV]], 3
+; TABLE-NEXT: [[IV_INDEXER:%.*]] = zext i8 [[IV_BITS]] to i16
+; TABLE-NEXT: [[DATA_INDEXER:%.*]] = shl i16 [[MSG]], [[IV_INDEXER]]
+; TABLE-NEXT: [[CRC_INDEXER_CAST:%.*]] = zext i8 [[CRC2]] to i16
+; TABLE-NEXT: [[CRC_DATA_INDEXER:%.*]] = xor i16 [[DATA_INDEXER]], [[CRC_INDEXER_CAST]]
+; TABLE-NEXT: [[INDEXER_HI_LO_BYTE:%.*]] = trunc i16 [[CRC_DATA_INDEXER]] to i8
+; TABLE-NEXT: [[INDEXER_EXT:%.*]] = zext i8 [[INDEXER_HI_LO_BYTE]] to i64
+; TABLE-NEXT: [[TBL_PTRADD:%.*]] = getelementptr inbounds i8, ptr @.crctable.10, i64 [[INDEXER_EXT]]
+; TABLE-NEXT: [[TBL_LD]] = load i8, ptr [[TBL_PTRADD]], align 1
+; TABLE-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; TABLE-NEXT: [[EXIT_COND1:%.*]] = icmp ne i8 [[IV]], 1
+; TABLE-NEXT: br i1 [[EXIT_COND1]], label %[[LOOP]], label %[[EXIT:.*]]
+; TABLE: [[EXIT]]:
+; TABLE-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i8 [ [[TBL_LD]], %[[LOOP]] ]
+; TABLE-NEXT: ret i8 [[CRC_NEXT_LCSSA]]
+;
+; CLMUL-LABEL: define i8 @crc8.be.tc16.misalign(
+; CLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) {
+; CLMUL-NEXT: [[ENTRY:.*:]]
+; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i32
+; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i16 [[MSG]] to i32
+; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i32 [[CRC_CAST]], [[DATA_CAST]]
+; CLMUL-NEXT: [[CRC_ALIGN_TC:%.*]] = shl i32 [[XOR_CRC_DATA1]], 8
+; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i32 [[CRC_ALIGN_TC]], 65535
+; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i32 @llvm.clmul.i32(i32 [[CRC_TCBITS]], i32 72779)
+; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i32 [[CLMUL_MU]], 16
+; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i32 @llvm.clmul.i32(i32 [[QUOT_LSHR]], i32 285)
+; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i32 [[CRC_CAST]], 16
+; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i32 [[CRC_SHL2]], [[CLMUL_GP]]
+; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i32 [[XOR_CRC_MULT]] to i8
+; CLMUL-NEXT: br label %[[LOOP:.*]]
+; CLMUL: [[LOOP]]:
+; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CLMUL: [[EXIT]]:
+; CLMUL-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i8 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; CLMUL-NEXT: ret i8 [[CRC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+ %crc = phi i8 [ %checksum, %entry ], [ %crc.next, %loop ]
+ %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+ %data.trunc = trunc i16 %data to i8
+ %xor.crc.data = xor i8 %crc, %data.trunc
+ %data.next = shl i16 %data, 1
+ %check.sb = icmp sge i8 %xor.crc.data, 0
+ %crc.shl = shl i8 %crc, 1
+ %crc.xor = xor i8 %crc.shl, 29
+ %crc.next = select i1 %check.sb, i8 %crc.shl, i8 %crc.xor
+ %iv.next = add nuw nsw i8 %iv, 1
+ %exit.cond = icmp samesign ult i8 %iv, 15
+ br i1 %exit.cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret i8 %crc.next
+}
+
+define i8 @crc8.be.tc8.data16.misalign(i16 %msg, i8 %checksum) {
+; TABLE-LABEL: define i8 @crc8.be.tc8.data16.misalign(
+; TABLE-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) {
+; TABLE-NEXT: [[ENTRY:.*]]:
+; TABLE-NEXT: br label %[[LOOP:.*]]
+; TABLE: [[LOOP]]:
+; TABLE-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[CRC2:%.*]] = phi i8 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[TBL_LD:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[IV_BITS:%.*]] = shl i8 [[IV]], 3
+; TABLE-NEXT: [[IV_INDEXER:%.*]] = zext i8 [[IV_BITS]] to i16
+; TABLE-NEXT: [[DATA_INDEXER:%.*]] = shl i16 [[MSG]], [[IV_INDEXER]]
+; TABLE-NEXT: [[CRC_INDEXER_CAST:%.*]] = zext i8 [[CRC2]] to i16
+; TABLE-NEXT: [[CRC_DATA_INDEXER:%.*]] = xor i16 [[DATA_INDEXER]], [[CRC_INDEXER_CAST]]
+; TABLE-NEXT: [[INDEXER_HI_LO_BYTE:%.*]] = trunc i16 [[CRC_DATA_INDEXER]] to i8
+; TABLE-NEXT: [[INDEXER_EXT:%.*]] = zext i8 [[INDEXER_HI_LO_BYTE]] to i64
+; TABLE-NEXT: [[TBL_PTRADD:%.*]] = getelementptr inbounds i8, ptr @.crctable.11, i64 [[INDEXER_EXT]]
+; TABLE-NEXT: [[TBL_LD]] = load i8, ptr [[TBL_PTRADD]], align 1
+; TABLE-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; TABLE-NEXT: [[EXIT_COND1:%.*]] = icmp ne i8 [[IV]], 0
+; TABLE-NEXT: br i1 [[EXIT_COND1]], label %[[LOOP]], label %[[EXIT:.*]]
+; TABLE: [[EXIT]]:
+; TABLE-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i8 [ [[TBL_LD]], %[[LOOP]] ]
+; TABLE-NEXT: ret i8 [[CRC_NEXT_LCSSA]]
+;
+; CLMUL-LABEL: define i8 @crc8.be.tc8.data16.misalign(
+; CLMUL-SAME: i16 [[MSG:%.*]], i8 [[CHECKSUM:%.*]]) {
+; CLMUL-NEXT: [[ENTRY:.*:]]
+; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i8 [[CHECKSUM]] to i16
+; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i16 [[CRC_CAST]], [[MSG]]
+; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i16 [[XOR_CRC_DATA1]], 255
+; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i16 @llvm.clmul.i16(i16 [[CRC_TCBITS]], i16 284)
+; CLMUL-NEXT: [[QUOT_LSHR:%.*]] = lshr i16 [[CLMUL_MU]], 8
+; CLMUL-NEXT: [[CLMUL_GP:%.*]] = call i16 @llvm.clmul.i16(i16 [[QUOT_LSHR]], i16 285)
+; CLMUL-NEXT: [[CRC_SHL2:%.*]] = shl i16 [[CRC_CAST]], 8
+; CLMUL-NEXT: [[XOR_CRC_MULT:%.*]] = xor i16 [[CRC_SHL2]], [[CLMUL_GP]]
+; CLMUL-NEXT: [[CRC_NEXT3:%.*]] = trunc i16 [[XOR_CRC_MULT]] to i8
+; CLMUL-NEXT: br label %[[LOOP:.*]]
+; CLMUL: [[LOOP]]:
+; CLMUL-NEXT: br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CLMUL: [[EXIT]]:
+; CLMUL-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i8 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; CLMUL-NEXT: ret i8 [[CRC_NEXT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+ %crc = phi i8 [ %checksum, %entry ], [ %crc.next, %loop ]
+ %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+ %data.trunc = trunc i16 %data to i8
+ %xor.crc.data = xor i8 %crc, %data.trunc
+ %data.next = shl i16 %data, 1
+ %check.sb = icmp sge i8 %xor.crc.data, 0
+ %crc.shl = shl i8 %crc, 1
+ %crc.xor = xor i8 %crc.shl, 29
+ %crc.next = select i1 %check.sb, i8 %crc.shl, i8 %crc.xor
+ %iv.next = add nuw nsw i8 %iv, 1
+ %exit.cond = icmp samesign ult i8 %iv, 7
+ br i1 %exit.cond, label %loop, label %exit
+
+exit: ; preds = %loop
+ ret i8 %crc.next
+}
+
define i32 @crc32.le.tc8.data32(i32 %checksum, i32 %msg) {
-; CHECK-LABEL: define i32 @crc32.le.tc8.data32(
-; CHECK-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[CRC2:%.*]] = phi i32 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[CRC_NEXT3:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[IV_BITS:%.*]] = shl i8 [[IV]], 3
-; CHECK-NEXT: [[IV_INDEXER:%.*]] = zext i8 [[IV_BITS]] to i32
-; CHECK-NEXT: [[DATA_INDEXER:%.*]] = lshr i32 [[MSG]], [[IV_INDEXER]]
-; CHECK-NEXT: [[CRC_DATA_INDEXER:%.*]] = xor i32 [[DATA_INDEXER]], [[CRC2]]
-; CHECK-NEXT: [[INDEXER_LO:%.*]] = trunc i32 [[CRC_DATA_INDEXER]] to i8
-; CHECK-NEXT: [[INDEXER_EXT:%.*]] = zext i8 [[INDEXER_LO]] to i64
-; CHECK-NEXT: [[TBL_PTRADD:%.*]] = getelementptr inbounds i32, ptr @.crctable.9, i64 [[INDEXER_EXT]]
-; CHECK-NEXT: [[TBL_LD:%.*]] = load i32, ptr [[TBL_PTRADD]], align 4
-; CHECK-NEXT: [[CRC_LE_SHIFT:%.*]] = lshr i32 [[CRC2]], 8
-; CHECK-NEXT: [[CRC_NEXT3]] = xor i32 [[CRC_LE_SHIFT]], [[TBL_LD]]
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
-; CHECK-NEXT: [[EXIT_COND1:%.*]] = icmp ne i8 [[IV]], 0
-; CHECK-NEXT: br i1 [[EXIT_COND1]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i32 [ [[CRC_NEXT3]], %[[LOOP]] ]
-; CHECK-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
+; TABLE-LABEL: define i32 @crc32.le.tc8.data32(
+; TABLE-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) {
+; TABLE-NEXT: [[ENTRY:.*]]:
+; TABLE-NEXT: br label %[[LOOP:.*]]
+; TABLE: [[LOOP]]:
+; TABLE-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[CRC2:%.*]] = phi i32 [ [[CHECKSUM]], %[[ENTRY]] ], [ [[CRC_NEXT3:%.*]], %[[LOOP]] ]
+; TABLE-NEXT: [[IV_BITS:%.*]] = shl i8 [[IV]], 3
+; TABLE-NEXT: [[IV_INDEXER:%.*]] = zext i8 [[IV_BITS]] to i32
+; TABLE-NEXT: [[DATA_INDEXER:%.*]] = lshr i32 [[MSG]], [[IV_INDEXER]]
+; TABLE-NEXT: [[CRC_DATA_INDEXER:%.*]] = xor i32 [[DATA_INDEXER]], [[CRC2]]
+; TABLE-NEXT: [[INDEXER_LO:%.*]] = trunc i32 [[CRC_DATA_INDEXER]] to i8
+; TABLE-NEXT: [[INDEXER_EXT:%.*]] = zext i8 [[INDEXER_LO]] to i64
+; TABLE-NEXT: [[TBL_PTRADD:%.*]] = getelementptr inbounds i32, ptr @.crctable.12, i64 [[INDEXER_EXT]]
+; TABLE-NEXT: [[TBL_LD:%.*]] = load i32, ptr [[TBL_PTRADD]], align 4
+; TABLE-NEXT: [[CRC_LE_SHIFT:%.*]] = lshr i32 [[CRC2]], 8
+; TABLE-NEXT: [[CRC_NEXT3]] = xor i32 [[CRC_LE_SHIFT]], [[TBL_LD]]
+; TABLE-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; TABLE-NEXT: [[EXIT_COND1:%.*]] = icmp ne i8 [[IV]], 0
+; TABLE-NEXT: br i1 [[EXIT_COND1]], label %[[LOOP]], label %[[EXIT:.*]]
+; TABLE: [[EXIT]]:
+; TABLE-NEXT: [[CRC_NEXT_LCSSA:%.*]] = phi i32 [ [[CRC_NEXT3]], %[[LOOP]] ]
+; TABLE-NEXT: ret i32 [[CRC_NEXT_LCSSA]]
+;
+; CLMUL-LABEL: define i32 @crc32.le.tc8.data32(
+; CLMUL-SAME: i32 [[CHECKSUM:%.*]], i32 [[MSG:%.*]]) {
+; CLMUL-NEXT: [[ENTRY:.*:]]
+; CLMUL-NEXT: [[CRC_CAST:%.*]] = zext i32 [[CHECKSUM]] to i40
+; CLMUL-NEXT: [[DATA_CAST:%.*]] = zext i32 [[MSG]] to i40
+; CLMUL-NEXT: [[XOR_CRC_DATA1:%.*]] = xor i40 [[CRC_CAST]], [[DATA_CAST]]
+; CLMUL-NEXT: [[CRC_TCBITS:%.*]] = and i40 [[XOR_CRC_DATA1]], 255
+; CLMUL-NEXT: [[CLMUL_MU:%.*]] = call i40 @llvm.clmul.i40(i40 [[CRC_TCBITS]], i40 273)
----------------
pfusik wrote:
CRC_TCBITS is 8-bit. 273 is 9-bit. Looks like a 16-bit clmul would suffice?
https://github.com/llvm/llvm-project/pull/203405
More information about the llvm-commits
mailing list