[llvm] [HashRecognize] Fix LHS ConstantRange check for BE (PR #148620)

[Ramkumar Ramachandra via llvm-commits]

artagnon wrote:

Sorry for the delay, but the logic behind this patch is easy to see with examples.

Let us look at a big-endian example first:

```llvm
define i16 @crc16.be(i16 %crc.init) {
entry:
  br label %loop

loop:                                              ; preds = %loop, %entry
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
  %crc.shl = shl i16 %crc, 1
  %crc.xor = xor i16 %crc.shl, 4129
  %check.sb = icmp sge i16 %crc, 0
  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
  %iv.next = add nuw nsw i32 %iv, 1
  %exit.cond = icmp samesign ult i32 %iv, 7
  br i1 %exit.cond, label %loop, label %exit

exit:                                              ; preds = %loop
  ret i16 %crc.next
}
```

Notice `%check.sb = icmp sge i16 %crc, 0`. All the big-endian sb-checks must be of this form. The RHS has range [0, smin), and the LHS has range [0, -1), [0, -3), [0, -7) etc. (as one bit is shifted every iteration). In this patch, we check LHS' range to prevent things like `%check.sb = icmp sge i16 %crc.shl, 0`.

See the little-endian case:

```llvm
define i32 @crc32.le(i32 %checksum, i32 %msg) {
entry:
  br label %loop

loop:                                              ; preds = %loop, %entry
  %crc = phi i32 [ %checksum, %entry ], [ %crc.next, %loop ]
  %data = phi i32 [ %msg, %entry ], [ %data.next, %loop ]
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
  %xor.crc.data = xor i32 %crc, %data
  %sb.crc.data = and i32 %xor.crc.data, 1
  %check.sb = icmp eq i32 %sb.crc.data, 0
  %crc.lshr = lshr i32 %crc, 1
  %crc.xor = xor i32 %crc.lshr, 33800
  %crc.next = select i1 %check.sb, i32 %crc.lshr, i32 %crc.xor
  %iv.next = add nuw nsw i8 %iv, 1
  %data.next = lshr i32 %data, 1
  %exit.cond = icmp samesign ult i8 %iv, 7
  br i1 %exit.cond, label %loop, label %exit

exit:                                              ; preds = %loop
  ret i32 %crc.next
}
```

Notice `%check.sb = icmp eq i32 %sb.crc.data, 0` where `%sb.crc.data = and i32 %xor.crc.data, 1`. All little-endian sb-checks must be of this form. The RHS has range [0, 1) and the LHS has range [0, 2). One bit is shifted every iteration, but we're only checking the least-significant-bit.

https://github.com/llvm/llvm-project/pull/148620