[llvm] Fix a signed integer overflow in BitstreamWriter.h which is found by UBSAN. (PR #75213)

[via llvm-commits]

https://github.com/MaggieYingYi created https://github.com/llvm/llvm-project/pull/75213

UBSAN finds undefined behavior (signed integer overflow) in `BitstreamWriter.h` line [242](https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/Bitstream/BitstreamWriter.h#L242) and [258](https://github.com/llvm/llvm-project/blob/main/llvm/include/llvm/Bitstream/BitstreamWriter.h#L258).

To reproduce the issue, I have copied the functions `EmitVBR` and `EmitVBR64` into a simple code main.cpp:
```
$ cat main.cpp
#include <cassert>
#include <cstdint>

void Emit(uint32_t Val, unsigned NumBits) {
  assert(NumBits && NumBits <= 32 && "Invalid value size!");
  assert((Val & ~(~0U >> (32-NumBits))) == 0 && "High bits set!");
}

void EmitVBR(uint32_t Val, unsigned NumBits) {
  assert(NumBits <= 32 && "Too many bits to emit!");
  uint32_t Threshold = 1U << (NumBits-1);

  // Emit the bits with VBR encoding, NumBits-1 bits at a time.
  while (Val >= Threshold) {
    Emit((Val & ((1 << (NumBits-1))-1)) | (1 << (NumBits-1)), NumBits);
    Val >>= NumBits-1;
  }

  Emit(Val, NumBits);
}

void EmitVBR64(uint64_t Val, unsigned NumBits) {
  assert(NumBits <= 32 && "Too many bits to emit!");
  if ((uint32_t)Val == Val)
    return EmitVBR((uint32_t)Val, NumBits);

  uint32_t Threshold = 1U << (NumBits-1);

  // Emit the bits with VBR encoding, NumBits-1 bits at a time.
  while (Val >= Threshold) {
    Emit(((uint32_t)Val & ((1 << (NumBits - 1)) - 1)) | (1 << (NumBits - 1)),
         NumBits);
    Val >>= NumBits-1;
  }

  Emit((uint32_t)Val, NumBits);
}

int main() {
  unsigned NumBits= 32;
  uint32_t UInt32 = 4294967295;
  uint64_t UInt64 = 1844674407370955161;
 
  EmitVBR64(UInt32, NumBits);
  EmitVBR64(UInt64, NumBits);
  return 0;
}
```

Compile the code using 

To reproduce this issue, build main.cpp with the `-fsanitize=undefined` option:
```
clang++.exe -fsanitize=undefined main.cpp -o main.exe
```

Run the main.exe:
```
$ main.exe
main.cpp:15:36: runtime error: signed integer overflow: -2147483648 - 1 cannot be represented in type 'int'
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior main.cpp:15:36 in
main.cpp:31:49: runtime error: signed integer overflow: -2147483648 - 1 cannot be represented in type 'int'
SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior main.cpp:31:49 in
```

The issue could be fixed by using 1U instead of 1.

>From aa826ab831ad3fd57a2473fc0cc45e3da292de2f Mon Sep 17 00:00:00 2001
From: Ying Yi <ying.yi at sony.com>
Date: Tue, 12 Dec 2023 12:13:47 +0000
Subject: [PATCH] Fixed a signed integer overflow in BitstreamWriter.h which is
 found by UBSAN.

---
 llvm/include/llvm/Bitstream/BitstreamWriter.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Bitstream/BitstreamWriter.h b/llvm/include/llvm/Bitstream/BitstreamWriter.h
index f7d362b5d70ceb..c726508cd52851 100644
--- a/llvm/include/llvm/Bitstream/BitstreamWriter.h
+++ b/llvm/include/llvm/Bitstream/BitstreamWriter.h
@@ -239,7 +239,8 @@ class BitstreamWriter {
 
     // Emit the bits with VBR encoding, NumBits-1 bits at a time.
     while (Val >= Threshold) {
-      Emit((Val & ((1 << (NumBits-1))-1)) | (1 << (NumBits-1)), NumBits);
+      Emit((Val & ((1U << (NumBits - 1)) - 1)) | (1U << (NumBits - 1)),
+           NumBits);
       Val >>= NumBits-1;
     }
 
@@ -255,7 +256,8 @@ class BitstreamWriter {
 
     // Emit the bits with VBR encoding, NumBits-1 bits at a time.
     while (Val >= Threshold) {
-      Emit(((uint32_t)Val & ((1 << (NumBits - 1)) - 1)) | (1 << (NumBits - 1)),
+      Emit(((uint32_t)Val & ((1U << (NumBits - 1)) - 1)) |
+               (1U << (NumBits - 1)),
            NumBits);
       Val >>= NumBits-1;
     }