[clang] [analyzer] Model overflow builtins (PR #102602)

[Balazs Benics via cfe-commits]

================
@@ -50,6 +122,107 @@ class BuiltinFunctionChecker : public Checker<eval::Call> {
 
 } // namespace
 
+const NoteTag *BuiltinFunctionChecker::createBuiltinNoOverflowNoteTag(
+    CheckerContext &C, bool bothFeasible, SVal Arg1, SVal Arg2,
+    SVal Result) const {
+  return C.getNoteTag([Result, Arg1, Arg2, bothFeasible](
+                          PathSensitiveBugReport &BR, llvm::raw_ostream &OS) {
+    if (!BR.isInteresting(Result))
+      return;
+
+    // Propagate interestingness to input argumets if result is interesting.
+    BR.markInteresting(Arg1);
+    BR.markInteresting(Arg2);
+
+    if (bothFeasible)
+      OS << "Assuming overflow does not happen";
+  });
+}
+
+const NoteTag *
+BuiltinFunctionChecker::createBuiltinOverflowNoteTag(CheckerContext &C) const {
+  return C.getNoteTag(
+      [](PathSensitiveBugReport &BR, llvm::raw_ostream &OS) {
+        OS << "Assuming overflow does happen";
+      },
+      /*isPrunable=*/true);
+}
+
+std::pair<bool, bool>
+BuiltinFunctionChecker::checkOverflow(CheckerContext &C, SVal RetVal,
+                                      QualType Res) const {
+  ProgramStateRef State = C.getState();
+  SValBuilder &SVB = C.getSValBuilder();
+  ASTContext &ACtx = C.getASTContext();
+
+  // Calling a builtin with a non-integer type result produces compiler error.
+  assert(Res->isIntegerType());
+
+  unsigned BitWidth = ACtx.getIntWidth(Res);
+  auto MinVal =
+      llvm::APSInt::getMinValue(BitWidth, Res->isUnsignedIntegerType());
+  auto MaxVal =
+      llvm::APSInt::getMaxValue(BitWidth, Res->isUnsignedIntegerType());
+
+  SVal IsLeMax =
+      SVB.evalBinOp(State, BO_LE, RetVal, nonloc::ConcreteInt(MaxVal), Res);
+  SVal IsGeMin =
+      SVB.evalBinOp(State, BO_GE, RetVal, nonloc::ConcreteInt(MinVal), Res);
+
+  auto [MayNotOverflow, MayOverflow] =
+      State->assume(IsLeMax.castAs<DefinedOrUnknownSVal>());
+  auto [MayNotUnderflow, MayUnderflow] =
+      State->assume(IsGeMin.castAs<DefinedOrUnknownSVal>());
+
+  return {MayOverflow || MayUnderflow, MayNotOverflow && MayNotUnderflow};
----------------
steakhal wrote:

```suggestion
  // Calling a builtin with a non-integer type result produces compiler error.
  assert(Res->isIntegerType());

  unsigned BitWidth = C.getASTContext().getIntWidth(Res);
  bool IsUnsigned = Res->isUnsignedIntegerType();

  nonloc::ConcreteInt MinVal{llvm::APSInt::getMinValue(BitWidth, IsUnsigned)};
  nonloc::ConcreteInt MaxVal{llvm::APSInt::getMaxValue(BitWidth, IsUnsigned)};

  SValBuilder &SVB = C.getSValBuilder();
  ProgramStateRef State = C.getState();
  SVal IsLeMax = SVB.evalBinOp(State, BO_LE, RetVal, MaxVal, Res);
  SVal IsGeMin = SVB.evalBinOp(State, BO_GE, RetVal, MinVal, Res);

  auto [MayNotOverflow, MayOverflow] =
      State->assume(IsLeMax.castAs<DefinedOrUnknownSVal>());
  auto [MayNotUnderflow, MayUnderflow] =
      State->assume(IsGeMin.castAs<DefinedOrUnknownSVal>());

  return {MayOverflow || MayUnderflow, MayNotOverflow && MayNotUnderflow};
```

Uh, the diff of this suggestion looks horrible even though I basically just reshuffled the lines to have a combination where the variable definitions are closer to their use, keep symmetry and try to prevent linebreaks.

https://github.com/llvm/llvm-project/pull/102602