[PATCH] Fix bug 14526 - Make malloc() static analysis M_ZERO-aware

[Meyer, Conrad]

Add M_ZERO awareness to malloc() static analysis in Clang for FreeBSD,
in a similar fashion to O_CREAT for open(2). There is some opportunity
for code-sharing here.

This should reduce the number of false positives when running static
analysis on FreeBSD.

Future work involves a better (symbolic) method of checking for named
flags without hardcoding values.

Sponsored by: EMC/Isilon storage division
Signed-off-by: Conrad Meyer <conrad.meyer at isilon.com>
---
 lib/StaticAnalyzer/Checkers/MallocChecker.cpp  | 52 +++++++++++++++++++++++++-
 lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp |  1 +
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index ca40beb..b89a6a3 100644
--- a/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -16,6 +16,7 @@
 #include "InterCheckerAPI.h"
 #include "clang/AST/Attr.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
@@ -208,6 +209,7 @@ private:
   mutable std::unique_ptr<BugType> BT_OffsetFree[CK_NumCheckKinds];
   mutable IdentifierInfo *II_malloc, *II_free, *II_realloc, *II_calloc,
                          *II_valloc, *II_reallocf, *II_strndup, *II_strdup;
+  mutable Optional<uint64_t> Val_M_ZERO;
 
   void initIdentifierInfo(ASTContext &C) const;
 
@@ -586,7 +588,55 @@ void MallocChecker::checkPostStmt(const CallExpr *CE, CheckerContext &C) const {
     initIdentifierInfo(C.getASTContext());
     IdentifierInfo *FunI = FD->getIdentifier();
 
-    if (FunI == II_malloc || FunI == II_valloc) {
+    if (FunI == II_malloc) {
+      if (CE->getNumArgs() < 1)
+        return;
+      if (CE->getNumArgs() < 2) {
+        State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
+      } else if (CE->getNumArgs() == 3) {
+        // This logic is largely cloned from O_CREAT in UnixAPIChecker, maybe some
+        // code could be shared.
+
+        if (!Val_M_ZERO.hasValue()) {
+          if (C.getASTContext().getTargetInfo().getTriple().getOS()
+                                                            == llvm::Triple::FreeBSD)
+            Val_M_ZERO = 0x00100;
+          else
+            // FIXME: We need a more general way of getting the M_ZERO value.
+            // See also: O_CREAT in UnixAPIChecker.cpp.
+            return;
+        }
+
+        ProgramStateRef state = C.getState();
+        const Expr *flagsEx = CE->getArg(2);
+        const SVal V = state->getSVal(flagsEx, C.getLocationContext());
+        if (!V.getAs<NonLoc>()) {
+          // The case where 'V' can be a location can only be due to a bad header,
+          // so in this case bail out.
+          return;
+        }
+
+        NonLoc flags = V.castAs<NonLoc>();
+        NonLoc zeroFlag = C.getSValBuilder()
+            .makeIntVal(Val_M_ZERO.getValue(), flagsEx->getType()).castAs<NonLoc>();
+        SVal maskedFlagsUC = C.getSValBuilder().evalBinOpNN(state, BO_And,
+                                                            flags, zeroFlag,
+                                                            flagsEx->getType());
+        if (maskedFlagsUC.isUnknownOrUndef())
+          return;
+        DefinedSVal maskedFlags = maskedFlagsUC.castAs<DefinedSVal>();
+
+        // Check if maskedFlags is non-zero.
+        ProgramStateRef trueState, falseState;
+        std::tie(trueState, falseState) = state->assume(maskedFlags);
+
+        // If M_ZERO is set, treat this like calloc (initialized).
+        if (!(trueState && !falseState))
+          State = CallocMem(C, CE);
+        else
+          State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
+      }
+    } else if (FunI == II_valloc) {
       if (CE->getNumArgs() < 1)
         return;
       State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
diff --git a/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index 8869654..aa7e25bc 100644
--- a/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -80,6 +80,7 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
       // FIXME: We need a more general way of getting the O_CREAT value.
       // We could possibly grovel through the preprocessor state, but
       // that would require passing the Preprocessor object to the ExprEngine.
+      // See also: MallocChecker.cpp / M_ZERO.
       return;
     }
   }
-- 
1.8.5.3