[PATCH] D20543: Fix a suspicious check in TargetLibraryInfo

[Steven Wu via llvm-commits]

steven_wu created this revision.
steven_wu added reviewers: ab, mzolotukhin.
steven_wu added a subscriber: llvm-commits.

The check for some of the builtin functions involves checking if the number
of the parameter is zero and the first parameter is a pointer type. This check
either returns false or an out-of-bound memory access.

rdar://problem/26424030

http://reviews.llvm.org/D20543

Files:
  lib/Analysis/TargetLibraryInfo.cpp
  test/Transforms/InferFunctionAttrs/annotate.ll

Index: test/Transforms/InferFunctionAttrs/annotate.ll
===================================================================
--- test/Transforms/InferFunctionAttrs/annotate.ll
+++ test/Transforms/InferFunctionAttrs/annotate.ll
@@ -38,3 +38,6 @@
 declare i32 @__nvvm_reflect(i8*)
 ; CHECK-NVPTX: declare i32 @__nvvm_reflect(i8*) [[G0:#[0-9]+]]
 ; CHECK-NVPTX: attributes [[G0]] = { nounwind readnone }
+
+; Make sure following testcase doesn't crash especially with ASAN
+declare void @free(...)
Index: lib/Analysis/TargetLibraryInfo.cpp
===================================================================
--- lib/Analysis/TargetLibraryInfo.cpp
+++ lib/Analysis/TargetLibraryInfo.cpp
@@ -663,9 +663,8 @@
   case LibFunc::memalign:
     return (FTy.getReturnType()->isPointerTy());
   case LibFunc::mkdir:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::mktime:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
+    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::realloc:
     return (NumParams == 2 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getReturnType()->isPointerTy());
@@ -697,11 +696,10 @@
     return (NumParams == 2 && FTy.getReturnType()->isPointerTy());
   case LibFunc::chmod:
   case LibFunc::chown:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::ctermid:
   case LibFunc::clearerr:
   case LibFunc::closedir:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
+    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::atoi:
   case LibFunc::atol:
   case LibFunc::atof:
@@ -730,7 +728,6 @@
   case LibFunc::flockfile:
   case LibFunc::funlockfile:
   case LibFunc::ftrylockfile:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::ferror:
     return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::fputc:
@@ -763,9 +760,10 @@
     return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy() &&
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::getc:
+    return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::getlogin_r:
+    return (NumParams == 2 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::getc_unlocked:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::getenv:
     return (NumParams == 1 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::gets:
@@ -867,7 +865,7 @@
             FTy.getParamType(1)->isPointerTy());
   case LibFunc::fseeko64:
   case LibFunc::ftello64:
-    return (NumParams == 0 && FTy.getParamType(0)->isPointerTy());
+    return (NumParams == 3 && FTy.getParamType(0)->isPointerTy());
   case LibFunc::tmpfile64:
     return (FTy.getReturnType()->isPointerTy());
   case LibFunc::fstat64:


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D20543.58157.patch
Type: text/x-patch
Size: 2868 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20160523/400281d4/attachment-0001.bin>