[llvm] 7f55557 - [Analysis] improve function signature checking for snprintf

[Sanjay Patel via llvm-commits]

Author: Sanjay Patel
Date: 2021-07-31T15:17:20-04:00
New Revision: 7f5555776513f174729a686ed01270e23462aaf7

URL: https://github.com/llvm/llvm-project/commit/7f5555776513f174729a686ed01270e23462aaf7
DIFF: https://github.com/llvm/llvm-project/commit/7f5555776513f174729a686ed01270e23462aaf7.diff

LOG: [Analysis] improve function signature checking for snprintf

The check for size_t parameter 1 was already here for snprintf_chk,
but it wasn't applied to regular snprintf. This could lead to
mismatching and eventually crashing as shown in:
https://llvm.org/PR50885

Added: 
    

Modified: 
    llvm/lib/Analysis/TargetLibraryInfo.cpp
    llvm/test/Transforms/InstCombine/simplify-libcalls.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 4a8818f2e2a88..c3a609ee4fe14 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -893,9 +893,10 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy,
            FTy.getReturnType()->isIntegerTy(32);
 
   case LibFunc_snprintf:
-    return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() &&
-            FTy.getParamType(2)->isPointerTy() &&
-            FTy.getReturnType()->isIntegerTy(32));
+    return NumParams == 3 && FTy.getParamType(0)->isPointerTy() &&
+           IsSizeTTy(FTy.getParamType(1)) &&
+           FTy.getParamType(2)->isPointerTy() &&
+           FTy.getReturnType()->isIntegerTy(32);
 
   case LibFunc_snprintf_chk:
     return NumParams == 5 && FTy.getParamType(0)->isPointerTy() &&

diff  --git a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
index 25b168515cb82..f80286a8cc7e7 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls.ll
@@ -217,6 +217,18 @@ define double @fake_ldexp_16(i16 %x) {
   ret double %z
 }
 
+; PR50885 - this would crash in ValueTracking.
+
+declare i32 @snprintf(i8*, double, i32*)
+
+define i32 @fake_snprintf(i32 %buf, double %len, i32 * %str) {
+; CHECK-LABEL: @fake_snprintf(
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @snprintf(i8* undef, double [[LEN:%.*]], i32* [[STR:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = call i32 @snprintf(i8* undef, double %len, i32* %str)
+  ret i32 %call
+}
 
 attributes #0 = { nobuiltin }
 attributes #1 = { builtin }