[clang] [clang][bytecode] Check allocation size limit for operator new (PR #109590)

[Timm Baeder via cfe-commits]

https://github.com/tbaederr created https://github.com/llvm/llvm-project/pull/109590

None

>From 6b37086bbaa01e9daa069771a3e54c6b80bd7e5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder at redhat.com>
Date: Sun, 22 Sep 2024 22:33:05 +0200
Subject: [PATCH] [clang][bytecode] Check allocation size limit for operator
 new

---
 clang/lib/AST/ByteCode/InterpBuiltin.cpp | 11 +++++++++-
 clang/lib/AST/ByteCode/InterpFrame.cpp   | 16 ++++++++++++--
 clang/test/AST/ByteCode/new-delete.cpp   | 27 +++++++++++++++++++++++-
 3 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 51c77b7da1a655..523f5cb993dbc7 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -1306,7 +1306,16 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
     return false;
   }
 
-  // FIXME: CheckArraySize for NumElems?
+  // NB: The same check we're using in CheckArraySize()
+  if (NumElems.getActiveBits() >
+          ConstantArrayType::getMaxSizeBits(S.getASTContext()) ||
+      NumElems.ugt(Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
+    // FIXME: NoThrow check?
+    const SourceInfo &Loc = S.Current->getSource(OpPC);
+    S.FFDiag(Loc, diag::note_constexpr_new_too_large)
+        << NumElems.getZExtValue();
+    return false;
+  }
 
   std::optional<PrimType> ElemT = S.getContext().classify(ElemType);
   DynamicAllocator &Allocator = S.getAllocator();
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index 28e189bb339e62..7c877a70fe6b97 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -102,14 +102,26 @@ static void print(llvm::raw_ostream &OS, const T &V, ASTContext &ASTCtx,
   V.toAPValue(ASTCtx).printPretty(OS, ASTCtx, Ty);
 }
 
+static bool shouldSkipInBacktrace(const Function *F) {
+  if (F->isBuiltin())
+    return true;
+  if (F->isLambdaStaticInvoker())
+    return true;
+
+  const FunctionDecl *FD = F->getDecl();
+  if (FD->getDeclName().getCXXOverloadedOperator() == OO_New ||
+      FD->getDeclName().getCXXOverloadedOperator() == OO_Array_New)
+    return true;
+  return false;
+}
+
 void InterpFrame::describe(llvm::raw_ostream &OS) const {
   // We create frames for builtin functions as well, but we can't reliably
   // diagnose them. The 'in call to' diagnostics for them add no value to the
   // user _and_ it doesn't generally work since the argument types don't always
   // match the function prototype. Just ignore them.
   // Similarly, for lambda static invokers, we would just print __invoke().
-  if (const auto *F = getFunction();
-      F && (F->isBuiltin() || F->isLambdaStaticInvoker()))
+  if (const auto *F = getFunction(); F && shouldSkipInBacktrace(F))
     return;
 
   const Expr *CallExpr = Caller->getExpr(getRetPC());
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index 2ba1286b250dc6..6cefbba307215a 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -592,7 +592,8 @@ namespace std {
   using size_t = decltype(sizeof(0));
   template<typename T> struct allocator {
     constexpr T *allocate(size_t N) {
-      return (T*)__builtin_operator_new(sizeof(T) * N); // both-note 2{{allocation performed here}}
+      return (T*)__builtin_operator_new(sizeof(T) * N); // both-note 2{{allocation performed here}} \
+                                                        // #alloc
     }
     constexpr void deallocate(void *p) {
       __builtin_operator_delete(p); // both-note 2{{std::allocator<...>::deallocate' used to delete pointer to object allocated with 'new'}} \
@@ -731,6 +732,30 @@ namespace Limits {
     return n;
   }
   static_assert(dynarray<char>(5, 0) == 'f');
+
+
+#if __LP64__
+  template <typename T>
+  struct S {
+      constexpr S(unsigned long long N)
+      : data(nullptr){
+          data = alloc.allocate(N); // both-note {{in call to 'this->alloc.allocate(18446744073709551615)}}
+      }
+      constexpr T operator[](std::size_t i) const {
+        return data[i];
+      }
+
+      constexpr ~S() {
+          alloc.deallocate(data);
+      }
+      std::allocator<T> alloc;
+      T* data;
+  };
+
+  constexpr std::size_t s = S<std::size_t>(~0UL)[42]; // both-error {{constexpr variable 's' must be initialized by a constant expression}} \
+                                                      // both-note@#alloc {{cannot allocate array; evaluated array bound 2305843009213693951 is too large}} \
+                                                      // both-note {{in call to}}
+#endif
 }
 
 #else