[clang] [Clang][objectsize] Generate object size calculation for sub-objects (PR #86858)

[Eli Friedman via cfe-commits]

================
@@ -1062,6 +1063,160 @@ CodeGenFunction::emitFlexibleArrayMemberSize(const Expr *E, unsigned Type,
   return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
 }
 
+namespace {
+
+class ObjectSizeVisitor
+    : public ConstStmtVisitor<ObjectSizeVisitor, const Expr *> {
+  ASTContext &Ctx;
+  bool SkipASE;
+
+public:
+  ObjectSizeVisitor(ASTContext &Ctx, bool SkipASE = false)
+      : Ctx(Ctx), SkipASE(SkipASE) {}
+
+  const Expr *Visit(const Expr *E) {
+    return ConstStmtVisitor<ObjectSizeVisitor, const Expr *>::Visit(E);
+  }
+
+  const Expr *VisitStmt(const Stmt *S) { return nullptr; }
+
+  const Expr *VisitDeclRefExpr(const DeclRefExpr *E) { return E; }
+  const Expr *VisitMemberExpr(const MemberExpr *E) { return E; }
+  const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
+    return SkipASE ? Visit(E->getBase()->IgnoreParenImpCasts()) : E;
+  }
+
+  const Expr *VisitCastExpr(const CastExpr *E) {
+    const Expr *NoopE = E->IgnoreParenNoopCasts(Ctx);
+    return NoopE != E ? Visit(NoopE->IgnoreParenImpCasts()) : nullptr;
+  }
+  const Expr *VisitUnaryAddrOf(const clang::UnaryOperator *E) {
+    return Visit(E->getSubExpr()->IgnoreParenImpCasts());
+  }
+};
+
+} // end anonymous namespace
+
+/// getLastDecl - Return the last FieldDecl in the struct.
+static const FieldDecl *getLastDecl(const RecordDecl *RD) {
+  const FieldDecl *LastFieldDecl = nullptr;
+  for (const FieldDecl *FD : RD->fields())
+    LastFieldDecl = FD;
+
+  if (LastFieldDecl) {
+    QualType Ty = LastFieldDecl->getType();
+    if (Ty->isPointerType())
+      Ty = Ty->getPointeeType();
+
+    if (const RecordDecl *Rec = Ty->getAsRecordDecl())
+      // The last FieldDecl is a structure. Look into that struct to find its
+      // last FieldDecl.
+      LastFieldDecl = getLastDecl(Rec);
+  }
+
+  return LastFieldDecl;
+}
+
+/// tryToCalculateSubObjectSize - It may be possible to calculate the
+/// sub-object size of an array and skip the generation of the llvm.objectsize
+/// intrinsic. This avoids the complication in conveying the sub-object's
+/// information to the backend. This calculation works for an N-dimentional
+/// array.
+llvm::Value *
+CodeGenFunction::tryToCalculateSubObjectSize(const Expr *E, unsigned Type,
+                                             llvm::IntegerType *ResType) {
+  if ((Type & 0x01) != 1)
+    // Only support sub-object calculation.
+    return nullptr;
+
+  ASTContext &Ctx = getContext();
+  const Expr *ObjectRef = ObjectSizeVisitor(Ctx).Visit(E);
+  if (!ObjectRef)
+    return nullptr;
+
+  QualType ObjectRefType = ObjectRef->getType();
+  if (ObjectRefType->isPointerType())
+    ObjectRefType = ObjectRefType->getPointeeType();
+
+  // Collect the base and index from the array.
+  QualType ObjectBaseRefTy;
+  const Expr *ArrayIdx = nullptr;
+
+  if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(ObjectRef)) {
+    ArrayIdx = ASE->getIdx()->IgnoreParenImpCasts();
+
+    const Expr *ArrayRefBase = ASE->getBase()->IgnoreParenImpCasts();
+    if (isa<ArraySubscriptExpr>(ArrayRefBase)) {
+      ObjectBaseRefTy = ArrayRefBase->getType();
+      if (ObjectBaseRefTy->isPointerType())
+        ObjectBaseRefTy = ObjectBaseRefTy->getPointeeType();
+    }
+  }
+
+  if (!ArrayIdx || ArrayIdx->HasSideEffects(Ctx))
+    return nullptr;
+
+  // Check to see if the Decl is a flexible array member. Processing of the
+  // 'counted_by' attribute is done by now. So we don't have any information on
+  // its size, so return MAX_INT.
+  //
+  // Rerun the visitor to find the base expr: MemberExpr or DeclRefExpr.
+  ObjectRef = ObjectSizeVisitor(Ctx, true).Visit(ObjectRef);
+  if (!ObjectRef)
+    return nullptr;
+
+  if (const auto *ME = dyn_cast<MemberExpr>(ObjectRef)) {
+    if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
+      const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
+          getLangOpts().getStrictFlexArraysLevel();
+      const RecordDecl *OuterRD =
+          FD->getParent()->getOuterLexicalRecordContext();
+
+      if (getLastDecl(OuterRD) == FD &&
+          Decl::isFlexibleArrayMemberLike(
+              Ctx, FD, FD->getType(), StrictFlexArraysLevel,
+              /*IgnoreTemplateOrMacroSubstitution=*/true))
+        return ConstantInt::get(ResType, -1, /*isSigned=*/true);
+    }
+  }
+
+  if (ObjectBaseRefTy.isNull()) {
+    ObjectBaseRefTy = ObjectRef->getType();
+    if (ObjectBaseRefTy->isPointerType())
+      ObjectBaseRefTy = ObjectBaseRefTy->getPointeeType();
+  }
+
+  // Generate the calculation:
+  //
+  //     S Object[n_1][n_2]...[n_m]; /* M-dimentional array */
+  //
+  //     ObjectRef = Object[n_1]...[n_x]; /* 0 < x < m */
+  //     ObjectBaseRef = Object[n_1]...[n_{x-1}];
+  //
+  //     ArrayRefSize = sizeof( typeof( ObjectRef ) );
+  //     ArrayRefBaseSize = sizeof( typeof( ObjectBaseRef ) );
+  //
+  //     Size = ArrayRefSize - (ArrayRefBaseSize * ArrayIdx);
+  //     return Size > 0 ? Size : 0;
+  //
+  Value *ArrayRefSize = ConstantInt::get(
+      ResType, Ctx.getTypeSizeInChars(ObjectRefType).getQuantity(),
+      /*isSigned=*/true);
+  Value *ArrayRefBaseSize = ConstantInt::get(
+      ResType, Ctx.getTypeSizeInChars(ObjectBaseRefTy).getQuantity(),
+      /*isSigned=*/true);
+
+  Value *Res = EmitScalarExpr(ArrayIdx);
----------------
efriedma-quic wrote:

Do we need to check whether ArrayIdx has side-effects?  (If EmittedE is non-null, we don't check for side-effects otherwise.)

https://github.com/llvm/llvm-project/pull/86858