[clang] [clang][dataflow] Don't propagate result objects in nested declarations. (PR #89903)

[via cfe-commits]

https://github.com/martinboehme updated https://github.com/llvm/llvm-project/pull/89903

>From 0ed905fc227bc58f3ae8f5ead856bba7e8376e3c Mon Sep 17 00:00:00 2001
From: Martin Braenne <mboehme at google.com>
Date: Wed, 24 Apr 2024 09:41:18 +0000
Subject: [PATCH 1/2] [clang][dataflow] Don't propagate result objects in
 nested declarations.

Trying to do so can cause crashes -- see newly added test and the comments in
the fix.
---
 .../FlowSensitive/DataflowEnvironment.cpp     | 12 ++++++++++
 .../Analysis/FlowSensitive/TransferTest.cpp   | 22 +++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index 3cb656adcbdc0c..aec8ee6a4a07fc 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -333,6 +333,18 @@ class ResultObjectVisitor : public RecursiveASTVisitor<ResultObjectVisitor> {
     }
   }
 
+  bool TraverseDecl(Decl *D) {
+    // Don't traverse nested record or function declarations.
+    // - We won't be analyzing code contained in these anyway
+    // - We don't model fields that are used only in these nested declaration,
+    //   so trying to propagate a result object to initializers of such fields
+    //   would cause an error.
+    if (isa<RecordDecl>(D) || isa<FunctionDecl>(D))
+      return true;
+
+    return RecursiveASTVisitor<ResultObjectVisitor>::TraverseDecl(D);
+  }
+
   bool TraverseBindingDecl(BindingDecl *BD) {
     // `RecursiveASTVisitor` doesn't traverse holding variables for
     // `BindingDecl`s by itself, so we need to tell it to.
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 215e208615ac23..5c7c39e52612ec 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -3309,6 +3309,28 @@ TEST(TransferTest, ResultObjectLocationPropagatesThroughConditionalOperator) {
       });
 }
 
+TEST(TransferTest, ResultObjectLocationDontVisitNestedRecordDecl) {
+  // This is a crash repro.
+  // We used to crash because when propagating result objects, we would visit
+  // nested record and function declarations, but we don't model fields used
+  // only in these.
+  std::string Code = R"(
+    struct S1 {};
+    struct S2 { S1 s1; };
+    void target() {
+      struct Nested {
+        void f() {
+          S2 s2 = { S1() };
+        }
+      };
+    }
+  )";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {});
+}
+
 TEST(TransferTest, StaticCast) {
   std::string Code = R"(
     void target(int Foo) {

>From 21dead242d19077052d495b03d641854109ddcbb Mon Sep 17 00:00:00 2001
From: Martin Braenne <mboehme at google.com>
Date: Wed, 24 Apr 2024 13:07:13 +0000
Subject: [PATCH 2/2] fixup! [clang][dataflow] Don't propagate result objects
 in nested declarations.

---
 clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index aec8ee6a4a07fc..636d2302093983 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -339,7 +339,7 @@ class ResultObjectVisitor : public RecursiveASTVisitor<ResultObjectVisitor> {
     // - We don't model fields that are used only in these nested declaration,
     //   so trying to propagate a result object to initializers of such fields
     //   would cause an error.
-    if (isa<RecordDecl>(D) || isa<FunctionDecl>(D))
+    if (isa_and_nonnull<RecordDecl>(D) || isa_and_nonnull<FunctionDecl>(D))
       return true;
 
     return RecursiveASTVisitor<ResultObjectVisitor>::TraverseDecl(D);