[clang] [Clang][Sema] fix crash in codegen stage when an lambda expression declared in an unevaluated context (PR #80802)

[Qizhi Hu via cfe-commits]

https://github.com/jcsxky updated https://github.com/llvm/llvm-project/pull/80802

>From 1706840eccdabcd3d5cc8bd7a80c9db96d6230c3 Mon Sep 17 00:00:00 2001
From: huqizhi <huqizhi at feysh.com>
Date: Tue, 6 Feb 2024 14:06:40 +0800
Subject: [PATCH] [Clang][Sema] fix crash in codegen stage when an lambda
 expression declared in an unevaluated context

---
 clang/docs/ReleaseNotes.rst                |  2 ++
 clang/lib/Sema/SemaTemplateInstantiate.cpp | 28 ++++++++++++++++++++--
 clang/test/CodeGen/PR76674.cpp             | 11 +++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/CodeGen/PR76674.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4d57ea4fd55b8c..7ad575e4f57fa6 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -204,6 +204,8 @@ Bug Fixes to C++ Support
   parameter where we did an incorrect specialization of the initialization of
   the default parameter.
   Fixes (`#68490 <https://github.com/llvm/llvm-project/issues/68490>`_)
+- Fix a crash in codegen when lambdas declared in an unevaluated context.
+  Fixes (`#76674 <https://github.com/llvm/llvm-project/issues/76674>`_)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 6d59180bc446d2..95793ae7301481 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -1614,7 +1614,19 @@ bool TemplateInstantiator::AlreadyTransformed(QualType T) {
   if (T.isNull())
     return true;
 
-  if (T->isInstantiationDependentType() || T->isVariablyModifiedType())
+  bool DependentLambdaType = false;
+  QualType DesugaredType = T.getDesugaredType(SemaRef.getASTContext());
+  CXXRecordDecl *RD = DesugaredType->getAsCXXRecordDecl();
+  if (RD && RD->isLambda()) {
+    QualType LambdaCallType = RD->getLambdaCallOperator()->getType();
+    if (LambdaCallType->isInstantiationDependentType() ||
+        LambdaCallType->isVariablyModifiedType()) {
+      DependentLambdaType = true;
+    }
+  }
+
+  if (T->isInstantiationDependentType() || T->isVariablyModifiedType() ||
+      DependentLambdaType)
     return false;
 
   getSema().MarkDeclarationsReferencedInType(Loc, T);
@@ -2683,9 +2695,21 @@ QualType Sema::SubstType(QualType T,
          "Cannot perform an instantiation without some context on the "
          "instantiation stack");
 
+  bool DependentLambdaType = false;
+  QualType DesugaredType = T.getDesugaredType(getASTContext());
+  CXXRecordDecl *RD = DesugaredType->getAsCXXRecordDecl();
+  if (RD && RD->isLambda()) {
+    QualType LambdaCallType = RD->getLambdaCallOperator()->getType();
+    if (LambdaCallType->isInstantiationDependentType() ||
+        LambdaCallType->isVariablyModifiedType()) {
+      DependentLambdaType = true;
+    }
+  }
+
   // If T is not a dependent type or a variably-modified type, there
   // is nothing to do.
-  if (!T->isInstantiationDependentType() && !T->isVariablyModifiedType())
+  if (!T->isInstantiationDependentType() && !T->isVariablyModifiedType() &&
+      !DependentLambdaType)
     return T;
 
   TemplateInstantiator Instantiator(*this, TemplateArgs, Loc, Entity);
diff --git a/clang/test/CodeGen/PR76674.cpp b/clang/test/CodeGen/PR76674.cpp
new file mode 100644
index 00000000000000..2ce931920afe4f
--- /dev/null
+++ b/clang/test/CodeGen/PR76674.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -std=c++20 -emit-llvm -o - %s
+// expected-no-diagnostics
+
+template <class>
+struct A {
+    template <class U>
+    using Func = decltype([] {return U{};});
+};
+
+A<int>::Func<int> f{};
+int i{f()};