[clang] [clang] Reject VLAs in `__is_layout_compatible()` (PR #87737)

[Vlad Serebrennikov via cfe-commits]

https://github.com/Endilll updated https://github.com/llvm/llvm-project/pull/87737

>From ef24f642ca78d357018d6023fb3d9011f115299b Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav at gmail.com>
Date: Fri, 5 Apr 2024 06:22:35 +0300
Subject: [PATCH 1/2] [clang] Reject VLAs in `__is_layout_compatible()`

This is a follow-up to #81506. Since `__is_layout_compatible()` is a C++ intrinsic (https://github.com/llvm/llvm-project/blob/ff1e72d68d1224271801ff5192a8c14fbd3be83b/clang/include/clang/Basic/TokenKinds.def#L523), I don't think we should define how it interacts with VLA extension unless we have a compelling reason to.

Since #81506 was merged after 18 cut-off, we don't have to follow any kind of deprecation process.
---
 clang/lib/Sema/SemaExprCXX.cpp     | 3 +++
 clang/test/SemaCXX/type-traits.cpp | 6 ++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 76bb78aa8b5458..db84f181012268 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -6026,6 +6026,9 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, QualType LhsT,
     return false;
   }
   case BTT_IsLayoutCompatible: {
+    if (LhsT->isVariableArrayType() || RhsT->isVariableArrayType())
+      Self.Diag(KeyLoc, diag::err_vla_unsupported)
+          << 1 << tok::kw___is_layout_compatible;
     return Self.IsLayoutCompatible(LhsT, RhsT);
   }
     default: llvm_unreachable("not a BTT");
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index 14ec17989ec7c7..28653e82e5a16e 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -1741,8 +1741,10 @@ void is_layout_compatible(int n)
   static_assert(!__is_layout_compatible(unsigned char, signed char));
   static_assert(__is_layout_compatible(int[], int[]));
   static_assert(__is_layout_compatible(int[2], int[2]));
-  static_assert(!__is_layout_compatible(int[n], int[2])); // FIXME: VLAs should be rejected
-  static_assert(!__is_layout_compatible(int[n], int[n])); // FIXME: VLAs should be rejected
+  static_assert(!__is_layout_compatible(int[n], int[2]));
+  // expected-error at -1 {{variable length arrays are not supported for '__is_layout_compatible'}}
+  static_assert(!__is_layout_compatible(int[n], int[n]));
+  // expected-error at -1 {{variable length arrays are not supported for '__is_layout_compatible'}}
   static_assert(__is_layout_compatible(int&, int&));
   static_assert(!__is_layout_compatible(int&, char&));
   static_assert(__is_layout_compatible(void(int), void(int)));

>From a8c24fb363a6d2ee7c0dbda2f9e9d96b9451d732 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav at gmail.com>
Date: Fri, 5 Apr 2024 18:51:58 +0300
Subject: [PATCH 2/2] Improve diagnostic wording

---
 clang/include/clang/Basic/DiagnosticSemaKinds.td | 2 +-
 clang/test/SemaCXX/type-traits.cpp               | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index df57f5e6ce11ba..a1dda2d2461c31 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -165,7 +165,7 @@ def ext_vla_folded_to_constant : ExtWarn<
   "variable length array folded to constant array as an extension">,
   InGroup<GNUFoldingConstant>;
 def err_vla_unsupported : Error<
-  "variable length arrays are not supported for %select{the current target|'%1'}0">;
+  "variable length arrays are not supported %select{for the current target|in '%1'}0">;
 def err_vla_in_coroutine_unsupported : Error<
   "variable length arrays in a coroutine are not supported">;
 def note_vla_unsupported : Note<
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index 28653e82e5a16e..e99ad11666e54c 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -740,7 +740,7 @@ void is_bounded_array(int n) {
   static_assert(!__is_bounded_array(cvoid *));
 
   int t32[n];
-  (void)__is_bounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported for '__is_bounded_array'}}
+  (void)__is_bounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported in '__is_bounded_array'}}
 }
 
 void is_unbounded_array(int n) {
@@ -772,7 +772,7 @@ void is_unbounded_array(int n) {
   static_assert(!__is_unbounded_array(cvoid *));
 
   int t32[n];
-  (void)__is_unbounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported for '__is_unbounded_array'}}
+  (void)__is_unbounded_array(decltype(t32)); // expected-error{{variable length arrays are not supported in '__is_unbounded_array'}}
 }
 
 void is_referenceable() {
@@ -1742,9 +1742,9 @@ void is_layout_compatible(int n)
   static_assert(__is_layout_compatible(int[], int[]));
   static_assert(__is_layout_compatible(int[2], int[2]));
   static_assert(!__is_layout_compatible(int[n], int[2]));
-  // expected-error at -1 {{variable length arrays are not supported for '__is_layout_compatible'}}
+  // expected-error at -1 {{variable length arrays are not supported in '__is_layout_compatible'}}
   static_assert(!__is_layout_compatible(int[n], int[n]));
-  // expected-error at -1 {{variable length arrays are not supported for '__is_layout_compatible'}}
+  // expected-error at -1 {{variable length arrays are not supported in '__is_layout_compatible'}}
   static_assert(__is_layout_compatible(int&, int&));
   static_assert(!__is_layout_compatible(int&, char&));
   static_assert(__is_layout_compatible(void(int), void(int)));