[PATCH] D22929: [CodeGen][ObjC] Fix infinite recursion in getObjCEncodingForTypeImpl

[Akira Hatanaka via cfe-commits]

ahatanak created this revision.
ahatanak added reviewers: doug.gregor, akyrtzi.
ahatanak added a subscriber: cfe-commits.

This patch fixes a stack overflow bug in ASTContext::getObjCEncodingForTypeImpl where it keeps expanding a class recursively. I added a check to avoid expanding a class if ExpandStructures is false.

https://reviews.llvm.org/D22929

Files:
  lib/AST/ASTContext.cpp
  test/CodeGenObjCXX/encode.mm

Index: test/CodeGenObjCXX/encode.mm
===================================================================
--- test/CodeGenObjCXX/encode.mm
+++ test/CodeGenObjCXX/encode.mm
@@ -224,3 +224,24 @@
   // CHECK: @_ZN7PR171421xE = constant [14 x i8] c"{E=^^?i^^?ii}\00"
   extern const char x[] = @encode(E);
 }
+
+// This test used to cause infinite recursion.
+template<typename T>
+struct S {
+  typedef T Ty;
+  Ty *t;
+};
+
+ at interface N
+{
+  S<N> a;
+}
+ at end
+
+ at implementation N
+ at end
+
+const char *expand_struct() {
+  // CHECK: @{{.*}} = private unnamed_addr constant [16 x i8] c"{N={S<N>=^{N}}}\00"
+  return @encode(N);
+}
Index: lib/AST/ASTContext.cpp
===================================================================
--- lib/AST/ASTContext.cpp
+++ lib/AST/ASTContext.cpp
@@ -5896,18 +5896,20 @@
     ObjCInterfaceDecl *OI = T->castAs<ObjCObjectType>()->getInterface();
     S += '{';
     S += OI->getObjCRuntimeNameAsString();
-    S += '=';
-    SmallVector<const ObjCIvarDecl*, 32> Ivars;
-    DeepCollectObjCIvars(OI, true, Ivars);
-    for (unsigned i = 0, e = Ivars.size(); i != e; ++i) {
-      const FieldDecl *Field = cast<FieldDecl>(Ivars[i]);
-      if (Field->isBitField())
-        getObjCEncodingForTypeImpl(Field->getType(), S, false, true, Field);
-      else
-        getObjCEncodingForTypeImpl(Field->getType(), S, false, true, FD,
-                                   false, false, false, false, false,
-                                   EncodePointerToObjCTypedef,
-                                   NotEncodedT);
+    if (ExpandStructures) {
+      S += '=';
+      SmallVector<const ObjCIvarDecl*, 32> Ivars;
+      DeepCollectObjCIvars(OI, true, Ivars);
+      for (unsigned i = 0, e = Ivars.size(); i != e; ++i) {
+        const FieldDecl *Field = cast<FieldDecl>(Ivars[i]);
+        if (Field->isBitField())
+          getObjCEncodingForTypeImpl(Field->getType(), S, false, true, Field);
+        else
+          getObjCEncodingForTypeImpl(Field->getType(), S, false, true, FD,
+                                     false, false, false, false, false,
+                                     EncodePointerToObjCTypedef,
+                                     NotEncodedT);
+      }
     }
     S += '}';
     return;


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D22929.65969.patch
Type: text/x-patch
Size: 2247 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20160728/d5dacc0d/attachment.bin>