[cfe-commits] r167801 - in /cfe/trunk: include/clang/Basic/TokenKinds.def include/clang/Lex/PreprocessorLexer.h lib/Lex/Lexer.cpp lib/Lex/PreprocessorLexer.cpp test/Index/Inputs/empty.h test/Index/pp-many-includes.c tools/libclang/CIndex.cpp

Argyrios Kyrtzidis akyrtzi at gmail.com
Mon Nov 12 17:03:15 PST 2012


Author: akirtzidis
Date: Mon Nov 12 19:03:15 2012
New Revision: 167801

URL: http://llvm.org/viewvc/llvm-project?rev=167801&view=rev
Log:
[preprocessor] When #including something that contributes no tokens at all,
don't recursively continue lexing.

This avoids a stack overflow with a sequence of many empty #includes.
rdar://11988695

Added:
    cfe/trunk/test/Index/Inputs/empty.h
    cfe/trunk/test/Index/pp-many-includes.c
Modified:
    cfe/trunk/include/clang/Basic/TokenKinds.def
    cfe/trunk/include/clang/Lex/PreprocessorLexer.h
    cfe/trunk/lib/Lex/Lexer.cpp
    cfe/trunk/lib/Lex/PreprocessorLexer.cpp
    cfe/trunk/tools/libclang/CIndex.cpp

Modified: cfe/trunk/include/clang/Basic/TokenKinds.def
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/TokenKinds.def?rev=167801&r1=167800&r2=167801&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/TokenKinds.def (original)
+++ cfe/trunk/include/clang/Basic/TokenKinds.def Mon Nov 12 19:03:15 2012
@@ -103,6 +103,8 @@
 TOK(eof)                 // End of file.
 TOK(eod)                 // End of preprocessing directive (end of line inside a
                          // directive).
+TOK(included_eof)        // End of included file, only if
+                         // PreprocessorLexer::EnableIncludedEOF is true.
 TOK(code_completion)     // Code completion marker
 TOK(cxx_defaultarg_end)  // C++ default argument end marker
 

Modified: cfe/trunk/include/clang/Lex/PreprocessorLexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessorLexer.h?rev=167801&r1=167800&r2=167801&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/PreprocessorLexer.h (original)
+++ cfe/trunk/include/clang/Lex/PreprocessorLexer.h Mon Nov 12 19:03:15 2012
@@ -61,6 +61,10 @@
   /// Note that in raw mode that the PP pointer may be null.
   bool LexingRawMode;
 
+  /// \brief When true, if EOF of the current lexer is found, tok::included_eof
+  /// is returned instead of continuing lexing higher in the include stack.
+  bool EnableIncludedEOF;
+
   /// \brief A state machine that detects the \#ifndef-wrapping a file
   /// idiom for the multiple-include optimization.
   MultipleIncludeOpt MIOpt;
@@ -79,7 +83,8 @@
     : PP(0), InitialNumSLocEntries(0),
       ParsingPreprocessorDirective(false),
       ParsingFilename(false),
-      LexingRawMode(false) {}
+      LexingRawMode(false),
+      EnableIncludedEOF(false) {}
 
   virtual ~PreprocessorLexer() {}
 
@@ -147,6 +152,11 @@
   /// \brief Return true if this lexer is in raw mode or not.
   bool isLexingRawMode() const { return LexingRawMode; }
 
+  /// \brief When true, if EOF of the current lexer is found, tok::included_eof
+  /// is returned instead of continuing lexing higher in the include stack.
+  /// False is the default behavior.
+  void setEnableIncludedEOF(bool Enable) { EnableIncludedEOF = Enable; }
+
   /// \brief Return the preprocessor object for this lexer.
   Preprocessor *getPP() const { return PP; }
 

Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=167801&r1=167800&r2=167801&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Mon Nov 12 19:03:15 2012
@@ -2595,8 +2595,14 @@
       // Read the PP instance variable into an automatic variable, because
       // LexEndOfFile will often delete 'this'.
       Preprocessor *PPCache = PP;
+      bool EnableIncludedEOFCache = EnableIncludedEOF;
       if (LexEndOfFile(Result, CurPtr-1))  // Retreat back into the file.
         return;   // Got a token to return.
+
+      if (EnableIncludedEOFCache) {
+        Result.setKind(tok::included_eof);
+        return;
+      }
       assert(PPCache && "Raw buffer::LexEndOfFile should return a token");
       return PPCache->Lex(Result);
     }
@@ -3234,5 +3240,21 @@
     }
     goto LexNextToken;   // GCC isn't tail call eliminating.
   }
+
+  if (PreprocessorLexer *PPLex = PP->getCurrentLexer()) {
+    // If we #include something that contributes no tokens at all, return with
+    // a tok::included_eof instead of recursively continuing lexing.
+    // This avoids a stack overflow with a sequence of many empty #includes.
+    PPLex->setEnableIncludedEOF(true);
+    PP->Lex(Result);
+    if (Result.isNot(tok::included_eof)) {
+      if (Result.isNot(tok::eof) && Result.isNot(tok::eod))
+        PPLex->setEnableIncludedEOF(false);
+      return;
+    }
+    if (PP->isCurrentLexer(this))
+      goto LexNextToken;
+  }
+
   return PP->Lex(Result);
 }

Modified: cfe/trunk/lib/Lex/PreprocessorLexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PreprocessorLexer.cpp?rev=167801&r1=167800&r2=167801&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PreprocessorLexer.cpp (original)
+++ cfe/trunk/lib/Lex/PreprocessorLexer.cpp Mon Nov 12 19:03:15 2012
@@ -22,7 +22,7 @@
 PreprocessorLexer::PreprocessorLexer(Preprocessor *pp, FileID fid)
   : PP(pp), FID(fid), InitialNumSLocEntries(0),
     ParsingPreprocessorDirective(false),
-    ParsingFilename(false), LexingRawMode(false) {
+    ParsingFilename(false), LexingRawMode(false), EnableIncludedEOF(false) {
   if (pp)
     InitialNumSLocEntries = pp->getSourceManager().local_sloc_entry_size();
 }

Added: cfe/trunk/test/Index/Inputs/empty.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/Inputs/empty.h?rev=167801&view=auto
==============================================================================
    (empty)

Added: cfe/trunk/test/Index/pp-many-includes.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/pp-many-includes.c?rev=167801&view=auto
==============================================================================
--- cfe/trunk/test/Index/pp-many-includes.c (added)
+++ cfe/trunk/test/Index/pp-many-includes.c Mon Nov 12 19:03:15 2012
@@ -0,0 +1,208 @@
+// RUN: env LIBCLANG_THREAD_SMALLSTACK=1 c-index-test -index-file %s | FileCheck %s
+
+// rdar://11988695
+// Check that we don't get stack overflow while including a lot of empty includes.
+
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+#include "Inputs/empty.h"
+
+// CHECK: [indexDeclaration]: kind: function | name: foo
+void foo();

Modified: cfe/trunk/tools/libclang/CIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/libclang/CIndex.cpp?rev=167801&r1=167800&r2=167801&view=diff
==============================================================================
--- cfe/trunk/tools/libclang/CIndex.cpp (original)
+++ cfe/trunk/tools/libclang/CIndex.cpp Mon Nov 12 19:03:15 2012
@@ -6119,6 +6119,8 @@
 bool RunSafely(llvm::CrashRecoveryContext &CRC,
                void (*Fn)(void*), void *UserData,
                unsigned Size) {
+  if (getenv("LIBCLANG_THREAD_SMALLSTACK"))
+    Size = 0x10000; // 64K.
   if (!Size)
     Size = GetSafetyThreadStackSize();
   if (Size)





More information about the cfe-commits mailing list