r365314 - [analyzer] Add analyzer option to limit the number of imported TUs
Endre Fulop via cfe-commits
cfe-commits at lists.llvm.org
Mon Jul 8 05:37:11 PDT 2019
Author: gamesh411
Date: Mon Jul 8 05:37:10 2019
New Revision: 365314
URL: http://llvm.org/viewvc/llvm-project?rev=365314&view=rev
Log:
[analyzer] Add analyzer option to limit the number of imported TUs
Summary:
During CTU analysis of complex projects, the loaded AST-contents of
imported TUs can grow bigger than available system memory. This option
introduces a threshold on the number of TUs to be imported for a single
TU in order to prevent such cases.
Differential Revision: https://reviews.llvm.org/D59798
Added:
cfe/trunk/test/Analysis/ctu-import-threshold.c
Modified:
cfe/trunk/include/clang/CrossTU/CrossTranslationUnit.h
cfe/trunk/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
cfe/trunk/lib/CrossTU/CrossTranslationUnit.cpp
cfe/trunk/test/Analysis/analyzer-config.c
cfe/trunk/unittests/CrossTU/CrossTranslationUnitTest.cpp
Modified: cfe/trunk/include/clang/CrossTU/CrossTranslationUnit.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/CrossTU/CrossTranslationUnit.h?rev=365314&r1=365313&r2=365314&view=diff
==============================================================================
--- cfe/trunk/include/clang/CrossTU/CrossTranslationUnit.h (original)
+++ cfe/trunk/include/clang/CrossTU/CrossTranslationUnit.h Mon Jul 8 05:37:10 2019
@@ -45,7 +45,8 @@ enum class index_error_code {
failed_to_generate_usr,
triple_mismatch,
lang_mismatch,
- lang_dialect_mismatch
+ lang_dialect_mismatch,
+ load_threshold_reached
};
class IndexError : public llvm::ErrorInfo<IndexError> {
@@ -134,7 +135,8 @@ public:
/// A definition with the same declaration will be looked up in the
/// index file which should be in the \p CrossTUDir directory, called
/// \p IndexName. In case the declaration is found in the index the
- /// corresponding AST file will be loaded.
+ /// corresponding AST file will be loaded. If the number of TUs imported
+ /// reaches \p CTULoadTreshold, no loading is performed.
///
/// \return Returns a pointer to the ASTUnit that contains the definition of
/// the looked up name or an Error.
@@ -182,6 +184,10 @@ private:
CompilerInstance &CI;
ASTContext &Context;
std::shared_ptr<ASTImporterSharedState> ImporterSharedSt;
+ /// \p CTULoadTreshold should serve as an upper limit to the number of TUs
+ /// imported in order to reduce the memory footprint of CTU analysis.
+ const unsigned CTULoadThreshold;
+ unsigned NumASTLoaded{0u};
};
} // namespace cross_tu
Modified: cfe/trunk/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def?rev=365314&r1=365313&r2=365314&view=diff
==============================================================================
--- cfe/trunk/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def (original)
+++ cfe/trunk/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def Mon Jul 8 05:37:10 2019
@@ -300,6 +300,14 @@ ANALYZER_OPTION(bool, ShouldTrackConditi
"Whether to place an event at each tracked condition.",
false)
+ANALYZER_OPTION(unsigned, CTUImportThreshold, "ctu-import-threshold",
+ "The maximal amount of translation units that is considered "
+ "for import when inlining functions during CTU analysis. "
+ "Lowering this threshold can alleviate the memory burder of "
+ "analysis with many interdependent definitions located in "
+ "various translation units.",
+ 100u)
+
//===----------------------------------------------------------------------===//
// Unsinged analyzer options.
//===----------------------------------------------------------------------===//
Modified: cfe/trunk/lib/CrossTU/CrossTranslationUnit.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/CrossTU/CrossTranslationUnit.cpp?rev=365314&r1=365313&r2=365314&view=diff
==============================================================================
--- cfe/trunk/lib/CrossTU/CrossTranslationUnit.cpp (original)
+++ cfe/trunk/lib/CrossTU/CrossTranslationUnit.cpp Mon Jul 8 05:37:10 2019
@@ -47,6 +47,8 @@ STATISTIC(NumNameConflicts, "The # of im
STATISTIC(NumTripleMismatch, "The # of triple mismatches");
STATISTIC(NumLangMismatch, "The # of language mismatches");
STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
+STATISTIC(NumASTLoadThresholdReached,
+ "The # of ASTs not loaded because of threshold");
// Same as Triple's equality operator, but we check a field only if that is
// known in both instances.
@@ -106,6 +108,8 @@ public:
return "Language mismatch";
case index_error_code::lang_dialect_mismatch:
return "Language dialect mismatch";
+ case index_error_code::load_threshold_reached:
+ return "Load threshold reached";
}
llvm_unreachable("Unrecognized index_error_code.");
}
@@ -184,7 +188,8 @@ template <typename T> static bool hasBod
}
CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
- : CI(CI), Context(CI.getASTContext()) {}
+ : CI(CI), Context(CI.getASTContext()),
+ CTULoadThreshold(CI.getAnalyzerOpts()->CTUImportThreshold) {}
CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
@@ -232,8 +237,8 @@ llvm::Expected<const T *> CrossTranslati
if (LookupName.empty())
return llvm::make_error<IndexError>(
index_error_code::failed_to_generate_usr);
- llvm::Expected<ASTUnit *> ASTUnitOrError =
- loadExternalAST(LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
+ llvm::Expected<ASTUnit *> ASTUnitOrError = loadExternalAST(
+ LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
if (!ASTUnitOrError)
return ASTUnitOrError.takeError();
ASTUnit *Unit = *ASTUnitOrError;
@@ -342,6 +347,13 @@ llvm::Expected<ASTUnit *> CrossTranslati
// a lookup name from a single translation unit. If multiple
// translation units contains decls with the same lookup name an
// error will be returned.
+
+ if (NumASTLoaded >= CTULoadThreshold) {
+ ++NumASTLoadThresholdReached;
+ return llvm::make_error<IndexError>(
+ index_error_code::load_threshold_reached);
+ }
+
ASTUnit *Unit = nullptr;
auto NameUnitCacheEntry = NameASTUnitMap.find(LookupName);
if (NameUnitCacheEntry == NameASTUnitMap.end()) {
@@ -379,6 +391,7 @@ llvm::Expected<ASTUnit *> CrossTranslati
ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts()));
Unit = LoadedUnit.get();
FileASTUnitMap[ASTFileName] = std::move(LoadedUnit);
+ ++NumASTLoaded;
if (DisplayCTUProgress) {
llvm::errs() << "CTU loaded AST file: "
<< ASTFileName << "\n";
Modified: cfe/trunk/test/Analysis/analyzer-config.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Analysis/analyzer-config.c?rev=365314&r1=365313&r2=365314&view=diff
==============================================================================
--- cfe/trunk/test/Analysis/analyzer-config.c (original)
+++ cfe/trunk/test/Analysis/analyzer-config.c Mon Jul 8 05:37:10 2019
@@ -27,6 +27,7 @@
// CHECK-NEXT: cplusplus.Move:WarnOn = KnownsAndLocals
// CHECK-NEXT: crosscheck-with-z3 = false
// CHECK-NEXT: ctu-dir = ""
+// CHECK-NEXT: ctu-import-threshold = 100
// CHECK-NEXT: ctu-index-name = externalDefMap.txt
// CHECK-NEXT: debug.AnalysisOrder:* = false
// CHECK-NEXT: debug.AnalysisOrder:Bind = false
@@ -90,4 +91,4 @@
// CHECK-NEXT: unroll-loops = false
// CHECK-NEXT: widen-loops = false
// CHECK-NEXT: [stats]
-// CHECK-NEXT: num-entries = 87
+// CHECK-NEXT: num-entries = 88
Added: cfe/trunk/test/Analysis/ctu-import-threshold.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Analysis/ctu-import-threshold.c?rev=365314&view=auto
==============================================================================
--- cfe/trunk/test/Analysis/ctu-import-threshold.c (added)
+++ cfe/trunk/test/Analysis/ctu-import-threshold.c Mon Jul 8 05:37:10 2019
@@ -0,0 +1,5 @@
+// Ensure analyzer option 'ctu-import-threshold' is a recognized option.
+//
+// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-threshold=30 -verify %s
+//
+// expected-no-diagnostics
Modified: cfe/trunk/unittests/CrossTU/CrossTranslationUnitTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/CrossTU/CrossTranslationUnitTest.cpp?rev=365314&r1=365313&r2=365314&view=diff
==============================================================================
--- cfe/trunk/unittests/CrossTU/CrossTranslationUnitTest.cpp (original)
+++ cfe/trunk/unittests/CrossTU/CrossTranslationUnitTest.cpp Mon Jul 8 05:37:10 2019
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "clang/CrossTU/CrossTranslationUnit.h"
+#include "clang/Frontend/CompilerInstance.h"
#include "clang/AST/ASTConsumer.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Tooling/Tooling.h"
@@ -70,12 +71,14 @@ public:
EXPECT_TRUE(llvm::sys::fs::exists(ASTFileName));
// Load the definition from the AST file.
- llvm::Expected<const FunctionDecl *> NewFDorError =
- CTU.getCrossTUDefinition(FD, "", IndexFileName);
- EXPECT_TRUE((bool)NewFDorError);
- const FunctionDecl *NewFD = *NewFDorError;
-
- *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody;
+ llvm::Expected<const FunctionDecl *> NewFDorError = handleExpected(
+ CTU.getCrossTUDefinition(FD, "", IndexFileName, false),
+ []() { return nullptr; }, [](IndexError &) {});
+
+ if (NewFDorError) {
+ const FunctionDecl *NewFD = *NewFDorError;
+ *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody;
+ }
}
private:
@@ -85,26 +88,37 @@ private:
class CTUAction : public clang::ASTFrontendAction {
public:
- CTUAction(bool *Success) : Success(Success) {}
+ CTUAction(bool *Success, unsigned OverrideLimit)
+ : Success(Success), OverrideLimit(OverrideLimit) {}
protected:
std::unique_ptr<clang::ASTConsumer>
CreateASTConsumer(clang::CompilerInstance &CI, StringRef) override {
+ CI.getAnalyzerOpts()->CTUImportThreshold = OverrideLimit;
return llvm::make_unique<CTUASTConsumer>(CI, Success);
}
private:
bool *Success;
+ const unsigned OverrideLimit;
};
} // end namespace
TEST(CrossTranslationUnit, CanLoadFunctionDefinition) {
bool Success = false;
- EXPECT_TRUE(tooling::runToolOnCode(new CTUAction(&Success), "int f(int);"));
+ EXPECT_TRUE(
+ tooling::runToolOnCode(new CTUAction(&Success, 1u), "int f(int);"));
EXPECT_TRUE(Success);
}
+TEST(CrossTranslationUnit, RespectsLoadThreshold) {
+ bool Success = false;
+ EXPECT_TRUE(
+ tooling::runToolOnCode(new CTUAction(&Success, 0u), "int f(int);"));
+ EXPECT_FALSE(Success);
+}
+
TEST(CrossTranslationUnit, IndexFormatCanBeParsed) {
llvm::StringMap<std::string> Index;
Index["a"] = "/b/f1";
More information about the cfe-commits
mailing list