[clang-tools-extra] c9b3250 - [clangd] Allow to build Clangd without decision forest
Ilya Biryukov via cfe-commits
cfe-commits at lists.llvm.org
Wed Dec 7 04:52:34 PST 2022
Author: Ilya Biryukov
Date: 2022-12-07T13:52:22+01:00
New Revision: c9b325088d14a816b101857eecd1042335aad4e0
URL: https://github.com/llvm/llvm-project/commit/c9b325088d14a816b101857eecd1042335aad4e0
DIFF: https://github.com/llvm/llvm-project/commit/c9b325088d14a816b101857eecd1042335aad4e0.diff
LOG: [clangd] Allow to build Clangd without decision forest
Make it possible to disable building the decision forest ranking
model for clangd. To unbreak build of Clangd on PPC32 in gentoo, see
https://bugs.gentoo.org/829602
Based on D138520.
Reviewed By: sammccall
Differential Revision: https://reviews.llvm.org/D139107
Added:
clang-tools-extra/clangd/DecisionForest.cpp
Modified:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/CodeComplete.cpp
clang-tools-extra/clangd/CodeComplete.h
clang-tools-extra/clangd/Feature.cpp
clang-tools-extra/clangd/Features.inc.in
clang-tools-extra/clangd/Quality.cpp
clang-tools-extra/clangd/benchmarks/CMakeLists.txt
clang-tools-extra/clangd/tool/ClangdMain.cpp
clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
Removed:
################################################################################
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index dec115a64a59e..00d791807ce40 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -18,6 +18,8 @@ if (NOT DEFINED CLANGD_BUILD_XPC)
unset(CLANGD_BUILD_XPC_DEFAULT)
endif ()
+# This involves generating and compiling large source files, which can run into toolchain limitations.
+option(CLANGD_DECISION_FOREST "Enable decision forest model for ranking code completion items" ON)
option(CLANGD_MALLOC_TRIM "Call malloc_trim(3) periodically in Clangd. (only takes effect when using glibc)" ON)
# -DCLANG_TIDY_CHECKS=Off avoids a dependency on clang-tidy, reducing rebuilds.
option(CLANGD_TIDY_CHECKS "Link all clang-tidy checks into clangd" ON)
@@ -29,6 +31,7 @@ llvm_canonicalize_cmake_booleans(
CLANGD_MALLOC_TRIM
CLANGD_TIDY_CHECKS
LLVM_ENABLE_ZLIB
+ CLANGD_DECISION_FOREST
)
configure_file(
@@ -43,8 +46,12 @@ set(LLVM_LINK_COMPONENTS
Option
)
-include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake)
-gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example)
+set(COMPLETIONMODEL_SOURCES)
+if(CLANGD_DECISION_FOREST)
+ include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake)
+ gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example)
+ list(APPEND COMPLETIONMODEL_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/CompletionModel.cpp)
+endif()
if(MSVC AND NOT CLANG_CL)
set_source_files_properties(CompileCommands.cpp PROPERTIES COMPILE_FLAGS -wd4130) # disables C4130: logical operation on address of string constant
@@ -66,6 +73,7 @@ add_clang_library(clangDaemon
ConfigCompile.cpp
ConfigProvider.cpp
ConfigYAML.cpp
+ DecisionForest.cpp
Diagnostics.cpp
DraftStore.cpp
DumpAST.cpp
@@ -102,7 +110,7 @@ add_clang_library(clangDaemon
TUScheduler.cpp
URI.cpp
XRefs.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/CompletionModel.cpp
+ ${COMPLETIONMODEL_SOURCES}
index/Background.cpp
index/BackgroundIndexLoader.cpp
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index ad78782187099..9015be6cf7c89 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -22,6 +22,7 @@
#include "CodeCompletionStrings.h"
#include "Compiler.h"
#include "ExpectedTypes.h"
+#include "Feature.h"
#include "FileDistance.h"
#include "FuzzyMatch.h"
#include "Headers.h"
@@ -76,6 +77,16 @@
namespace clang {
namespace clangd {
+
+#if CLANGD_DECISION_FOREST
+const CodeCompleteOptions::CodeCompletionRankingModel
+ CodeCompleteOptions::DefaultRankingModel =
+ CodeCompleteOptions::DecisionForest;
+#else
+const CodeCompleteOptions::CodeCompletionRankingModel
+ CodeCompleteOptions::DefaultRankingModel = CodeCompleteOptions::Heuristics;
+#endif
+
namespace {
CompletionItemKind toCompletionItemKind(index::SymbolKind Kind) {
diff --git a/clang-tools-extra/clangd/CodeComplete.h b/clang-tools-extra/clangd/CodeComplete.h
index 19ef4c17d3b0f..b661eac5987f6 100644
--- a/clang-tools-extra/clangd/CodeComplete.h
+++ b/clang-tools-extra/clangd/CodeComplete.h
@@ -130,7 +130,9 @@ struct CodeCompleteOptions {
enum CodeCompletionRankingModel {
Heuristics,
DecisionForest,
- } RankingModel = DecisionForest;
+ };
+ static const CodeCompletionRankingModel DefaultRankingModel;
+ CodeCompletionRankingModel RankingModel = DefaultRankingModel;
/// Callback used to score a CompletionCandidate if DecisionForest ranking
/// model is enabled.
diff --git a/clang-tools-extra/clangd/DecisionForest.cpp b/clang-tools-extra/clangd/DecisionForest.cpp
new file mode 100644
index 0000000000000..0b54a6c65beb0
--- /dev/null
+++ b/clang-tools-extra/clangd/DecisionForest.cpp
@@ -0,0 +1,98 @@
+//===--- DecisionForest.cpp --------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Features.inc"
+
+#if !CLANGD_DECISION_FOREST
+#include "Quality.h"
+#include <cstdlib>
+
+namespace clang {
+namespace clangd {
+DecisionForestScores
+evaluateDecisionForest(const SymbolQualitySignals &Quality,
+ const SymbolRelevanceSignals &Relevance, float Base) {
+ llvm::errs() << "Clangd was compiled without decision forest support.\n";
+ std::abort();
+}
+
+} // namespace clangd
+} // namespace clang
+
+#else // !CLANGD_DECISION_FOREST
+
+#include "CompletionModel.h"
+#include "Quality.h"
+#include <cmath>
+
+namespace clang {
+namespace clangd {
+
+DecisionForestScores
+evaluateDecisionForest(const SymbolQualitySignals &Quality,
+ const SymbolRelevanceSignals &Relevance, float Base) {
+ Example E;
+ E.setIsDeprecated(Quality.Deprecated);
+ E.setIsReservedName(Quality.ReservedName);
+ E.setIsImplementationDetail(Quality.ImplementationDetail);
+ E.setNumReferences(Quality.References);
+ E.setSymbolCategory(Quality.Category);
+
+ SymbolRelevanceSignals::DerivedSignals Derived =
+ Relevance.calculateDerivedSignals();
+ int NumMatch = 0;
+ if (Relevance.ContextWords) {
+ for (const auto &Word : Relevance.ContextWords->keys()) {
+ if (Relevance.Name.contains_insensitive(Word)) {
+ ++NumMatch;
+ }
+ }
+ }
+ E.setIsNameInContext(NumMatch > 0);
+ E.setNumNameInContext(NumMatch);
+ E.setFractionNameInContext(
+ Relevance.ContextWords && !Relevance.ContextWords->empty()
+ ? NumMatch * 1.0 / Relevance.ContextWords->size()
+ : 0);
+ E.setIsInBaseClass(Relevance.InBaseClass);
+ E.setFileProximityDistanceCost(Derived.FileProximityDistance);
+ E.setSemaFileProximityScore(Relevance.SemaFileProximityScore);
+ E.setSymbolScopeDistanceCost(Derived.ScopeProximityDistance);
+ E.setSemaSaysInScope(Relevance.SemaSaysInScope);
+ E.setScope(Relevance.Scope);
+ E.setContextKind(Relevance.Context);
+ E.setIsInstanceMember(Relevance.IsInstanceMember);
+ E.setHadContextType(Relevance.HadContextType);
+ E.setHadSymbolType(Relevance.HadSymbolType);
+ E.setTypeMatchesPreferred(Relevance.TypeMatchesPreferred);
+
+ DecisionForestScores Scores;
+ // Exponentiating DecisionForest prediction makes the score of each tree a
+ // multiplciative boost (like NameMatch). This allows us to weigh the
+ // prediction score and NameMatch appropriately.
+ Scores.ExcludingName = pow(Base, Evaluate(E));
+ // Following cases are not part of the generated training dataset:
+ // - Symbols with `NeedsFixIts`.
+ // - Forbidden symbols.
+ // - Keywords: Dataset contains only macros and decls.
+ if (Relevance.NeedsFixIts)
+ Scores.ExcludingName *= 0.5;
+ if (Relevance.Forbidden)
+ Scores.ExcludingName *= 0;
+ if (Quality.Category == SymbolQualitySignals::Keyword)
+ Scores.ExcludingName *= 4;
+
+ // NameMatch should be a multiplier on total score to support rescoring.
+ Scores.Total = Relevance.NameMatch * Scores.ExcludingName;
+ return Scores;
+}
+
+} // namespace clangd
+} // namespace clang
+
+#endif // !CLANGD_DECISION_FOREST
\ No newline at end of file
diff --git a/clang-tools-extra/clangd/Feature.cpp b/clang-tools-extra/clangd/Feature.cpp
index 5664a8869bb00..8ff72949bf6fb 100644
--- a/clang-tools-extra/clangd/Feature.cpp
+++ b/clang-tools-extra/clangd/Feature.cpp
@@ -66,6 +66,10 @@ std::string featureString() {
#if !CLANGD_TIDY_CHECKS
"-tidy"
#endif
+
+#if !CLANGD_DECISION_FOREST
+ "-decision_forest"
+#endif
;
}
diff --git a/clang-tools-extra/clangd/Features.inc.in b/clang-tools-extra/clangd/Features.inc.in
index b6c3729b0bb8d..764fee7850d49 100644
--- a/clang-tools-extra/clangd/Features.inc.in
+++ b/clang-tools-extra/clangd/Features.inc.in
@@ -4,3 +4,4 @@
#define ENABLE_GRPC_REFLECTION @ENABLE_GRPC_REFLECTION@
#define CLANGD_MALLOC_TRIM @CLANGD_MALLOC_TRIM@
#define CLANGD_TIDY_CHECKS @CLANGD_TIDY_CHECKS@
+#define CLANGD_DECISION_FOREST @CLANGD_DECISION_FOREST@
diff --git a/clang-tools-extra/clangd/Quality.cpp b/clang-tools-extra/clangd/Quality.cpp
index a167fc38d0add..70a85d73a4823 100644
--- a/clang-tools-extra/clangd/Quality.cpp
+++ b/clang-tools-extra/clangd/Quality.cpp
@@ -9,7 +9,6 @@
#include "Quality.h"
#include "AST.h"
#include "ASTSignals.h"
-#include "CompletionModel.h"
#include "FileDistance.h"
#include "SourceCode.h"
#include "index/Symbol.h"
@@ -529,65 +528,6 @@ float evaluateSymbolAndRelevance(float SymbolQuality, float SymbolRelevance) {
return SymbolQuality * SymbolRelevance;
}
-DecisionForestScores
-evaluateDecisionForest(const SymbolQualitySignals &Quality,
- const SymbolRelevanceSignals &Relevance, float Base) {
- Example E;
- E.setIsDeprecated(Quality.Deprecated);
- E.setIsReservedName(Quality.ReservedName);
- E.setIsImplementationDetail(Quality.ImplementationDetail);
- E.setNumReferences(Quality.References);
- E.setSymbolCategory(Quality.Category);
-
- SymbolRelevanceSignals::DerivedSignals Derived =
- Relevance.calculateDerivedSignals();
- int NumMatch = 0;
- if (Relevance.ContextWords) {
- for (const auto &Word : Relevance.ContextWords->keys()) {
- if (Relevance.Name.contains_insensitive(Word)) {
- ++NumMatch;
- }
- }
- }
- E.setIsNameInContext(NumMatch > 0);
- E.setNumNameInContext(NumMatch);
- E.setFractionNameInContext(
- Relevance.ContextWords && !Relevance.ContextWords->empty()
- ? NumMatch * 1.0 / Relevance.ContextWords->size()
- : 0);
- E.setIsInBaseClass(Relevance.InBaseClass);
- E.setFileProximityDistanceCost(Derived.FileProximityDistance);
- E.setSemaFileProximityScore(Relevance.SemaFileProximityScore);
- E.setSymbolScopeDistanceCost(Derived.ScopeProximityDistance);
- E.setSemaSaysInScope(Relevance.SemaSaysInScope);
- E.setScope(Relevance.Scope);
- E.setContextKind(Relevance.Context);
- E.setIsInstanceMember(Relevance.IsInstanceMember);
- E.setHadContextType(Relevance.HadContextType);
- E.setHadSymbolType(Relevance.HadSymbolType);
- E.setTypeMatchesPreferred(Relevance.TypeMatchesPreferred);
-
- DecisionForestScores Scores;
- // Exponentiating DecisionForest prediction makes the score of each tree a
- // multiplciative boost (like NameMatch). This allows us to weigh the
- // prediction score and NameMatch appropriately.
- Scores.ExcludingName = pow(Base, Evaluate(E));
- // Following cases are not part of the generated training dataset:
- // - Symbols with `NeedsFixIts`.
- // - Forbidden symbols.
- // - Keywords: Dataset contains only macros and decls.
- if (Relevance.NeedsFixIts)
- Scores.ExcludingName *= 0.5;
- if (Relevance.Forbidden)
- Scores.ExcludingName *= 0;
- if (Quality.Category == SymbolQualitySignals::Keyword)
- Scores.ExcludingName *= 4;
-
- // NameMatch should be a multiplier on total score to support rescoring.
- Scores.Total = Relevance.NameMatch * Scores.ExcludingName;
- return Scores;
-}
-
// Produces an integer that sorts in the same order as F.
// That is: a < b <==> encodeFloat(a) < encodeFloat(b).
static uint32_t encodeFloat(float F) {
diff --git a/clang-tools-extra/clangd/benchmarks/CMakeLists.txt b/clang-tools-extra/clangd/benchmarks/CMakeLists.txt
index b1bd26f2e5599..13ca21ad70d81 100644
--- a/clang-tools-extra/clangd/benchmarks/CMakeLists.txt
+++ b/clang-tools-extra/clangd/benchmarks/CMakeLists.txt
@@ -1,4 +1,6 @@
-add_subdirectory(CompletionModel)
+if(CLANGD_DECISION_FOREST)
+ add_subdirectory(CompletionModel)
+endif()
add_benchmark(IndexBenchmark IndexBenchmark.cpp)
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 84c8781bae9ab..3f5f53f0d1562 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -797,6 +797,13 @@ clangd accepts flags on the commandline, and in the CLANGD_FLAGS environment var
}
}
+#if !CLANGD_DECISION_FOREST
+ if (RankingModel == clangd::CodeCompleteOptions::DecisionForest) {
+ llvm::errs() << "Clangd was compiled without decision forest support.\n";
+ return 1;
+ }
+#endif
+
// Setup tracing facilities if CLANGD_TRACE is set. In practice enabling a
// trace flag in your editor's config is annoying, launching with
// `CLANGD_TRACE=trace.json vim` is easier.
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 7c1a2d39ccbff..4c788ea7efbc4 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -11,6 +11,7 @@
#include "ClangdServer.h"
#include "CodeComplete.h"
#include "Compiler.h"
+#include "Feature.h"
#include "Matchers.h"
#include "Protocol.h"
#include "Quality.h"
@@ -174,6 +175,7 @@ Symbol withReferences(int N, Symbol S) {
return S;
}
+#if CLANGD_DECISION_FOREST
TEST(DecisionForestRankingModel, NameMatchSanityTest) {
clangd::CodeCompleteOptions Opts;
Opts.RankingModel = CodeCompleteOptions::DecisionForest;
@@ -207,6 +209,7 @@ TEST(DecisionForestRankingModel, ReferencesAffectRanking) {
.Completions,
ElementsAre(named("clangA"), named("clangD")));
}
+#endif // CLANGD_DECISION_FOREST
TEST(DecisionForestRankingModel, DecisionForestScorerCallbackTest) {
clangd::CodeCompleteOptions Opts;
More information about the cfe-commits
mailing list