[clang-tools-extra] r357102 - [clangd] Support utf-8 offsets (rather than utf-16) as a protocol extension
Sam McCall via cfe-commits
cfe-commits at lists.llvm.org
Wed Mar 27 10:47:49 PDT 2019
Author: sammccall
Date: Wed Mar 27 10:47:49 2019
New Revision: 357102
URL: http://llvm.org/viewvc/llvm-project?rev=357102&view=rev
Log:
[clangd] Support utf-8 offsets (rather than utf-16) as a protocol extension
Summary:
Still some pieces to go here: unit tests for new SourceCode functionality and
a command-line flag to force utf-8 mode. But wanted to get early feedback.
Reviewers: hokein
Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, arphaman, kadircet, jdoerfert, cfe-commits
Tags: #clang
Differential Revision: https://reviews.llvm.org/D58275
Added:
clang-tools-extra/trunk/test/clangd/utf8.test
Modified:
clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
clang-tools-extra/trunk/clangd/ClangdLSPServer.h
clang-tools-extra/trunk/clangd/Protocol.cpp
clang-tools-extra/trunk/clangd/Protocol.h
clang-tools-extra/trunk/clangd/SourceCode.cpp
clang-tools-extra/trunk/clangd/SourceCode.h
clang-tools-extra/trunk/clangd/index/IndexAction.cpp
clang-tools-extra/trunk/clangd/index/SymbolLocation.h
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
clang-tools-extra/trunk/unittests/clangd/SourceCodeTests.cpp
Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp Wed Mar 27 10:47:49 2019
@@ -13,6 +13,7 @@
#include "Trace.h"
#include "URI.h"
#include "clang/Tooling/Core/Replacement.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
@@ -93,6 +94,7 @@ public:
MessageHandler(ClangdLSPServer &Server) : Server(Server) {}
bool onNotify(llvm::StringRef Method, llvm::json::Value Params) override {
+ WithContext HandlerContext(handlerContext());
log("<-- {0}", Method);
if (Method == "exit")
return false;
@@ -109,6 +111,7 @@ public:
bool onCall(llvm::StringRef Method, llvm::json::Value Params,
llvm::json::Value ID) override {
+ WithContext HandlerContext(handlerContext());
// Calls can be canceled by the client. Add cancellation context.
WithContext WithCancel(cancelableRequestContext(ID));
trace::Span Tracer(Method);
@@ -129,6 +132,7 @@ public:
bool onReply(llvm::json::Value ID,
llvm::Expected<llvm::json::Value> Result) override {
+ WithContext HandlerContext(handlerContext());
// We ignore replies, just log them.
if (Result)
log("<-- reply({0})", ID);
@@ -259,6 +263,13 @@ private:
if (It != RequestCancelers.end())
It->second.first(); // Invoke the canceler.
}
+
+ Context handlerContext() const {
+ return Context::current().derive(
+ kCurrentOffsetEncoding,
+ Server.NegotiatedOffsetEncoding.getValueOr(OffsetEncoding::UTF16));
+ }
+
// We run cancelable requests in a context that does two things:
// - allows cancellation using RequestCancelers[ID]
// - cleans up the entry in RequestCancelers when it's no longer needed
@@ -302,6 +313,20 @@ void ClangdLSPServer::notify(llvm::Strin
void ClangdLSPServer::onInitialize(const InitializeParams &Params,
Callback<llvm::json::Value> Reply) {
+ // Determine character encoding first as it affects constructed ClangdServer.
+ if (Params.capabilities.offsetEncoding && !NegotiatedOffsetEncoding) {
+ NegotiatedOffsetEncoding = OffsetEncoding::UTF16; // fallback
+ for (OffsetEncoding Supported : *Params.capabilities.offsetEncoding)
+ if (Supported != OffsetEncoding::UnsupportedEncoding) {
+ NegotiatedOffsetEncoding = Supported;
+ break;
+ }
+ }
+ llvm::Optional<WithContextValue> WithOffsetEncoding;
+ if (NegotiatedOffsetEncoding)
+ WithOffsetEncoding.emplace(kCurrentOffsetEncoding,
+ *NegotiatedOffsetEncoding);
+
if (Params.rootUri && *Params.rootUri)
ClangdServerOpts.WorkspaceRoot = Params.rootUri->file();
else if (Params.rootPath && !Params.rootPath->empty())
@@ -331,7 +356,7 @@ void ClangdLSPServer::onInitialize(const
SupportsHierarchicalDocumentSymbol =
Params.capabilities.HierarchicalDocumentSymbol;
SupportFileStatus = Params.initializationOptions.FileStatus;
- Reply(llvm::json::Object{
+ llvm::json::Object Result{
{{"capabilities",
llvm::json::Object{
{"textDocumentSync", (int)TextDocumentSyncKind::Incremental},
@@ -369,7 +394,10 @@ void ClangdLSPServer::onInitialize(const
ExecuteCommandParams::CLANGD_APPLY_TWEAK}},
}},
{"typeHierarchyProvider", true},
- }}}});
+ }}}};
+ if (NegotiatedOffsetEncoding)
+ Result["offsetEncoding"] = *NegotiatedOffsetEncoding;
+ Reply(std::move(Result));
}
void ClangdLSPServer::onShutdown(const ShutdownParams &Params,
@@ -875,19 +903,19 @@ void ClangdLSPServer::onSymbolInfo(const
std::move(Reply));
}
-ClangdLSPServer::ClangdLSPServer(class Transport &Transp,
- const FileSystemProvider &FSProvider,
- const clangd::CodeCompleteOptions &CCOpts,
- llvm::Optional<Path> CompileCommandsDir,
- bool UseDirBasedCDB,
- const ClangdServer::Options &Opts)
+ClangdLSPServer::ClangdLSPServer(
+ class Transport &Transp, const FileSystemProvider &FSProvider,
+ const clangd::CodeCompleteOptions &CCOpts,
+ llvm::Optional<Path> CompileCommandsDir, bool UseDirBasedCDB,
+ llvm::Optional<OffsetEncoding> ForcedOffsetEncoding,
+ const ClangdServer::Options &Opts)
: Transp(Transp), MsgHandler(new MessageHandler(*this)),
FSProvider(FSProvider), CCOpts(CCOpts),
SupportedSymbolKinds(defaultSymbolKinds()),
SupportedCompletionItemKinds(defaultCompletionItemKinds()),
UseDirBasedCDB(UseDirBasedCDB),
- CompileCommandsDir(std::move(CompileCommandsDir)),
- ClangdServerOpts(Opts) {
+ CompileCommandsDir(std::move(CompileCommandsDir)), ClangdServerOpts(Opts),
+ NegotiatedOffsetEncoding(ForcedOffsetEncoding) {
// clang-format off
MsgHandler->bind("initialize", &ClangdLSPServer::onInitialize);
MsgHandler->bind("shutdown", &ClangdLSPServer::onShutdown);
Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.h?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h Wed Mar 27 10:47:49 2019
@@ -40,6 +40,7 @@ public:
ClangdLSPServer(Transport &Transp, const FileSystemProvider &FSProvider,
const clangd::CodeCompleteOptions &CCOpts,
llvm::Optional<Path> CompileCommandsDir, bool UseDirBasedCDB,
+ llvm::Optional<OffsetEncoding> ForcedOffsetEncoding,
const ClangdServer::Options &Opts);
~ClangdLSPServer();
@@ -165,6 +166,7 @@ private:
// It is destroyed before run() returns, to ensure worker threads exit.
ClangdServer::Options ClangdServerOpts;
llvm::Optional<ClangdServer> Server;
+ llvm::Optional<OffsetEncoding> NegotiatedOffsetEncoding;
};
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/Protocol.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/Protocol.cpp?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/Protocol.cpp (original)
+++ clang-tools-extra/trunk/clangd/Protocol.cpp Wed Mar 27 10:47:49 2019
@@ -16,6 +16,7 @@
#include "clang/Basic/LLVM.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringSwitch.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/JSON.h"
@@ -311,6 +312,11 @@ bool fromJSON(const llvm::json::Value &P
}
}
}
+ if (auto *OffsetEncoding = O->get("offsetEncoding")) {
+ R.offsetEncoding.emplace();
+ if (!fromJSON(*OffsetEncoding, *R.offsetEncoding))
+ return false;
+ }
return true;
}
@@ -932,5 +938,26 @@ bool fromJSON(const llvm::json::Value &P
return fromJSON(Params, Base);
}
+llvm::json::Value toJSON(const OffsetEncoding &OE) {
+ switch (OE) {
+ case OffsetEncoding::UTF8:
+ return "utf-8";
+ case OffsetEncoding::UTF16:
+ return "utf-16";
+ case OffsetEncoding::UnsupportedEncoding:
+ return "unknown";
+ }
+}
+bool fromJSON(const llvm::json::Value &V, OffsetEncoding &OE) {
+ auto Str = V.getAsString();
+ if (!Str)
+ return false;
+ OE = llvm::StringSwitch<OffsetEncoding>(*Str)
+ .Case("utf-8", OffsetEncoding::UTF8)
+ .Case("utf-16", OffsetEncoding::UTF16)
+ .Default(OffsetEncoding::UnsupportedEncoding);
+ return true;
+}
+
} // namespace clangd
} // namespace clang
Modified: clang-tools-extra/trunk/clangd/Protocol.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/Protocol.h?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/Protocol.h (original)
+++ clang-tools-extra/trunk/clangd/Protocol.h Wed Mar 27 10:47:49 2019
@@ -338,6 +338,18 @@ SymbolKind adjustKindToCapability(Symbol
// https://github.com/Microsoft/language-server-protocol/issues/344
SymbolKind indexSymbolKindToSymbolKind(index::SymbolKind Kind);
+// Determines the encoding used to measure offsets and lengths of source in LSP.
+enum class OffsetEncoding {
+ // Any string is legal on the wire. Unrecognized encodings parse as this.
+ UnsupportedEncoding,
+ // Length counts code units of UTF-16 encoded text. (Standard LSP behavior).
+ UTF16,
+ // Length counts bytes of UTF-8 encoded text. (Clangd extension).
+ UTF8,
+};
+llvm::json::Value toJSON(const OffsetEncoding &);
+bool fromJSON(const llvm::json::Value &, OffsetEncoding &);
+
// This struct doesn't mirror LSP!
// The protocol defines deeply nested structures for client capabilities.
// Instead of mapping them all, this just parses out the bits we care about.
@@ -369,6 +381,9 @@ struct ClientCapabilities {
/// Client supports CodeAction return value for textDocument/codeAction.
/// textDocument.codeAction.codeActionLiteralSupport.
bool CodeActionStructure = false;
+
+ /// Supported encodings for LSP character offsets. (clangd extension).
+ llvm::Optional<std::vector<OffsetEncoding>> offsetEncoding;
};
bool fromJSON(const llvm::json::Value &, ClientCapabilities &);
Modified: clang-tools-extra/trunk/clangd/SourceCode.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/SourceCode.cpp?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/SourceCode.cpp (original)
+++ clang-tools-extra/trunk/clangd/SourceCode.cpp Wed Mar 27 10:47:49 2019
@@ -7,7 +7,9 @@
//===----------------------------------------------------------------------===//
#include "SourceCode.h"
+#include "Context.h"
#include "Logger.h"
+#include "Protocol.h"
#include "clang/AST/ASTContext.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Lex/Lexer.h"
@@ -67,8 +69,23 @@ static size_t measureUTF16(llvm::StringR
return std::min(Result, U8.size());
}
+Key<OffsetEncoding> kCurrentOffsetEncoding;
+static bool useUTF16ForLSP() {
+ auto *Enc = Context::current().get(kCurrentOffsetEncoding);
+ switch (Enc ? *Enc : OffsetEncoding::UTF16) {
+ case OffsetEncoding::UTF16:
+ return true;
+ case OffsetEncoding::UTF8:
+ return false;
+ case OffsetEncoding::UnsupportedEncoding:
+ llvm_unreachable("cannot use an unsupported encoding");
+ }
+}
+
// Like most strings in clangd, the input is UTF-8 encoded.
size_t lspLength(llvm::StringRef Code) {
+ if (!useUTF16ForLSP())
+ return Code.size();
// A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
// Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx.
size_t Count = 0;
@@ -98,14 +115,25 @@ llvm::Expected<size_t> positionToOffset(
llvm::errc::invalid_argument);
StartOfLine = NextNL + 1;
}
+ StringRef Line =
+ Code.substr(StartOfLine).take_until([](char C) { return C == '\n'; });
- size_t NextNL = Code.find('\n', StartOfLine);
- if (NextNL == llvm::StringRef::npos)
- NextNL = Code.size();
-
+ if (!useUTF16ForLSP()) {
+ // Bounds-checking only.
+ if (P.character > int(Line.size())) {
+ if (AllowColumnsBeyondLineLength)
+ return StartOfLine + Line.size();
+ else
+ return llvm::make_error<llvm::StringError>(
+ llvm::formatv("UTF-8 offset {0} overruns line {1}", P.character,
+ P.line),
+ llvm::errc::invalid_argument);
+ }
+ return StartOfLine + P.character;
+ }
+ // P.character is in UTF-16 code units, so we have to transcode.
bool Valid;
- size_t ByteOffsetInLine = measureUTF16(
- Code.substr(StartOfLine, NextNL - StartOfLine), P.character, Valid);
+ size_t ByteOffsetInLine = measureUTF16(Line, P.character, Valid);
if (!Valid && !AllowColumnsBeyondLineLength)
return llvm::make_error<llvm::StringError>(
llvm::formatv("UTF-16 offset {0} is invalid for line {1}", P.character,
Modified: clang-tools-extra/trunk/clangd/SourceCode.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/SourceCode.h?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/SourceCode.h (original)
+++ clang-tools-extra/trunk/clangd/SourceCode.h Wed Mar 27 10:47:49 2019
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SOURCECODE_H
#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SOURCECODE_H
+#include "Context.h"
#include "Protocol.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/LangOptions.h"
@@ -34,8 +35,14 @@ using FileDigest = decltype(llvm::SHA1::
FileDigest digest(StringRef Content);
Optional<FileDigest> digestFile(const SourceManager &SM, FileID FID);
+// This context variable controls the behavior of functions in this file
+// that convert between LSP offsets and native clang byte offsets.
+// If not set, defaults to UTF-16 for backwards-compatibility.
+extern Key<OffsetEncoding> kCurrentOffsetEncoding;
+
// Counts the number of UTF-16 code units needed to represent a string (LSP
// specifies string lengths in UTF-16 code units).
+// Use of UTF-16 may be overridden by kCurrentOffsetEncoding.
size_t lspLength(StringRef Code);
/// Turn a [line, column] pair into an offset in Code.
Modified: clang-tools-extra/trunk/clangd/index/IndexAction.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/IndexAction.cpp?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/IndexAction.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/IndexAction.cpp Wed Mar 27 10:47:49 2019
@@ -9,7 +9,6 @@
#include "IndexAction.h"
#include "index/SymbolOrigin.h"
#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Index/IndexDataConsumer.h"
#include "clang/Index/IndexingAction.h"
#include "clang/Tooling/Tooling.h"
Modified: clang-tools-extra/trunk/clangd/index/SymbolLocation.h
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolLocation.h?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/index/SymbolLocation.h (original)
+++ clang-tools-extra/trunk/clangd/index/SymbolLocation.h Wed Mar 27 10:47:49 2019
@@ -20,6 +20,13 @@ struct SymbolLocation {
// Specify a position (Line, Column) of symbol. Using Line/Column allows us to
// build LSP responses without reading the file content.
//
+ // clangd uses the following definitions, which differ slightly from LSP:
+ // - Line is the number of newline characters (\n) before the point.
+ // - Column is (by default) the number of UTF-16 code between the last \n
+ // (or start of file) and the point.
+ // If the `offsetEncoding` protocol extension is used to negotiate UTF-8,
+ // then it is instead the number of *bytes* since the last \n.
+ //
// Position is encoded into 32 bits to save space.
// If Line/Column overflow, the value will be their maximum value.
struct Position {
@@ -37,8 +44,7 @@ struct SymbolLocation {
static constexpr uint32_t MaxColumn = (1 << 12) - 1;
private:
- uint32_t Line : 20; // 0-based
- // Using UTF-16 code units.
+ uint32_t Line : 20; // 0-based
uint32_t Column : 12; // 0-based
};
Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Wed Mar 27 10:47:49 2019
@@ -9,10 +9,12 @@
#include "Features.inc"
#include "ClangdLSPServer.h"
#include "Path.h"
+#include "Protocol.h"
#include "Trace.h"
#include "Transport.h"
#include "index/Serialization.h"
#include "clang/Basic/Version.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
@@ -219,6 +221,16 @@ static llvm::cl::opt<bool> SuggestMissin
"includes using index."),
llvm::cl::init(true));
+static llvm::cl::opt<OffsetEncoding> ForceOffsetEncoding(
+ "offset-encoding",
+ llvm::cl::desc("Force the offsetEncoding used for character positions. "
+ "This bypasses negotiation via client capabilities."),
+ llvm::cl::values(clEnumValN(OffsetEncoding::UTF8, "utf-8",
+ "Offsets are in UTF-8 bytes"),
+ clEnumValN(OffsetEncoding::UTF16, "utf-16",
+ "Offsets are in UTF-16 code units")),
+ llvm::cl::init(OffsetEncoding::UnsupportedEncoding));
+
namespace {
/// \brief Supports a test URI scheme with relaxed constraints for lit tests.
@@ -458,9 +470,13 @@ int main(int argc, char *argv[]) {
}
Opts.ClangTidyOptProvider = ClangTidyOptProvider.get();
Opts.SuggestMissingIncludes = SuggestMissingIncludes;
+ llvm::Optional<OffsetEncoding> OffsetEncodingFromFlag;
+ if (ForceOffsetEncoding != OffsetEncoding::UnsupportedEncoding)
+ OffsetEncodingFromFlag = ForceOffsetEncoding;
ClangdLSPServer LSPServer(
*TransportLayer, FSProvider, CCOpts, CompileCommandsDirPath,
- /*UseDirBasedCDB=*/CompileArgsFrom == FilesystemCompileArgs, Opts);
+ /*UseDirBasedCDB=*/CompileArgsFrom == FilesystemCompileArgs,
+ OffsetEncodingFromFlag, Opts);
llvm::set_thread_name("clangd.main");
return LSPServer.run() ? 0
: static_cast<int>(ErrorResultCode::NoShutdownRequest);
Added: clang-tools-extra/trunk/test/clangd/utf8.test
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/utf8.test?rev=357102&view=auto
==============================================================================
--- clang-tools-extra/trunk/test/clangd/utf8.test (added)
+++ clang-tools-extra/trunk/test/clangd/utf8.test Wed Mar 27 10:47:49 2019
@@ -0,0 +1,32 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+# This test verifies that we can negotiate UTF-8 offsets via protocol extension.
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{"offsetEncoding":["utf-8","utf-16"]},"trace":"off"}}
+# CHECK: "offsetEncoding": "utf-8"
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///main.cpp","languageId":"cpp","version":1,"text":"/*ö*/int x;\nint y=x;"}}}
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/definition","params":{"textDocument":{"uri":"test:///main.cpp"},"position":{"line":1,"character":6}}}
+# /*ö*/int x;
+# 01234567890
+# x is character (and utf-16) range [9,10) but byte range [10,11).
+# CHECK: "id": 1,
+# CHECK-NEXT: "jsonrpc": "2.0",
+# CHECK-NEXT: "result": [
+# CHECK-NEXT: {
+# CHECK-NEXT: "range": {
+# CHECK-NEXT: "end": {
+# CHECK-NEXT: "character": 11,
+# CHECK-NEXT: "line": 0
+# CHECK-NEXT: },
+# CHECK-NEXT: "start": {
+# CHECK-NEXT: "character": 10,
+# CHECK-NEXT: "line": 0
+# CHECK-NEXT: }
+# CHECK-NEXT: },
+# CHECK-NEXT: "uri": "file://{{.*}}/main.cpp"
+# CHECK-NEXT: }
+# CHECK-NEXT: ]
+---
+{"jsonrpc":"2.0","id":10000,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
Modified: clang-tools-extra/trunk/unittests/clangd/SourceCodeTests.cpp
URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/SourceCodeTests.cpp?rev=357102&r1=357101&r2=357102&view=diff
==============================================================================
--- clang-tools-extra/trunk/unittests/clangd/SourceCodeTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/SourceCodeTests.cpp Wed Mar 27 10:47:49 2019
@@ -6,6 +6,8 @@
//
//===----------------------------------------------------------------------===//
#include "Annotations.h"
+#include "Context.h"
+#include "Protocol.h"
#include "SourceCode.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/raw_os_ostream.h"
@@ -21,14 +23,9 @@ using llvm::Failed;
using llvm::HasValue;
MATCHER_P2(Pos, Line, Col, "") {
- return arg.line == Line && arg.character == Col;
+ return arg.line == int(Line) && arg.character == int(Col);
}
-// The = â ð¡ below are ASCII (1 byte), BMP (3 bytes), and astral (4 bytes).
-const char File[] = R"(0:0 = 0
-1:0 â 8
-2:0 ð¡ 18)";
-
/// A helper to make tests easier to read.
Position position(int line, int character) {
Position Pos;
@@ -52,8 +49,28 @@ TEST(SourceCodeTests, lspLength) {
EXPECT_EQ(lspLength("Â¥"), 1UL);
// astral
EXPECT_EQ(lspLength("ð"), 2UL);
+
+ WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8);
+ EXPECT_EQ(lspLength(""), 0UL);
+ EXPECT_EQ(lspLength("ascii"), 5UL);
+ // BMP
+ EXPECT_EQ(lspLength("â"), 3UL);
+ EXPECT_EQ(lspLength("Â¥"), 2UL);
+ // astral
+ EXPECT_EQ(lspLength("ð"), 4UL);
}
+// The = â ð¡ below are ASCII (1 byte), BMP (3 bytes), and astral (4 bytes).
+const char File[] = R"(0:0 = 0
+1:0 â 8
+2:0 ð¡ 18)";
+struct Line {
+ unsigned Number;
+ unsigned Offset;
+ unsigned Length;
+};
+Line FileLines[] = {Line{0, 0, 7}, Line{1, 8, 9}, Line{2, 18, 11}};
+
TEST(SourceCodeTests, PositionToOffset) {
// line out of bounds
EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed());
@@ -113,6 +130,23 @@ TEST(SourceCodeTests, PositionToOffset)
// line out of bounds
EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed());
EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 1)), llvm::Failed());
+
+ // Test UTF-8, where transformations are trivial.
+ WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8);
+ EXPECT_THAT_EXPECTED(positionToOffset(File, position(-1, 2)), llvm::Failed());
+ EXPECT_THAT_EXPECTED(positionToOffset(File, position(3, 0)), llvm::Failed());
+ for (Line L : FileLines) {
+ EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, -1)),
+ llvm::Failed()); // out of range
+ for (unsigned I = 0; I <= L.Length; ++I)
+ EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, I)),
+ llvm::HasValue(L.Offset + I));
+ EXPECT_THAT_EXPECTED(positionToOffset(File, position(L.Number, L.Length+1)),
+ llvm::HasValue(L.Offset + L.Length));
+ EXPECT_THAT_EXPECTED(
+ positionToOffset(File, position(L.Number, L.Length + 1), false),
+ llvm::Failed()); // out of range
+ }
}
TEST(SourceCodeTests, OffsetToPosition) {
@@ -134,6 +168,13 @@ TEST(SourceCodeTests, OffsetToPosition)
EXPECT_THAT(offsetToPosition(File, 28), Pos(2, 8)) << "end of last line";
EXPECT_THAT(offsetToPosition(File, 29), Pos(2, 9)) << "EOF";
EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 9)) << "out of bounds";
+
+ WithContextValue UTF8(kCurrentOffsetEncoding, OffsetEncoding::UTF8);
+ for (Line L : FileLines) {
+ for (unsigned I = 0; I <= L.Length; ++I)
+ EXPECT_THAT(offsetToPosition(File, L.Offset + I), Pos(L.Number, I));
+ }
+ EXPECT_THAT(offsetToPosition(File, 30), Pos(2, 11)) << "out of bounds";
}
TEST(SourceCodeTests, IsRangeConsecutive) {
More information about the cfe-commits
mailing list