Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit db890d4

Browse filesBrowse files
MaskRayMaskRay
committed
Support LSP semantic tokens
This patch implements `textDocument/semanticTokens/{full,range}`. If the client supports semantic tokens, $ccls/publishSemanticHighlight (now deprecated) is disabled. These token modifiers are mostly useful to emphasize certain symbols: `static, classScope, globalScope, namespaceScope`. To enable a colorful syntax highlighting scheme, set the highlight.rainbow initialization option to 10. https://maskray.me/blog/2024-10-20-ccls-and-lsp-semantic-tokens Note that the older $ccls/publishSemanticHighlight protocol with highlight.lsRanges==true (used by vscode-ccls) is no longer supported.
1 parent 50fd8d0 commit db890d4
Copy full SHA for db890d4

File tree

Expand file treeCollapse file tree

8 files changed

+272
-43
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

8 files changed

+272
-43
lines changed
Open diff view settings
Collapse file

‎src/config.hh‎

Copy file name to clipboardExpand all lines: src/config.hh
+6-3Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ struct Config {
117117
bool hierarchicalDocumentSymbolSupport = true;
118118
// TextDocumentClientCapabilities.definition.linkSupport
119119
bool linkSupport = true;
120+
// ClientCapabilities.workspace.semanticTokens.refreshSupport
121+
bool semanticTokensRefresh = true;
120122

121123
// If false, disable snippets and complete just the identifier part.
122124
// TextDocumentClientCapabilities.completion.completionItem.snippetSupport
@@ -226,8 +228,9 @@ struct Config {
226228
// Disable semantic highlighting for files larger than the size.
227229
int64_t largeFileSize = 2 * 1024 * 1024;
228230

229-
// true: LSP line/character; false: position
230-
bool lsRanges = false;
231+
// If non-zero, enable rainbow semantic tokens by assinging an extra modifier
232+
// indicating the rainbow ID to each symbol.
233+
int rainbow = 0;
231234

232235
// Like index.{whitelist,blacklist}, don't publish semantic highlighting to
233236
// blacklisted files.
@@ -342,7 +345,7 @@ REFLECT_STRUCT(Config::Completion, caseSensitivity, detailedLabel,
342345
maxNum, placeholder);
343346
REFLECT_STRUCT(Config::Diagnostics, blacklist, onChange, onOpen, onSave,
344347
spellChecking, whitelist)
345-
REFLECT_STRUCT(Config::Highlight, largeFileSize, lsRanges, blacklist, whitelist)
348+
REFLECT_STRUCT(Config::Highlight, largeFileSize, rainbow, blacklist, whitelist)
346349
REFLECT_STRUCT(Config::Index::Name, suppressUnwrittenScope);
347350
REFLECT_STRUCT(Config::Index, blacklist, comments, initialNoLinkage,
348351
initialBlacklist, initialWhitelist, maxInitializerLines,
Collapse file

‎src/enum.inc‎

Copy file name to clipboard
+36Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#ifndef TOKEN_MODIFIER
2+
#define TOKEN_MODIFIER(name, str)
3+
#endif
4+
// vscode
5+
TOKEN_MODIFIER(Declaration, "declaration")
6+
TOKEN_MODIFIER(Definition, "definition")
7+
TOKEN_MODIFIER(Static, "static")
8+
9+
// ccls extensions
10+
TOKEN_MODIFIER(Read, "read")
11+
TOKEN_MODIFIER(Write, "write")
12+
TOKEN_MODIFIER(ClassScope, "classScope")
13+
TOKEN_MODIFIER(FunctionScope, "functionScope")
14+
TOKEN_MODIFIER(NamespaceScope, "namespaceScope")
15+
16+
// Rainbow semantic tokens
17+
TOKEN_MODIFIER(Id0, "id0")
18+
TOKEN_MODIFIER(Id1, "id1")
19+
TOKEN_MODIFIER(Id2, "id2")
20+
TOKEN_MODIFIER(Id3, "id3")
21+
TOKEN_MODIFIER(Id4, "id4")
22+
TOKEN_MODIFIER(Id5, "id5")
23+
TOKEN_MODIFIER(Id6, "id6")
24+
TOKEN_MODIFIER(Id7, "id7")
25+
TOKEN_MODIFIER(Id8, "id8")
26+
TOKEN_MODIFIER(Id9, "id9")
27+
TOKEN_MODIFIER(Id10, "id10")
28+
TOKEN_MODIFIER(Id11, "id11")
29+
TOKEN_MODIFIER(Id12, "id12")
30+
TOKEN_MODIFIER(Id13, "id13")
31+
TOKEN_MODIFIER(Id14, "id14")
32+
TOKEN_MODIFIER(Id15, "id15")
33+
TOKEN_MODIFIER(Id16, "id16")
34+
TOKEN_MODIFIER(Id17, "id17")
35+
TOKEN_MODIFIER(Id18, "id18")
36+
TOKEN_MODIFIER(Id19, "id19")
Collapse file

‎src/indexer.hh‎

Copy file name to clipboardExpand all lines: src/indexer.hh
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,12 @@ void reflect(BinaryWriter &visitor, SymbolRef &value);
132132
void reflect(BinaryWriter &visitor, Use &value);
133133
void reflect(BinaryWriter &visitor, DeclRef &value);
134134

135+
enum class TokenModifier {
136+
#define TOKEN_MODIFIER(name, str) name,
137+
#include "enum.inc"
138+
#undef TOKEN_MODIFIER
139+
};
140+
135141
template <typename T> using VectorAdapter = std::vector<T, std::allocator<T>>;
136142

137143
template <typename D> struct NameMixin {
Collapse file

‎src/lsp.hh‎

Copy file name to clipboardExpand all lines: src/lsp.hh
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ enum class SymbolKind : uint8_t {
166166
// For C++, this is interpreted as "template parameter" (including
167167
// non-type template parameters).
168168
TypeParameter = 26,
169+
FirstNonStandard,
169170

170171
// ccls extensions
171172
// See also https://github.com/Microsoft/language-server-protocol/issues/344
@@ -174,6 +175,8 @@ enum class SymbolKind : uint8_t {
174175
Parameter = 253,
175176
StaticMethod = 254,
176177
Macro = 255,
178+
FirstExtension = TypeAlias,
179+
LastExtension = Macro,
177180
};
178181

179182
struct SymbolInformation {
Collapse file

‎src/message_handler.cc‎

Copy file name to clipboardExpand all lines: src/message_handler.cc
+135-27Lines changed: 135 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,25 @@
1111
#include <rapidjson/document.h>
1212
#include <rapidjson/reader.h>
1313

14+
#include <llvm/ADT/STLExtras.h>
15+
1416
#include <algorithm>
1517
#include <stdexcept>
1618

1719
using namespace clang;
1820

21+
#if LLVM_VERSION_MAJOR < 15 // llvmorg-15-init-6118-gb39f43775796
22+
namespace llvm {
23+
template <typename T, typename E>
24+
constexpr bool is_contained(std::initializer_list<T> set, const E &e) {
25+
for (const T &v : set)
26+
if (v == e)
27+
return true;
28+
return false;
29+
}
30+
}
31+
#endif
32+
1933
MAKE_HASHABLE(ccls::SymbolIdx, t.usr, t.kind);
2034

2135
namespace ccls {
@@ -51,23 +65,26 @@ REFLECT_STRUCT(DidChangeWorkspaceFoldersParam, event);
5165
REFLECT_STRUCT(WorkspaceSymbolParam, query, folders);
5266

5367
namespace {
68+
struct Occur {
69+
lsRange range;
70+
Role role;
71+
};
5472
struct CclsSemanticHighlightSymbol {
5573
int id = 0;
5674
SymbolKind parentKind;
5775
SymbolKind kind;
5876
uint8_t storage;
5977
std::vector<std::pair<int, int>> ranges;
6078

61-
// `lsRanges` is used to compute `ranges`.
62-
std::vector<lsRange> lsRanges;
79+
// `lsOccur` is used to compute `ranges`.
80+
std::vector<Occur> lsOccurs;
6381
};
6482

6583
struct CclsSemanticHighlight {
6684
DocumentUri uri;
6785
std::vector<CclsSemanticHighlightSymbol> symbols;
6886
};
69-
REFLECT_STRUCT(CclsSemanticHighlightSymbol, id, parentKind, kind, storage,
70-
ranges, lsRanges);
87+
REFLECT_STRUCT(CclsSemanticHighlightSymbol, id, parentKind, kind, storage, ranges);
7188
REFLECT_STRUCT(CclsSemanticHighlight, uri, symbols);
7289

7390
struct CclsSetSkippedRanges {
@@ -76,10 +93,16 @@ struct CclsSetSkippedRanges {
7693
};
7794
REFLECT_STRUCT(CclsSetSkippedRanges, uri, skippedRanges);
7895

96+
struct SemanticTokensPartialResult {
97+
std::vector<int> data;
98+
};
99+
REFLECT_STRUCT(SemanticTokensPartialResult, data);
100+
79101
struct ScanLineEvent {
80102
Position pos;
81103
Position end_pos; // Second key when there is a tie for insertion events.
82104
int id;
105+
Role role;
83106
CclsSemanticHighlightSymbol *symbol;
84107
bool operator<(const ScanLineEvent &o) const {
85108
// See the comments below when insertion/deletion events are inserted.
@@ -190,6 +213,8 @@ MessageHandler::MessageHandler() {
190213
bind("textDocument/rangeFormatting", &MessageHandler::textDocument_rangeFormatting);
191214
bind("textDocument/references", &MessageHandler::textDocument_references);
192215
bind("textDocument/rename", &MessageHandler::textDocument_rename);
216+
bind("textDocument/semanticTokens/full", &MessageHandler::textDocument_semanticTokensFull);
217+
bind("textDocument/semanticTokens/range", &MessageHandler::textDocument_semanticTokensRange);
193218
bind("textDocument/signatureHelp", &MessageHandler::textDocument_signatureHelp);
194219
bind("textDocument/typeDefinition", &MessageHandler::textDocument_typeDefinition);
195220
bind("workspace/didChangeConfiguration", &MessageHandler::workspace_didChangeConfiguration);
@@ -281,16 +306,16 @@ void emitSkippedRanges(WorkingFile *wfile, QueryFile &file) {
281306
pipeline::notify("$ccls/publishSkippedRanges", params);
282307
}
283308

284-
void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
309+
static std::unordered_map<SymbolIdx, CclsSemanticHighlightSymbol> computeSemanticTokens(DB *db, WorkingFile *wfile,
310+
QueryFile &file) {
285311
static GroupMatch match(g_config->highlight.whitelist,
286312
g_config->highlight.blacklist);
287313
assert(file.def);
288-
if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
289-
!match.matches(file.def->path))
290-
return;
291-
292314
// Group symbols together.
293315
std::unordered_map<SymbolIdx, CclsSemanticHighlightSymbol> grouped_symbols;
316+
if (!match.matches(file.def->path))
317+
return grouped_symbols;
318+
294319
for (auto [sym, refcnt] : file.symbol2refcnt) {
295320
if (refcnt <= 0)
296321
continue;
@@ -369,14 +394,14 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
369394
if (std::optional<lsRange> loc = getLsRange(wfile, sym.range)) {
370395
auto it = grouped_symbols.find(sym);
371396
if (it != grouped_symbols.end()) {
372-
it->second.lsRanges.push_back(*loc);
397+
it->second.lsOccurs.push_back({*loc, sym.role});
373398
} else {
374399
CclsSemanticHighlightSymbol symbol;
375400
symbol.id = idx;
376401
symbol.parentKind = parent_kind;
377402
symbol.kind = kind;
378403
symbol.storage = storage;
379-
symbol.lsRanges.push_back(*loc);
404+
symbol.lsOccurs.push_back({*loc, sym.role});
380405
grouped_symbols[sym] = symbol;
381406
}
382407
}
@@ -387,17 +412,17 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
387412
int id = 0;
388413
for (auto &entry : grouped_symbols) {
389414
CclsSemanticHighlightSymbol &symbol = entry.second;
390-
for (auto &loc : symbol.lsRanges) {
415+
for (auto &occur : symbol.lsOccurs) {
391416
// For ranges sharing the same start point, the one with leftmost end
392417
// point comes first.
393-
events.push_back({loc.start, loc.end, id, &symbol});
418+
events.push_back({occur.range.start, occur.range.end, id, occur.role, &symbol});
394419
// For ranges sharing the same end point, their relative order does not
395-
// matter, therefore we arbitrarily assign loc.end to them. We use
420+
// matter, therefore we arbitrarily assign occur.range.end to them. We use
396421
// negative id to indicate a deletion event.
397-
events.push_back({loc.end, loc.end, ~id, &symbol});
422+
events.push_back({occur.range.end, occur.range.end, ~id, occur.role, &symbol});
398423
id++;
399424
}
400-
symbol.lsRanges.clear();
425+
symbol.lsOccurs.clear();
401426
}
402427
std::sort(events.begin(), events.end());
403428

@@ -413,26 +438,33 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
413438
// Attribute range [events[i-1].pos, events[i].pos) to events[top-1].symbol
414439
// .
415440
if (top && !(events[i - 1].pos == events[i].pos))
416-
events[top - 1].symbol->lsRanges.push_back(
417-
{events[i - 1].pos, events[i].pos});
441+
events[top - 1].symbol->lsOccurs.push_back({{events[i - 1].pos, events[i].pos}, events[i].role});
418442
if (events[i].id >= 0)
419443
events[top++] = events[i];
420444
else
421445
deleted[~events[i].id] = 1;
422446
}
447+
return grouped_symbols;
448+
}
449+
450+
void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
451+
// Disable $ccls/publishSemanticHighlight if semantic tokens support is
452+
// enabled or the file is too large.
453+
if (g_config->client.semanticTokensRefresh || wfile->buffer_content.size() > g_config->highlight.largeFileSize)
454+
return;
455+
auto grouped_symbols = computeSemanticTokens(db, wfile, file);
423456

424457
CclsSemanticHighlight params;
425458
params.uri = DocumentUri::fromPath(wfile->filename);
426459
// Transform lsRange into pair<int, int> (offset pairs)
427-
if (!g_config->highlight.lsRanges) {
428-
std::vector<std::pair<lsRange, CclsSemanticHighlightSymbol *>> scratch;
460+
{
461+
std::vector<std::pair<Occur, CclsSemanticHighlightSymbol *>> scratch;
429462
for (auto &entry : grouped_symbols) {
430-
for (auto &range : entry.second.lsRanges)
431-
scratch.emplace_back(range, &entry.second);
432-
entry.second.lsRanges.clear();
463+
for (auto &occur : entry.second.lsOccurs)
464+
scratch.push_back({occur, &entry.second});
465+
entry.second.lsOccurs.clear();
433466
}
434-
std::sort(scratch.begin(), scratch.end(),
435-
[](auto &l, auto &r) { return l.first.start < r.first.start; });
467+
std::sort(scratch.begin(), scratch.end(), [](auto &l, auto &r) { return l.first.range < r.first.range; });
436468
const auto &buf = wfile->buffer_content;
437469
int l = 0, c = 0, i = 0, p = 0;
438470
auto mov = [&](int line, int col) {
@@ -455,7 +487,7 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
455487
return c < col;
456488
};
457489
for (auto &entry : scratch) {
458-
lsRange &r = entry.first;
490+
lsRange &r = entry.first.range;
459491
if (mov(r.start.line, r.start.character))
460492
continue;
461493
int beg = p;
@@ -466,8 +498,84 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
466498
}
467499

468500
for (auto &entry : grouped_symbols)
469-
if (entry.second.ranges.size() || entry.second.lsRanges.size())
501+
if (entry.second.ranges.size() || entry.second.lsOccurs.size())
470502
params.symbols.push_back(std::move(entry.second));
471503
pipeline::notify("$ccls/publishSemanticHighlight", params);
472504
}
505+
506+
void MessageHandler::textDocument_semanticTokensFull(TextDocumentParam &param, ReplyOnce &reply) {
507+
SemanticTokensRangeParams parameters{param.textDocument, lsRange{{0, 0}, {UINT16_MAX, INT16_MAX}}};
508+
textDocument_semanticTokensRange(parameters, reply);
509+
}
510+
511+
void MessageHandler::textDocument_semanticTokensRange(SemanticTokensRangeParams &param, ReplyOnce &reply) {
512+
int file_id;
513+
auto [file, wf] = findOrFail(param.textDocument.uri.getPath(), reply, &file_id);
514+
if (!wf)
515+
return;
516+
517+
auto grouped_symbols = computeSemanticTokens(db, wf, *file);
518+
std::vector<std::pair<Occur, CclsSemanticHighlightSymbol *>> scratch;
519+
for (auto &entry : grouped_symbols) {
520+
for (auto &occur : entry.second.lsOccurs)
521+
scratch.emplace_back(occur, &entry.second);
522+
entry.second.lsOccurs.clear();
523+
}
524+
std::sort(scratch.begin(), scratch.end(), [](auto &l, auto &r) { return l.first.range < r.first.range; });
525+
526+
SemanticTokensPartialResult result;
527+
int line = 0, column = 0;
528+
for (auto &entry : scratch) {
529+
lsRange &r = entry.first.range;
530+
CclsSemanticHighlightSymbol &symbol = *entry.second;
531+
if (r.start.line != line)
532+
column = 0;
533+
result.data.push_back(r.start.line - line);
534+
line = r.start.line;
535+
result.data.push_back(r.start.character - column);
536+
column = r.start.character;
537+
result.data.push_back(r.end.character - r.start.character);
538+
539+
int tokenType = (int)symbol.kind, modifier = 0;
540+
if (tokenType == (int)SymbolKind::StaticMethod) {
541+
tokenType = (int)SymbolKind::Method;
542+
modifier |= 1 << (int)TokenModifier::Static;
543+
} else if (tokenType >= (int)SymbolKind::FirstExtension) {
544+
tokenType += (int)SymbolKind::FirstNonStandard - (int)SymbolKind::FirstExtension;
545+
}
546+
547+
// Set modifiers.
548+
if (entry.first.role & Role::Declaration)
549+
modifier |= 1 << (int)TokenModifier::Declaration;
550+
if (entry.first.role & Role::Definition)
551+
modifier |= 1 << (int)TokenModifier::Definition;
552+
if (entry.first.role & Role::Read)
553+
modifier |= 1 << (int)TokenModifier::Read;
554+
if (entry.first.role & Role::Write)
555+
modifier |= 1 << (int)TokenModifier::Write;
556+
if (symbol.storage == SC_Static)
557+
modifier |= 1 << (int)TokenModifier::Static;
558+
559+
if (llvm::is_contained({SymbolKind::Constructor, SymbolKind::Field, SymbolKind::Method, SymbolKind::StaticMethod},
560+
symbol.kind))
561+
modifier |= 1 << (int)TokenModifier::ClassScope;
562+
else if (llvm::is_contained({SymbolKind::File, SymbolKind::Namespace}, symbol.parentKind))
563+
modifier |= 1 << (int)TokenModifier::NamespaceScope;
564+
else if (llvm::is_contained(
565+
{SymbolKind::Constructor, SymbolKind::Function, SymbolKind::Method, SymbolKind::StaticMethod},
566+
symbol.parentKind))
567+
modifier |= 1 << (int)TokenModifier::FunctionScope;
568+
569+
// Rainbow semantic tokens
570+
static_assert((int)TokenModifier::Id0 + 20 < 31);
571+
if (int rainbow = g_config->highlight.rainbow)
572+
modifier |= 1 << ((int)TokenModifier::Id0 + symbol.id % std::min(rainbow, 20));
573+
574+
result.data.push_back(tokenType);
575+
result.data.push_back(modifier);
576+
}
577+
578+
reply(result);
579+
}
580+
473581
} // namespace ccls

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.