Commit 45a32b17 authored by Waqar Ahmed's avatar Waqar Ahmed
Browse files

Implement LSP semantic tokens protocol



This change implements the LSP semantic tokens protocol that has been
introduced with LSP 3.16. Both full and fullDelta requests are implemented.

`range` request has not been implemented

For now modifiers are not handled, this is for three reasons
- clangd doesn't support them yet and that is what I am testing against
- For properly handling modifiers we may have to extend KSyntaxHighlighting
- For now I am aiming to make it simple, solid and fast, modifier
handling can be added later
Signed-off-by: Waqar Ahmed's avatarWaqar Ahmed <waqar.17a@gmail.com>
parent d5354b0a
......@@ -43,6 +43,7 @@ target_sources(
lspclientservermanager.cpp
lspclientsymbolview.cpp
lspsemantichighlighting.cpp
semantic_tokens_legend.cpp
lsptooltip.cpp
plugin.qrc
${UI_SOURCES}
......
......@@ -430,6 +430,8 @@ class LSPClientActionView : public QObject
}
};
SemanticHighlighter m_semHighlightingManager;
Q_SIGNALS:
/**
* Signal for outgoing message, the host application will handle them!
......@@ -626,6 +628,48 @@ public:
}
}
void doSemanticHighlighting(KTextEditor::View *view)
{
if (!view) {
return;
}
auto server = m_serverManager->findServer(view);
if (server) {
auto reqId = m_semHighlightingManager.resultIdForDoc(view->document()->url());
m_semHighlightingManager.setLegend(&server->capabilities().semanticTokenProvider.legend);
// m_semHighlightingManager.setTypes(server->capabilities().semanticTokenProvider.types);
/**
* Full delta only if server supports it or if we don't have a result ID for this document
*/
if (reqId.isEmpty() || !server->capabilities().semanticTokenProvider.fullDelta) {
auto h = [this, view](const LSPSemanticTokens &st) {
auto url = view->document()->url();
m_semHighlightingManager.setCurrentView(view);
m_semHighlightingManager.insert(url, st.resultId, st.data);
m_semHighlightingManager.highlight(url);
};
server->documentSemanticTokensFull(view->document()->url(), {}, this, h);
} else {
auto h = [this, view](const LSPSemanticTokensDelta &st) {
auto url = view->document()->url();
m_semHighlightingManager.setCurrentView(view);
for (const auto &semTokenEdit : st.edits) {
m_semHighlightingManager.update(url, st.resultId, semTokenEdit.start, semTokenEdit.deleteCount, semTokenEdit.data);
}
if (!st.data.empty()) {
m_semHighlightingManager.insert(url, st.resultId, st.data);
}
m_semHighlightingManager.highlight(url);
};
server->documentSemanticTokensFullDelta(view->document()->url(), reqId, this, h);
}
}
}
// This is taken from KDevelop :)
KTextEditor::View *viewFromWidget(QWidget *widget)
{
......@@ -2126,86 +2170,6 @@ public:
addMessage(lvl, i18nc("@info", "LSP Client"), msg);
}
Q_SLOT void clearSemanticHighlighting(KTextEditor::Document *document)
{
auto &documentRanges = m_semanticHighlightRanges[document];
for (const auto &lineRanges : documentRanges) {
qDeleteAll(lineRanges);
}
documentRanges.clear();
}
void onSemanticHighlighting(const LSPSemanticHighlightingParams &params)
{
auto *view = viewForUrl(params.textDocument.uri);
if (!view) {
qCWarning(LSPCLIENT) << "failed to find view for uri" << params.textDocument.uri;
return;
}
auto server = m_serverManager->findServer(view);
if (!server) {
qCWarning(LSPCLIENT) << "failed to find server for view" << params.textDocument.uri;
return;
}
auto *document = view->document();
auto *miface = qobject_cast<KTextEditor::MovingInterface *>(document);
Q_ASSERT(miface);
// TODO: translate between locked revision, if possible?
auto version = params.textDocument.version;
if (version == -1) { // use version from disk
version = miface->lastSavedRevision();
if (version == -1) { // never saved
version = miface->revision();
}
}
if (version != miface->revision()) {
qCDebug(LSPCLIENT) << "discarding highlighting, versions don't match:" << params.textDocument.version << version << miface->revision();
return;
}
// ensure runtime match
// clang-format off
connect(document,
SIGNAL(aboutToInvalidateMovingInterfaceContent(KTextEditor::Document*)),
this,
SLOT(clearSemanticHighlighting(KTextEditor::Document*)),
Qt::UniqueConnection);
connect(document,
SIGNAL(aboutToDeleteMovingInterfaceContent(KTextEditor::Document*)),
this,
SLOT(clearSemanticHighlighting(KTextEditor::Document*)),
Qt::UniqueConnection);
// clang-format on
// TODO: we should try to recycle the moving ranges instead of recreating them all the time
const auto &scopes = server->capabilities().semanticHighlightingProvider.scopes;
// qDebug() << params.textDocument.uri << scopes;
auto &documentRanges = m_semanticHighlightRanges[document];
for (const auto &line : params.lines) {
auto &lineRanges = documentRanges[line.line];
qDeleteAll(lineRanges);
lineRanges.clear();
// qDebug() << "line:" << line.line << ", toks " << line.tokens.size();
for (const auto &token : line.tokens) {
// qDebug() << "token:" << token.character << token.length << token.scope;
auto attribute = scopes.attrForScope(token.scope);
if (!attribute) {
continue;
}
const auto columnStart = static_cast<int>(token.character);
const auto columnEnd = columnStart + static_cast<int>(token.length);
auto *range = miface->newMovingRange({line.line, columnStart, line.line, columnEnd});
range->setAttribute(attribute);
lineRanges.append(range);
}
}
}
void onDocumentUrlChanged(KTextEditor::Document *doc)
{
// url already changed by this time and new url not useful
......@@ -2236,12 +2200,16 @@ public:
void onTextChanged(KTextEditor::Document *doc)
{
if (m_onTypeFormattingTriggers.empty()) {
KTextEditor::View *activeView = m_mainWindow->activeView();
if (!activeView || activeView->document() != doc) {
return;
}
KTextEditor::View *activeView = m_mainWindow->activeView();
if (!activeView || activeView->document() != doc) {
if (m_plugin->m_semanticHighlighting) {
doSemanticHighlighting(activeView);
}
if (m_onTypeFormattingTriggers.empty()) {
return;
}
......@@ -2281,7 +2249,6 @@ public:
connect(server.data(), &LSPClientServer::publishDiagnostics, this, &self_type::onDiagnostics, Qt::UniqueConnection);
connect(server.data(), &LSPClientServer::showMessage, this, &self_type::onMessage, Qt::UniqueConnection);
connect(server.data(), &LSPClientServer::logMessage, this, &self_type::onMessage, Qt::UniqueConnection);
connect(server.data(), &LSPClientServer::semanticHighlighting, this, &self_type::onSemanticHighlighting, Qt::UniqueConnection);
connect(server.data(), &LSPClientServer::applyEdit, this, &self_type::onApplyEdit, Qt::UniqueConnection);
// update format trigger characters
......@@ -2299,6 +2266,25 @@ public:
// only consider basename (full path may have been custom specified)
auto lspServer = QFileInfo(server->cmdline().front()).fileName();
isClangd = lspServer == QStringLiteral("clangd");
const bool semHighlightingEnabled = m_plugin->m_semanticHighlighting;
const bool serverSupportsSemHighlighting = caps.semanticTokenProvider.full || caps.semanticTokenProvider.fullDelta;
if (semHighlightingEnabled && serverSupportsSemHighlighting) {
if (doc) {
connect(doc,
SIGNAL(aboutToInvalidateMovingInterfaceContent(KTextEditor::Document *)),
this,
SLOT(clearSemanticTokensHighlighting(KTextEditor::Document *)),
Qt::UniqueConnection);
connect(doc,
SIGNAL(aboutToDeleteMovingInterfaceContent(KTextEditor::Document *)),
this,
SLOT(clearSemanticTokensHighlighting(KTextEditor::Document *)),
Qt::UniqueConnection);
doSemanticHighlighting(activeView);
}
}
}
if (m_findDef) {
......@@ -2371,6 +2357,13 @@ public:
}
}
Q_SLOT void clearSemanticTokensHighlighting(KTextEditor::Document *doc)
{
if (doc) {
m_semHighlightingManager.remove(doc->url());
}
}
void viewDestroyed(QObject *view)
{
m_completionViews.remove(static_cast<KTextEditor::View *>(view));
......
......@@ -16,6 +16,7 @@
#include <QVector>
#include "lspsemantichighlighting.h"
#include "semantic_tokens_legend.h"
#include <KTextEditor/Cursor>
#include <KTextEditor/Range>
......@@ -65,9 +66,13 @@ struct LSPSignatureHelpOptions {
struct LSPDocumentOnTypeFormattingOptions : public LSPSignatureHelpOptions {
};
struct LSPSemanticHighlightingOptions {
// cf. https://manual.macromates.com/en/language_grammars
SemanticHighlighting scopes;
// Ref: https://microsoft.github.io/language-server-protocol/specification#textDocument_semanticTokens
struct LSPSemanticTokensOptions {
bool full = false;
bool fullDelta = false;
bool range = false;
SemanticTokensLegend legend;
// QVector<QString> types;
};
struct LSPServerCapabilities {
......@@ -89,7 +94,7 @@ struct LSPServerCapabilities {
bool renameProvider = false;
// CodeActionOptions not useful/considered at present
bool codeActionProvider = false;
LSPSemanticHighlightingOptions semanticHighlightingProvider;
LSPSemanticTokensOptions semanticTokenProvider;
};
enum class LSPMarkupKind { None = 0, PlainText = 1, MarkDown = 2 };
......@@ -331,4 +336,21 @@ struct LSPApplyWorkspaceEditResponse {
QString failureReason;
};
struct LSPSemanticTokensEdit {
uint32_t start = 0;
uint32_t deleteCount = 0;
std::vector<uint32_t> data;
};
struct LSPSemanticTokens {
QString resultId;
std::vector<uint32_t> data;
};
struct LSPSemanticTokensDelta {
QString resultId;
std::vector<LSPSemanticTokensEdit> edits;
std::vector<uint32_t> data;
};
#endif
......@@ -20,6 +20,7 @@
#include <QJsonObject>
#include <QTime>
#include <QtEndian>
#include <iostream>
#include <utility>
// good/bad old school; allows easier concatenate
......@@ -55,6 +56,7 @@ static const QString MEMBER_DIAGNOSTICS = QStringLiteral("diagnostics");
static const QString MEMBER_TARGET_URI = QStringLiteral("targetUri");
static const QString MEMBER_TARGET_RANGE = QStringLiteral("targetRange");
static const QString MEMBER_TARGET_SELECTION_RANGE = QStringLiteral("targetSelectionRange");
static const QString MEMBER_PREVIOUS_RESULT_ID = QStringLiteral("previousResultId");
// message construction helpers
static QJsonObject to_json(const LSPPosition &pos)
......@@ -268,22 +270,40 @@ static void from_json(LSPDocumentOnTypeFormattingOptions &options, const QJsonVa
}
}
static void from_json(LSPSemanticHighlightingOptions &options, const QJsonValue &json)
static void from_json(LSPSemanticTokensOptions &options, const QJsonObject &json)
{
if (!json.isObject()) {
if (json.isEmpty()) {
return;
}
const auto scopes = json.toObject().value(QStringLiteral("scopes"));
options.scopes.clear();
QVector<QString> entries;
for (const auto &scope_entry : scopes.toArray()) {
const auto json_entries = scope_entry.toArray();
entries.reserve(json_entries.size());
for (const auto &inner_json_entry : json_entries) {
entries.push_back(inner_json_entry.toString());
}
if (json.value(QStringLiteral("full")).isObject()) {
auto full = json.value(QStringLiteral("full")).toObject();
options.fullDelta = full.value(QStringLiteral("delta")).toBool();
options.full = options.fullDelta;
} else {
options.full = json.value(QStringLiteral("full")).toBool();
}
options.scopes.scopesToAttrVector(entries);
options.range = json.value(QStringLiteral("range")).toBool();
const auto legend = json.value(QStringLiteral("legend")).toObject();
const auto tokenTypes = legend.value(QStringLiteral("tokenTypes")).toArray();
std::vector<QString> types;
types.reserve(tokenTypes.size());
std::transform(tokenTypes.cbegin(), tokenTypes.cend(), std::back_inserter(types), [](const QJsonValue &jv) {
return jv.toString();
});
// options.types = QVector<QString>(types.begin(), types.end());
options.legend.initialize(types);
// Disabled
// const auto tokenMods = legend.value(QStringLiteral("tokenModifiers")).toArray();
// std::vector<QString> modifiers;
// modifiers.reserve(tokenMods.size());
// std::transform(tokenMods.cbegin(), tokenMods.cend(), std::back_inserter(modifiers), [](const QJsonValue &jv) {
// return jv.toString();
// });
}
static void from_json(LSPServerCapabilities &caps, const QJsonObject &json)
......@@ -315,7 +335,7 @@ static void from_json(LSPServerCapabilities &caps, const QJsonObject &json)
caps.renameProvider = toBoolOrObject(json.value(QStringLiteral("renameProvider")));
auto codeActionProvider = json.value(QStringLiteral("codeActionProvider"));
caps.codeActionProvider = codeActionProvider.toBool() || codeActionProvider.isObject();
from_json(caps.semanticHighlightingProvider, json.value(QStringLiteral("semanticHighlighting")).toObject());
from_json(caps.semanticTokenProvider, json.value(QStringLiteral("semanticTokensProvider")).toObject());
}
// follow suit; as performed in kate docmanager
......@@ -709,6 +729,76 @@ static QList<LSPCodeAction> parseCodeAction(const QJsonValue &result)
return ret;
}
static LSPSemanticTokens parseSemanticTokens(const QJsonValue &result)
{
LSPSemanticTokens ret;
auto json = result.toObject();
ret.resultId = json.value(QStringLiteral("resultId")).toString();
if (ret.resultId.isEmpty()) {
qCDebug(LSPCLIENT) << "unexpected emtpy result id when parsing semantic tokens";
return ret;
}
auto data = json.value(QStringLiteral("data")).toArray();
ret.data.reserve(data.size());
std::transform(data.cbegin(), data.cend(), std::back_inserter(ret.data), [](const QJsonValue &jv) {
return jv.toInt();
});
return ret;
}
static LSPSemanticTokensDelta parseSemanticTokensDelta(const QJsonValue &result)
{
LSPSemanticTokensDelta ret;
auto json = result.toObject();
ret.resultId = json.value(QStringLiteral("resultId")).toString();
// std::cout << QJsonDocument(json).toJson().constData() << '\n';
if (ret.resultId.isEmpty()) {
qCDebug(LSPCLIENT) << "unexpected emtpy result id when parsing semantic tokens";
return ret;
}
/**
* Intentionally disabled as "edits" part is already being handled by MovingRange
*
* Any new text that is entered is sent as new data that replaces all old data.
*
*/
auto edits = json.value(QStringLiteral("edits")).toArray();
for (const auto &edit_jsonValue : edits) {
if (!edit_jsonValue.isObject()) {
continue;
}
auto edit = edit_jsonValue.toObject();
LSPSemanticTokensEdit e;
e.start = edit.value(QStringLiteral("start")).toInt();
e.deleteCount = edit.value(QStringLiteral("deleteCount")).toInt();
auto data = edit.value(QStringLiteral("data")).toArray();
e.data.reserve(data.size());
std::transform(data.cbegin(), data.cend(), std::back_inserter(e.data), [](const QJsonValue &jv) {
return jv.toInt();
});
ret.edits.push_back(e);
}
auto data = json.value(QStringLiteral("data")).toArray();
ret.data.reserve(data.size());
std::transform(data.cbegin(), data.cend(), std::back_inserter(ret.data), [](const QJsonValue &jv) {
return jv.toInt();
});
return ret;
}
static LSPPublishDiagnosticsParams parseDiagnostics(const QJsonObject &result)
{
LSPPublishDiagnosticsParams ret;
......@@ -727,14 +817,6 @@ static LSPApplyWorkspaceEditParams parseApplyWorkspaceEditParams(const QJsonObje
return ret;
}
static LSPVersionedTextDocumentIdentifier parseVersionedTextDocumentIdentifier(const QJsonObject &result)
{
LSPVersionedTextDocumentIdentifier ret;
ret.uri = normalizeUrl(QUrl(result.value(MEMBER_URI).toString()));
ret.version = result.value(QStringLiteral("version")).toInt(-1);
return ret;
}
static LSPShowMessageParams parseMessage(const QJsonObject &result)
{
LSPShowMessageParams ret;
......@@ -744,39 +826,6 @@ static LSPShowMessageParams parseMessage(const QJsonObject &result)
return ret;
}
static LSPSemanticHighlightingParams parseSemanticHighlighting(const QJsonObject &result)
{
LSPSemanticHighlightingParams ret;
ret.textDocument = parseVersionedTextDocumentIdentifier(result.value(QStringLiteral("textDocument")).toObject());
for (const auto &line_json : result.value(QStringLiteral("lines")).toArray()) {
const auto line_obj = line_json.toObject();
LSPSemanticHighlightingInformation info;
info.line = line_obj.value(QStringLiteral("line")).toInt(-1);
const auto tokenString = line_obj.value(QStringLiteral("tokens"));
constexpr auto TokenSize = sizeof(LSPSemanticHighlightingToken);
// the raw tokens are in big endian, we may need to convert that to little endian
const auto rawTokens = QByteArray::fromBase64(tokenString.toString().toUtf8());
if (rawTokens.size() % TokenSize != 0) {
qCDebug(LSPCLIENT) << "unexpected raw token size" << rawTokens.size() << "for string" << tokenString << "in line" << info.line;
continue;
}
const auto numTokens = rawTokens.size() / TokenSize;
const auto *begin = reinterpret_cast<const LSPSemanticHighlightingToken *>(rawTokens.constData());
const auto *end = begin + numTokens;
info.tokens.resize(numTokens);
std::transform(begin, end, info.tokens.begin(), [](const LSPSemanticHighlightingToken &rawToken) {
LSPSemanticHighlightingToken token;
token.character = qFromBigEndian(rawToken.character);
token.length = qFromBigEndian(rawToken.length);
token.scope = qFromBigEndian(rawToken.scope);
return token;
});
ret.lines.push_back(info);
}
return ret;
}
using GenericReplyType = QJsonValue;
using GenericReplyHandler = ReplyHandler<GenericReplyType>;
......@@ -1241,6 +1290,16 @@ public:
return send(init_request(QStringLiteral("textDocument/codeAction"), params), h);
}
RequestHandle documentSemanticTokensFull(const QUrl &document, bool delta, const QString requestId, const GenericReplyHandler &h)
{
auto params = textDocumentParams(document);
if (delta && !requestId.isEmpty()) {
params[MEMBER_PREVIOUS_RESULT_ID] = requestId;
return send(init_request(QStringLiteral("textDocument/semanticTokens/full/delta"), params), h);
}
return send(init_request(QStringLiteral("textDocument/semanticTokens/full"), params), h);
}
void executeCommand(const QString &command, const QJsonValue &args)
{
auto params = executeCommandParams(command, args);
......@@ -1286,8 +1345,6 @@ public:
auto method = msg[MEMBER_METHOD].toString();
if (method == QLatin1String("textDocument/publishDiagnostics")) {
Q_EMIT q->publishDiagnostics(parseDiagnostics(msg[MEMBER_PARAMS].toObject()));
} else if (method == QLatin1String("textDocument/semanticHighlighting")) {
Q_EMIT q->semanticHighlighting(parseSemanticHighlighting(msg[MEMBER_PARAMS].toObject()));
} else if (method == QLatin1String("window/showMessage")) {
Q_EMIT q->showMessage(parseMessage(msg[MEMBER_PARAMS].toObject()));
} else if (method == QLatin1String("window/logMessage")) {
......@@ -1526,6 +1583,20 @@ LSPClientServer::RequestHandle LSPClientServer::documentCodeAction(const QUrl &d
return d->documentCodeAction(document, range, kinds, std::move(diagnostics), make_handler(h, context, parseCodeAction));
}
LSPClientServer::RequestHandle
LSPClientServer::documentSemanticTokensFull(const QUrl &document, const QString requestId, const QObject *context, const SemanticTokensReplyHandler &h)
{
return d->documentSemanticTokensFull(document, /* delta = */ false, requestId, make_handler(h, context, parseSemanticTokens));
}
LSPClientServer::RequestHandle LSPClientServer::documentSemanticTokensFullDelta(const QUrl &document,
const QString requestId,
const QObject *context,
const SemanticTokensDeltaReplyHandler &h)
{
return d->documentSemanticTokensFull(document, /* delta = */ true, requestId, make_handler(h, context, parseSemanticTokensDelta));
}
void LSPClientServer::executeCommand(const QString &command, const QJsonValue &args)
{
return d->executeCommand(command, args);
......
......@@ -64,6 +64,8 @@ using CodeActionReplyHandler = ReplyHandler<QList<LSPCodeAction>>;
using WorkspaceEditReplyHandler = ReplyHandler<LSPWorkspaceEdit>;
using ApplyEditReplyHandler = ReplyHandler<LSPApplyWorkspaceEditResponse>;
using SwitchSourceHeaderHandler = ReplyHandler<QString>;
using SemanticTokensReplyHandler = ReplyHandler<LSPSemanticTokens>;
using SemanticTokensDeltaReplyHandler = ReplyHandler<LSPSemanticTokensDelta>;
class LSPClientPlugin;
......@@ -147,6 +149,12 @@ public:
QList<LSPDiagnostic> diagnostics,
const QObject *context,
const CodeActionReplyHandler &h);
RequestHandle documentSemanticTokensFull(const QUrl &document, const QString requestId, const QObject *context, const SemanticTokensReplyHandler &h);
RequestHandle
documentSemanticTokensFullDelta(const QUrl &document, const QString requestId, const QObject *context, const SemanticTokensDeltaReplyHandler &h);
void executeCommand(const QString &command, const QJsonValue &args);
// sync
......@@ -164,7 +172,6 @@ Q_SIGNALS:
void showMessage(const LSPShowMessageParams &);
void logMessage(const LSPLogMessageParams &);
void publishDiagnostics(const LSPPublishDiagnosticsParams &);
void semanticHighlighting(const LSPSemanticHighlightingParams &);
// request = signal
void applyEdit(const LSPApplyWorkspaceEditParams &req, const ApplyEditReplyHandler &h, bool &handled);
......
/*
SPDX-FileCopyrightText: 2021 Waqar Ahmed <waqar.17a@gmail.com>
SPDX-License-Identifier: MIT
*/
#include "lspsemantichighlighting.h"
#include <QColor>
#include <QString>
#include <QVector>
#include <KTextEditor/MovingInterface>
#include <KTextEditor/View>
#include <KTextEditor/Editor>
#include "semantic_tokens_legend.h"
#include <KSyntaxHighlighting/Theme>
void SemanticHighlighter::remove(const QUrl &url)
{
m_docUrlToResultId.remove(url);
auto &data = m_docSemanticInfo[url];
auto &movingRanges = data.movingRanges;
for (auto mr : movingRanges) {
delete mr;
}
m_docUrlToResultId.remove(url);
}
SemanticHighlighting::SemanticHighlighting(QObject *parent)
: QObject(parent)
void SemanticHighlighter::insert(const QUrl &url, const QString &resultId, const std::vector<uint32_t> &data)
{
themeChange(KTextEditor::Editor::instance());
connect(KTextEditor::Editor::instance(), &KTextEditor::Editor::configChanged, this, &SemanticHighlighting::themeChange);
m_docUrlToResultId[url] = resultId;