add a LaTeX unicode completion plugin

replaces LaTeX commands with the corresponding unicode symbols

addons/CMakeLists.txt

@@ -13,6 +13,7 @@ ecm_optional_add_subdirectory(kate-ctags)
 ecm_optional_add_subdirectory(katebuild-plugin)
 ecm_optional_add_subdirectory(katesql) 
 ecm_optional_add_subdirectory(konsole)
+ecm_optional_add_subdirectory(latexunicodecompletion)
 ecm_optional_add_subdirectory(lspclient) # Language Server Protocol (LSP) client plugin.
 ecm_optional_add_subdirectory(preview) # Live preview of sources in target format.
 ecm_optional_add_subdirectory(project) # Small & smart project manager.

addons/latexunicodecompletion/CMakeLists.txt

+include(ECMQtDeclareLoggingCategory)
+add_subdirectory(hat-trie)
+
+add_library(latexcompletionplugin MODULE)
+target_compile_definitions(latexcompletionplugin PRIVATE TRANSLATION_DOMAIN="latexcompletionplugin")
+ecm_qt_declare_logging_category(
+  latexcompletionplugin
+  HEADER logging.h
+  IDENTIFIER LATEXCOMPLETION
+  CATEGORY_NAME "katelatexcompletionplugin"
+)
+
+kde_target_enable_exceptions(latexcompletionplugin PRIVATE)
+target_link_libraries(latexcompletionplugin PRIVATE KF5::I18n KF5::TextEditor tsl::hat_trie)
+
+target_sources(
+  latexcompletionplugin
+  PRIVATE
+    latexcompletionplugin.cpp
+    completionmodel.cpp
+)
+
+# ensure we are able to load plugins pre-install, too, directories must match!
+set_target_properties(latexcompletionplugin PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/ktexteditor")
+install(TARGETS latexcompletionplugin DESTINATION ${KDE_INSTALL_PLUGINDIR}/ktexteditor)

addons/latexunicodecompletion/Messages.sh

+#! /bin/sh
+$XGETTEXT *.cpp -o $podir/katesnippetsplugin.pot

addons/latexunicodecompletion/completionmodel.cpp

+/*
+    SPDX-FileCopyrightText: 2021 Ilia Kats <ilia-kats@gmx.net>
+
+    SPDX-License-Identifier: LGPL-2.0-or-later
+*/
+
+#include "completionmodel.h"
+#include "completiontrie.h"
+#include "logging.h"
+
+#include <QIcon>
+#include <QRegularExpression>
+
+#include <KLocalizedString>
+#include <KTextEditor/Document>
+#include <KTextEditor/View>
+
+static const QRegularExpression latexexpr(QStringLiteral("\\\\:?[\\w]+:?$"),
+                                          QRegularExpression::DontCaptureOption); // no unicode here, LaTeX expressions are ASCII only
+
+LatexCompletionModel::LatexCompletionModel(QObject *parent)
+    : KTextEditor::CodeCompletionModel(parent)
+{
+    setHasGroups(false);
+}
+void LatexCompletionModel::completionInvoked(KTextEditor::View *view,
+                                             const KTextEditor::Range &range,
+                                             KTextEditor::CodeCompletionModel::InvocationType invocationType)
+{
+    Q_UNUSED(invocationType);
+    beginResetModel();
+    m_matches.clear();
+    auto word = view->document()->text(range);
+    if (!word.isEmpty() && word[0] == QLatin1Char('\\')) {
+        try {
+            auto prefixrange = completiontrie.equal_prefix_range(word.toStdString());
+            for (auto it = prefixrange.first; it != prefixrange.second; ++it) {
+                m_matches.push_back(QPair(QString::fromStdString(it.key()), &(*it)));
+            }
+        } catch (const std::exception &e) {
+            qCCritical(LATEXCOMPLETION) << "caught exception while generating completions for " << word;
+            qCCritical(LATEXCOMPLETION) << e.what();
+        } catch (...) {
+            qCCritical(LATEXCOMPLETION) << "caught exception while generating completions for " << word;
+        }
+    }
+    endResetModel();
+}
+
+bool LatexCompletionModel::shouldAbortCompletion(KTextEditor::View *view, const KTextEditor::Range &range, const QString &currentCompletion)
+{
+    if (view->cursorPosition() < range.start() || view->cursorPosition() > range.end())
+        return true;
+    return !latexexpr.match(currentCompletion).hasMatch();
+}
+
+KTextEditor::Range LatexCompletionModel::completionRange(KTextEditor::View *view, const KTextEditor::Cursor &position)
+{
+    auto text = view->document()->line(position.line());
+    KTextEditor::Cursor start = position;
+    int pos = text.left(position.column()).lastIndexOf(latexexpr);
+    if (pos >= 0)
+        start.setColumn(pos);
+    return KTextEditor::Range(start, position);
+}
+
+void LatexCompletionModel::executeCompletionItem(KTextEditor::View *view, const KTextEditor::Range &word, const QModelIndex &index) const
+{
+    view->document()->replaceText(word, data(index.sibling(index.row(), Postfix), Qt::DisplayRole).toString());
+}
+
+int LatexCompletionModel::rowCount(const QModelIndex &parent) const
+{
+    if (!parent.isValid()) {
+        return 1; // One root node to define the custom group
+    } else if (parent.parent().isValid()) {
+        return 0; // Completion-items have no children
+    } else {
+        return m_matches.size();
+    }
+}
+
+QModelIndex LatexCompletionModel::index(int row, int column, const QModelIndex &parent) const
+{
+    if (!parent.isValid()) {
+        if (row == 0) {
+            return createIndex(row, column, quintptr(0));
+        } else {
+            return QModelIndex();
+        }
+
+    } else if (parent.parent().isValid()) {
+        return QModelIndex();
+    }
+
+    if (row < 0 || row >= m_matches.size() || column < 0 || column >= ColumnCount) {
+        return QModelIndex();
+    }
+
+    return createIndex(row, column, 1);
+}
+
+QModelIndex LatexCompletionModel::parent(const QModelIndex &index) const
+{
+    if (index.internalId()) {
+        return createIndex(0, 0, quintptr(0));
+    } else {
+        return QModelIndex();
+    }
+}
+#include <iostream>
+QVariant LatexCompletionModel::data(const QModelIndex &index, int role) const
+{
+    if (role == UnimportantItemRole)
+        return false;
+    else if (role == InheritanceDepth)
+        return 1;
+
+    if (!index.parent().isValid()) { // header
+        switch (role) {
+        case Qt::DisplayRole:
+            return i18n("LaTeX completion");
+        case GroupRole:
+            return Qt::DisplayRole;
+        }
+    }
+
+    if (index.isValid() && m_matches.size()) {
+        auto symbol = m_matches[index.row()];
+        if (role == IsExpandable)
+            return true; // if it's not expandable, the description will often be cut off
+                         // because apprarently the ItemSelected role is not taken into account
+                         // when determining the completion widget width. So expanding is
+                         // the only way to make sure that the complete description is available.
+        else if (role == ItemSelected || role == ExpandingWidget)
+            return QStringLiteral("<table><tr><td>%1</td><td>%2</td></tr></table>").arg(symbol.second->codepoint, symbol.second->name);
+        else if (role == Qt::DisplayRole) {
+            if (index.column() == Name)
+                return symbol.first;
+            else if (index.column() == Postfix)
+                return symbol.second->chars;
+        } else if (index.column() == Icon && role == Qt::DecorationRole) {
+            static const QIcon icon(QIcon::fromTheme(QStringLiteral("texcompiler")).pixmap(QSize(16, 16)));
+            return icon;
+        }
+    }
+    return QVariant();
+}

addons/latexunicodecompletion/completionmodel.h

+/*
+    SPDX-FileCopyrightText: 2021 Ilia Kats <ilia-kats@gmx.net>
+
+    SPDX-License-Identifier: LGPL-2.0-or-later
+*/
+
+#ifndef KATELATEXCOMPLETIONMODEL_H
+#define KATELATEXCOMPLETIONMODEL_H
+
+#include <QModelIndex>
+
+#include <KTextEditor/CodeCompletionModel>
+#include <KTextEditor/CodeCompletionModelControllerInterface>
+#include <KTextEditor/Cursor>
+#include <KTextEditor/Range>
+
+namespace KTextEditor
+{
+class View;
+}
+
+struct Completion;
+class LatexCompletionModel : public KTextEditor::CodeCompletionModel, public KTextEditor::CodeCompletionModelControllerInterface
+{
+    Q_OBJECT
+    Q_INTERFACES(KTextEditor::CodeCompletionModelControllerInterface)
+public:
+    LatexCompletionModel(QObject *parent);
+    KTextEditor::Range completionRange(KTextEditor::View *view, const KTextEditor::Cursor &position) override;
+    bool shouldAbortCompletion(KTextEditor::View *view, const KTextEditor::Range &range, const QString &currentCompletion) override;
+    void completionInvoked(KTextEditor::View *view, const KTextEditor::Range &range, InvocationType invocationType) override;
+    void executeCompletionItem(KTextEditor::View *view, const KTextEditor::Range &word, const QModelIndex &index) const override;
+    inline KTextEditor::CodeCompletionModelControllerInterface::MatchReaction matchingItem(const QModelIndex &) override
+    {
+        return None;
+    };
+    int rowCount(const QModelIndex &parent) const override;
+    QModelIndex index(int row, int column, const QModelIndex &parent) const override;
+    QModelIndex parent(const QModelIndex &index) const override;
+    QVariant data(const QModelIndex &index, int role) const override;
+
+private:
+    QVector<QPair<QString, const Completion *>> m_matches;
+};
+#endif

addons/latexunicodecompletion/fetch_unicode_table.py

+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: 2021 Ilia Kats <ilia-kats@gmx.net>
+# SPDX-License-Identifier: LGPL-2.0-or-later
+
+
+JULIA_UNICODE_DOCUMENTATION_URL = "https://docs.julialang.org/en/v1/manual/unicode-input/"
+CONTAINER_ID = "documenter-page"
+OUTFNAME = "completiontrie.h"
+
+from urllib import request
+from html.parser import HTMLParser
+
+class JuliaUnicodeCompletionsParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.table = []
+        self._in_container = False
+        self._in_table = False
+        self._in_header = False
+        self._in_body = False
+        self._in_cell = False
+        self._finished = False
+
+        self._current_row = None
+
+    def handle_starttag(self, tag, attrs):
+        if self._finished:
+            return
+        if not self._in_container:
+            for a in attrs:
+                if a[0] == "id" and a[1] == CONTAINER_ID:
+                    self._in_container = True
+                    break
+        elif not self._in_table and tag == "table":
+            self._in_table = True
+        elif self._in_table:
+            if tag == "tr":
+                if not self._in_header and not self._in_body:
+                    self._in_header = True
+                else:
+                    self._in_body = True
+                    self._current_row = []
+            elif tag == "td" and self._in_body:
+                self._in_cell = True
+
+    def handle_data(self, data):
+        if self._finished:
+            return
+        if self._in_body:
+            self._current_row.append(data)
+
+    def handle_endtag(self, tag):
+        if self._finished:
+            return
+        if self._in_body:
+            if tag == "tr":
+                self.table.append(tuple(self._current_row))
+                self._current_row = []
+            elif tag == "table":
+                self._finished = True
+
+parser = JuliaUnicodeCompletionsParser()
+with request.urlopen(JULIA_UNICODE_DOCUMENTATION_URL) as page:
+    parser.feed(page.read().decode(page.headers.get_content_charset()))
+parser.close()
+
+parser.table.sort(key=lambda x: x[2])
+with open(OUTFNAME, "w") as out:
+    out.write("""\
+#include <tsl/htrie_map.h>
+#include <QString>
+struct Completion {
+    QString codepoint;
+    QString chars;
+    QString name;
+};
+
+static const tsl::htrie_map<char, Completion> completiontrie({
+""")
+
+    for i, completion in enumerate(parser.table):
+        latexsym = completion[2].replace("\\", "\\\\")
+        if i > 0:
+            out.write(",")
+        out.write(f"{{\n\"{latexsym}\",\n{{\n"
+                  f"    QStringLiteral(\"{completion[0]}\"),\n"
+                  f"    QStringLiteral(u\"{completion[1]}\"),\n"
+                  f"    QStringLiteral(\"{completion[3]}\")\n}}\n}}\n")
+    out.write("""\
+});
+""")

addons/latexunicodecompletion/hat-trie/.codecov.yml

+comment: off

addons/latexunicodecompletion/hat-trie/.travis.yml

+language: cpp
+
+before_install:
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get update -y -qq; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y -qq libboost-test-dev; fi
+
+compiler:
+  - clang
+  - gcc
+
+dist: trusty
+os:
+  - linux
+  - osx  
+
+script:
+  - cd tests
+  - mkdir build
+  - cd build
+  - cmake ..
+  - make
+  - ./tsl_hat_trie_tests

addons/latexunicodecompletion/hat-trie/CMakeLists.txt

+cmake_minimum_required(VERSION 3.1)
+
+project(tsl_hat_trie)
+
+add_library(tsl_hat_trie INTERFACE)
+# Use tsl::hat_trie as target, more consistent with other libraries conventions (Boost, Qt, ...)
+add_library(tsl::hat_trie ALIAS tsl_hat_trie)
+
+target_include_directories(tsl_hat_trie INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_sources(tsl_hat_trie INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/array-hash/array_growth_policy.h"
+                                      "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/array-hash/array_hash.h"
+                                      "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/array-hash/array_map.h"
+                                      "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/array-hash/array_set.h"
+                                      "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/htrie_hash.h"
+                                      "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/htrie_map.h"
+                                      "${CMAKE_CURRENT_SOURCE_DIR}/include/tsl/htrie_set.h")
+
+
+if(${CMAKE_VERSION} VERSION_GREATER "3.7") 
+    # Only available since version 3.8
+    target_compile_features(tsl_hat_trie INTERFACE cxx_std_11)
+endif()

addons/latexunicodecompletion/hat-trie/LICENSE

+MIT License
+
+Copyright (c) 2017 Thibaut
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

addons/latexunicodecompletion/hat-trie/appveyor.yml

+os: Visual Studio 2015
+
+platform:
+  - Win32
+  - x64
+
+configuration:
+  - Debug
+  - Release
+  
+build_script:
+  - set BOOST_ROOT=C:\Libraries\boost_1_62_0
+  - if %PLATFORM% == Win32 set BOOST_LIBRARYDIR=C:\Libraries\boost_1_62_0\lib32-msvc-14.0
+  - if %PLATFORM% == x64 set BOOST_LIBRARYDIR=C:\Libraries\boost_1_62_0\lib64-msvc-14.0
+  - cd tests
+  - mkdir build
+  - cd build
+  - if %PLATFORM% == Win32 cmake .. -G"Visual Studio 14 2015"
+  - if %PLATFORM% == x64 cmake .. -G"Visual Studio 14 2015 Win64"
+  - cmake --build . --config %CONFIGURATION%
+  
+test_script: 
+  - set PATH=%PATH%;%BOOST_LIBRARYDIR%
+  - .\%CONFIGURATION%\tsl_hat_trie_tests.exe
+  

addons/latexunicodecompletion/hat-trie/include/tsl/array-hash/array_growth_policy.h

+/**
+ * MIT License
+ * 
+ * Copyright (c) 2017 Tessil
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef TSL_ARRAY_GROWTH_POLICY_H
+#define TSL_ARRAY_GROWTH_POLICY_H 
+
+
+#include <algorithm>
+#include <array>
+#include <climits>
+#include <cmath>
+#include <cstddef>
+#include <iterator>
+#include <limits>
+#include <ratio>
+#include <stdexcept>
+
+
+namespace tsl {
+namespace ah {
+    
+/**
+ * Grow the hash table by a factor of GrowthFactor keeping the bucket count to a power of two. It allows
+ * the table to use a mask operation instead of a modulo operation to map a hash to a bucket.
+ * 
+ * GrowthFactor must be a power of two >= 2.
+ */
+template<std::size_t GrowthFactor>
+class power_of_two_growth_policy {
+public:
+    /**
+     * Called on the hash table creation and on rehash. The number of buckets for the table is passed in parameter.
+     * This number is a minimum, the policy may update this value with a higher value if needed (but not lower).
+     *
+     * If 0 is given, min_bucket_count_in_out must still be 0 after the policy creation and
+     * bucket_for_hash must always return 0 in this case.
+     */
+    explicit power_of_two_growth_policy(std::size_t& min_bucket_count_in_out) {
+        if(min_bucket_count_in_out > max_bucket_count()) {
+            throw std::length_error("The hash table exceeds its maxmimum size.");
+        }
+        
+        if(min_bucket_count_in_out > 0) {
+            min_bucket_count_in_out = round_up_to_power_of_two(min_bucket_count_in_out);
+            m_mask = min_bucket_count_in_out - 1;
+        }
+        else {
+            m_mask = 0;
+        }
+    }
+    
+    /**
+     * Return the bucket [0, bucket_count()) to which the hash belongs. 
+     * If bucket_count() is 0, it must always return 0.
+     */
+    std::size_t bucket_for_hash(std::size_t hash) const noexcept {
+        return hash & m_mask;
+    }
+    
+    /**
+     * Return the number of buckets that should be used on next growth.
+     */
+    std::size_t next_bucket_count() const {
+        if((m_mask + 1) > max_bucket_count() / GrowthFactor) {
+            throw std::length_error("The hash table exceeds its maxmimum size.");
+        }
+        
+        return (m_mask + 1) * GrowthFactor;
+    }
+    
+    /**
+     * Return the maximum number of buckets supported by the policy.
+     */
+    std::size_t max_bucket_count() const {
+        // Largest power of two.
+        return (std::numeric_limits<std::size_t>::max() / 2) + 1;
+    }
+    
+    /**
+     * Reset the growth policy as if it was created with a bucket count of 0.
+     * After a clear, the policy must always return 0 when bucket_for_hash is called.
+     */
+    void clear() noexcept {
+        m_mask = 0;
+    }
+    
+private:
+    static std::size_t round_up_to_power_of_two(std::size_t value) {
+        if(is_power_of_two(value)) {
+            return value;
+        }
+        
+        if(value == 0) {
+            return 1;
+        }
+            
+        --value;
+        for(std::size_t i = 1; i < sizeof(std::size_t) * CHAR_BIT; i *= 2) {
+            value |= value >> i;
+        }
+        
+        return value + 1;
+    }
+    
+    static constexpr bool is_power_of_two(std::size_t value) {
+        return value != 0 && (value & (value - 1)) == 0;
+    }
+    
+protected:
+    static_assert(is_power_of_two(GrowthFactor) && GrowthFactor >= 2, "GrowthFactor must be a power of two >= 2.");
+    
+    std::size_t m_mask;
+};
+
+
+/**
+ * Grow the hash table by GrowthFactor::num / GrowthFactor::den and use a modulo to map a hash
+ * to a bucket. Slower but it can be useful if you want a slower growth.
+ */
+template<class GrowthFactor = std::ratio<3, 2>>
+class mod_growth_policy {
+public:
+    explicit mod_growth_policy(std::size_t& min_bucket_count_in_out) {
+        if(min_bucket_count_in_out > max_bucket_count()) {
+            throw std::length_error("The hash table exceeds its maxmimum size.");
+        }
+        
+        if(min_bucket_count_in_out > 0) {
+            m_mod = min_bucket_count_in_out;
+        }
+        else {
+            m_mod = 1;
+        }
+    }
+    
+    std::size_t bucket_for_hash(std::size_t hash) const noexcept {