use UTF16 strings instead of UTF8

addons/latexunicodecompletion/autotests/testcompletiontable.cpp

@@ -8,7 +8,8 @@
 
 #include "completiontable.h"
 
-#include <cstring>
+#include <algorithm>
+#include <string>
 
 #include <QObject>
 #include <QTest>
@@ -20,7 +21,10 @@ private Q_SLOTS:
     void testSorting()
     {
         for (int i = 0; i < n_completions - 1; ++i) {
-            QVERIFY(strcmp(completiontable[i].completion, completiontable[i + 1].completion) < 0);
+            QVERIFY(std::char_traits<char16_t>::compare(completiontable[i].completion,
+                                                        completiontable[i + 1].completion,
+                                                        std::min(completiontable[i].completion_strlen, completiontable[i + 1].completion_strlen))
+                    < 0);
         }
     }
 };

addons/latexunicodecompletion/completionmodel.cpp

@@ -8,7 +8,7 @@
 #include "completiontable.h"
 
 #include <algorithm>
-#include <cstring>
+#include <string>
 
 #include <QIcon>
 #include <QRegularExpression>
@@ -16,10 +16,10 @@
 #include <KTextEditor/Document>
 #include <KTextEditor/View>
 
-bool startsWith(const Completion &comp, const std::string &prefix)
+bool startsWith(const Completion &comp, const std::u16string &prefix)
 {
     if (prefix.size() <= comp.completion_strlen)
-        return std::strncmp(prefix.data(), comp.completion, prefix.size()) == 0;
+        return std::char_traits<char16_t>::compare(prefix.data(), comp.completion, prefix.size()) == 0;
     return false;
 }
 
@@ -35,14 +35,14 @@ void LatexCompletionModel::completionInvoked(KTextEditor::View *view,
     Q_UNUSED(invocationType);
     beginResetModel();
     m_matches.first = m_matches.second = -1;
-    auto word = view->document()->text(range).toStdString();
+    auto word = view->document()->text(range).toStdU16String();
     const Completion *beginit = (Completion *)&completiontable;
     const Completion *endit = beginit + n_completions;
     if (!word.empty() && word[0] == QLatin1Char('\\')) {
-        auto prefixrangestart = std::lower_bound(beginit, endit, word, [](const Completion &a, const std::string &b) -> bool {
+        auto prefixrangestart = std::lower_bound(beginit, endit, word, [](const Completion &a, const std::u16string &b) -> bool {
             return startsWith(a, b) ? false : a.completion < b;
         });
-        auto prefixrangeend = std::upper_bound(beginit, endit, word, [](const std::string &a, const Completion &b) -> bool {
+        auto prefixrangeend = std::upper_bound(beginit, endit, word, [](const std::u16string &a, const Completion &b) -> bool {
             return startsWith(b, a) ? false : a < b.completion;
         });
         if (prefixrangestart != endit) {
@@ -99,12 +99,12 @@ QVariant LatexCompletionModel::data(const QModelIndex &index, int role) const
                          // the only way to make sure that the complete description is available.
         else if (role == ItemSelected || role == ExpandingWidget)
             return QStringLiteral("<table><tr><td>%1</td><td>%2</td></tr></table>")
-                .arg(QString::fromUtf8(completion.codepoint), QString::fromUtf8(completion.name));
+                .arg(QString::fromUtf16(completion.codepoint), QString::fromUtf16(completion.name));
         else if (role == Qt::DisplayRole) {
             if (index.column() == Name)
-                return QString::fromUtf8(completion.completion);
+                return QString::fromUtf16(completion.completion);
             else if (index.column() == Postfix)
-                return QString::fromUtf8(completion.chars);
+                return QString::fromUtf16(completion.chars);
         } else if (index.column() == Icon && role == Qt::DecorationRole) {
             static const QIcon icon(QIcon::fromTheme(QStringLiteral("texcompiler")));
             return icon;

addons/latexunicodecompletion/fetch_unicode_table.py

@@ -77,10 +77,10 @@ with open(OUTFNAME, "w", encoding="utf-8") as out:
 #include <QString>
 #include <QRegularExpression>
 struct Completion {{
-    const char *completion;
-    const char *codepoint;
-    const char *chars;
-    const char *name;
+    const char16_t *completion;
+    const char16_t *codepoint;
+    const char16_t *chars;
+    const char16_t *name;
     const uint16_t completion_strlen;
 }};
 
@@ -93,14 +93,14 @@ static constexpr Completion completiontable[] = {{
         for letter in completion[2][1:]:
             if letter not in wordchars:
                 completionchars.add(letter)
-        latexsymlength = len(completion[2].encode("utf-8"))
+        latexsymlength = len(completion[2].encode("utf-16")) - 2 # Python adds the BOM
         latexsym = completion[2].replace("\\", "\\\\")
         if i > 0:
             out.write(",")
-        out.write(f"{{\n    u8\"{latexsym}\",\n"
-                  f"    u8\"{completion[0]}\",\n"
-                  f"    u8\"{completion[1]}\",\n"
-                  f"    u8\"{completion[3]}\",\n"
+        out.write(f"{{\n    u\"{latexsym}\",\n"
+                  f"    u\"{completion[0]}\",\n"
+                  f"    u\"{completion[1]}\",\n"
+                  f"    u\"{completion[3]}\",\n"
                   f"    {latexsymlength}\n}}\n")
     out.write("""\
 };