Commit 39354287 authored by Andreas Cord-Landwehr's avatar Andreas Cord-Landwehr
Browse files

Introduce translate-shell integration for editor

Revisit editor integration and provide adapter to utilize
translate-shell for getting translations from the internet. This
replaces the completly broken existing implementation (due to changed
webservice APIs) and also makes the whole integration async.
parent 6431930f
......@@ -10,20 +10,21 @@ find_package(Qt5 5.2.0 CONFIG REQUIRED Test)
# set( EXECUTABLE_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR} )
set( unittest_INCLUDE_DIRS
../src
../src/practice
../src/collection
include_directories(
../src
../src/practice
../src/editor
../src/collection
)
set(unittest_LINK_LIBS
Qt5::Test
${parley_LINK_LIBS}
parley_LIB
Qt5::Test
${parley_LINK_LIBS}
parley_LIB
)
set(parley_unittest_helpers
parleyunittestutilities.cpp
parleyunittestutilities.cpp
)
macro(PARLEY_GUI_UNITTESTS)
......@@ -51,6 +52,7 @@ include_directories( ${unittest_INCLUDE_DIRS})
parley_non_gui_unittests(
testentrytest.cpp
sessionmanagerfixedtest.cpp
testentrytest.cpp
sessionmanagerfixedtest.cpp
translateshelltest.cpp
)
/*
SPDX-FileCopyrightText: 2021 Andreas Cord-Landwehr <cordlandwehr@kde.org>
SPDX-License-Identifier: LGPL-2.1-or-later
*/
#include "translateshelltest.h"
#include "translateshelladapter.h"
#include <QProcess>
#include <QSignalSpy>
// LC_ALL=C trans -l en -s en -t de homework -show-alternatives=n -show-original=n -show-languages=n -show-original-dictionary=n -no-warn
static const char *s_homework_en_de = R"""(Hausaufgaben
Definitions of homework
noun
(die) Hausaufgaben
homework
(die) Heimarbeit
homework, outwork
(die) Hausaufgabe
homework, homework assignment, assignment
(die) Hausarbeit
housework, homework, homework assignment, assignment
(die) Aufgabe
task, duty, mission, job, function, homework
)""";
// translation request of untranslatable string
// LC_ALL=C trans -l en -s en -t de FOOBAA -show-alternatives=n -show-original=n -show-languages=n -show-original-dictionary=n -no-warn
static const char *s_foobaa_en_de = R"""(FOOBAA
Translations of FOOBAA
)""";
// translation request of string with with special char, where warning "Did you mean: homework" is skipped with argument
// LC_ALL=C trans -l en -s en -t de FOOBAA -show-alternatives=n -show-original=n -show-languages=n -show-original-dictionary=n -no-warn
static const char *s_homework_special_char_en_de = R"""(_Hausaufgaben
Definitions of _homework
noun
(die) Hausaufgaben
homework
(die) Heimarbeit
homework, outwork
(die) Hausaufgabe
homework, homework assignment, assignment
(die) Hausarbeit
housework, homework, homework assignment, assignment
(die) Aufgabe
task, duty, mission, job, function, homework
)""";
// LC_ALL=C trans -l en -s en -t de run -show-alternatives=n -show-original=n -show-languages=n -show-original-dictionary=n -no-warn
static const char *s_run_en_de = R"""(Lauf
Definitions of run
noun
(der) Run
run
(der) Lauf
running, run, course, barrel, race, operation
(die) Auflage
edition, circulation, rest, run, overlay, condition
(die) Fahrt
ride, trip, journey, tour, run, voyage
(die) Laufzeit
term, period of validity, operational time, running time, run-time
(die) Strecke
route, distance, track, line, stretch, run
(die) Serie
series, serial, set, run, string, succession
(der) Ansturm
rush, stampede, onslaught, run, onrush, crowd
(die) Folge
episode, result, sequence, consequence, order, succession
(die) Reihe
series, row, number, range, set, line
(die) Sequenz
sequence, progression, run, flush
(das) Gehege
enclosure, reserve, pen, preserve, compound, run
(der) Hühnerhof
run
(die) Masche
mesh, stitch, ploy, trick, pitch, run
(die) Laufmasche
run, ladder
(die) Spielzeit
season, playing time, period, run, inning
(der) Ausflug
tour, trip, excursion, outing, flight, hike
(die) Tendenz
trend, tendency, bias, direction, drift, run
(der) Flug
flight
verb
laufen
run, walk, go, operate, work, race
verlaufen
run, proceed, go off
starten
start, launch, take off, set off, power up, blast off
führen
lead, run, carry, guide, conduct, keep
ablaufen
run, expire, drain off, run out, pass, go off
fahren
drive, ride, run, move, pass, motor
rennen
race, run, sprint
durchführen
lead through, take through, run through, carry out, implement, run
leiten
guide, conduct, lead, direct, manage, run
verkehren
operate, run, associate, have intercourse, consort, haunt
auslaufen
leak, run out, run, drain, leak out, sail
fließen
flow, pass, run, move, stream, circulate
unterhalten
support, maintain, keep, operate, entertain, amuse
verwalten
manage, administer, maintain, govern, hold, conduct
ausgehen
go out, go, start, assume, emanate, run out
rinnen
run, stream, pour
halten
keep, hold, maintain, stop, stick, stay
steuern
control, manage, steer, drive, navigate, run
abwickeln
unwind, complete, carry out, transact, unroll, uncoil
schmeißen
throw, chuck, fling, sling, slam, bung
zerfließen
melt away, run
färben
color, dye, colour
verfließen
pass, run, go by, become blurred
tropfen
drop, drip, leak
einlassen
admit, let in, embed, set, run, bed in
tröpfeln
dribble, trickle, drip, run
triefen
drip, ooze, run, water, be dripping wet
plagen
plague, afflict, torment, infest, harass, run
)""";
// LC_ALL=C trans -l en -s en -t de nice -show-alternatives=n -show-original=n -show-languages=n -show-original-dictionary=n -no-warn
static const char *s_nice_en_de = R"""(nett
Definitions of nice
adjective
schön
beautiful, nice, lovely, good, pretty, fine
nett
nice, cute, kind, lovely, pleasant, neat
gut
good, fine, nice, solid, beneficial, sharp
hübsch
pretty, nice, lovely, fine, neat, comely
sauber
clean, cleanly, neat, tidy, nice, fresh
fein
fine, delicate, nice, subtle, sensitive, elegant
lieb
dear, sweet, kind, nice, good, lovely
sympathisch
nice, pleasant, likable, simpatico
lecker
delicious, yummy, nice, scrumptious, lovely, savory
genau
exact, accurate, precise, close, specific, detailed
heikel
delicate, tricky, difficult, awkward, fussy, ticklish
pingelig
picky, fussy, finicky, niggling, over-particular, nit-picking
anspruchsvoll
demanding, exacting, fastidious, ambitious, discriminating, challenging
)""";
// C_ALL=C trans -l en -s en -t de "this is a dog" -show-alternatives=y -show-original=n -show-languages=n -show-original-dictionary=n -no-warn
static const char *s_sentence_dog_en_de = R"""(das ist ein Hund
Translations of this is a dog
this is a dog
das ist ein Hund, dies ist ein Hund
)""";
void TranslateShellTest::translationShellOutputParserTest()
{
{
TranslateShellAdapter::Translation result = TranslateShellAdapter::parseTranslateShellResult(QString(s_homework_en_de));
QCOMPARE(result.m_suggestions, QStringList{"Hausaufgaben"});
}
{
TranslateShellAdapter::Translation result = TranslateShellAdapter::parseTranslateShellResult(QString(s_foobaa_en_de));
QCOMPARE(result.m_suggestions, QStringList{"FOOBAA"});
}
{
TranslateShellAdapter::Translation result = TranslateShellAdapter::parseTranslateShellResult(QString(s_homework_special_char_en_de));
QCOMPARE(result.m_suggestions, QStringList{"_Hausaufgaben"});
}
{
TranslateShellAdapter::Translation result = TranslateShellAdapter::parseTranslateShellResult(QString(s_run_en_de));
QCOMPARE(result.m_suggestions, QStringList{"Lauf"});
}
{
TranslateShellAdapter::Translation result = TranslateShellAdapter::parseTranslateShellResult(QString(s_nice_en_de));
QCOMPARE(result.m_suggestions, QStringList{"nett"});
}
{
TranslateShellAdapter::Translation result = TranslateShellAdapter::parseTranslateShellResult(QString(s_sentence_dog_en_de));
QCOMPARE(result.m_suggestions, QStringList{"das ist ein Hund"});
}
}
void TranslateShellTest::translateShellProcessInteractionTest()
{
// important: we cannot rely on correct answers from the webservice, which may stop answering our requests
// after a surprisingly small number of API interactions
QFuture<TranslateShellAdapter::Translation> translation = TranslateShellAdapter::translateAsync("haus", "de", "en");
translation.waitForFinished();
if (translation.result().m_suggestions.size() == 0) {
QWARN("did not receive any translation results");
return;
}
if (translation.result().m_suggestions.first() != "House") { // translation "haus" -> "House" is expected to be stable
QWARN("translation result differes from expectation");
}
}
QTEST_GUILESS_MAIN(TranslateShellTest);
/*
SPDX-FileCopyrightText: 2021 Andreas Cord-Landwehr <cordlandwehr@kde.org>
SPDX-License-Identifier: LGPL-2.1-or-later
*/
#ifndef TRANSLATESHELLTEST_H
#define TRANSLATESHELLTEST_H
#include <QObject>
#include <QTest>
class TranslateShellTest : public QObject
{
Q_OBJECT
private Q_SLOTS:
void translationShellOutputParserTest();
void translateShellProcessInteractionTest();
};
#endif
......@@ -42,6 +42,7 @@ set(parley_LIB_SRCS
editor/summarywordwidget.cpp
editor/synonymwidget.cpp
editor/latexwidget.cpp
editor/translateshelladapter.cpp
practice/abstractwidget.cpp
practice/sessionmanagerbase.cpp
practice/sessionmanagercontinuous.cpp
......
......@@ -392,6 +392,5 @@ void VocabularyModel::resetLanguages()
void VocabularyModel::automaticTranslation(bool enabled)
{
qDebug() << "auto trans enabled: " << enabled;
Prefs::setAutomaticTranslation(true);
Prefs::setAutomaticTranslation(enabled);
}
......@@ -36,9 +36,9 @@ public:
enum roles { TranslationRole = Qt::UserRole, EntryRole, LocaleRole, AudioRole, ImageRole };
explicit VocabularyModel(QObject *parent = 0);
explicit VocabularyModel(QObject *parent = nullptr);
~VocabularyModel();
~VocabularyModel() override;
int rowCount(const QModelIndex &) const override;
int columnCount(const QModelIndex &) const override;
......
......@@ -472,9 +472,6 @@ void EditorWindow::removeGrades()
void EditorWindow::initScripts()
{
m_scriptManager = new ScriptManager(this);
m_vocabularyView->setTranslator(m_scriptManager->translator());
// Load scripts
m_scriptManager->loadScripts();
}
......
/*
SPDX-FileCopyrightText: 2021 Andreas Cord-Landwehr <cordlandwehr@kde.org>
SPDX-License-Identifier: LGPL-2.1-or-later
*/
#include "translateshelladapter.h"
#include <QDebug>
#include <QProcess>
#include <QtConcurrent>
bool TranslateShellAdapter::isTranslateShellAvailable() const
{
if (!m_translateShellAvailable) {
QProcess process;
process.start(QStringLiteral("trans"), QStringList() << "--version");
process.waitForFinished(1000);
if (process.error() != QProcess::ProcessError::FailedToStart) {
qDebug() << "Translateshell process found";
m_translateShellAvailable = true;
}
}
return m_translateShellAvailable;
}
QFuture<TranslateShellAdapter::Translation>
TranslateShellAdapter::translateAsync(const QString &word, const QString &sourceLanguage, const QString &targetLanguage)
{
QFuture<TranslateShellAdapter::Translation> result = QtConcurrent::run([word, sourceLanguage, targetLanguage]() {
return TranslateShellAdapter::translate(word, sourceLanguage, targetLanguage);
});
return result;
}
TranslateShellAdapter::Translation TranslateShellAdapter::translate(const QString &word, const QString &sourceLanguage, const QString &targetLanguage)
{
TranslateShellAdapter::Translation translation;
QProcess process;
process.start(QStringLiteral("trans"),
{"-l",
"en", // output language of CLI
"-t",
targetLanguage,
"-s",
sourceLanguage,
word,
"-no-ansi",
"-show-alternatives=y",
"-show-original=n",
"-show-languages=n",
"-show-original-dictionary=n",
"-show-dictionary=y",
"-no-warn"});
qDebug() << "RUNNING" << process.arguments();
process.waitForFinished();
if (process.exitCode() != 0) {
TranslateShellAdapter::Translation translation;
translation.m_error = true;
return {};
} else {
return TranslateShellAdapter::parseTranslateShellResult(process.readAll());
}
}
TranslateShellAdapter::Translation TranslateShellAdapter::parseTranslateShellResult(const QString &output)
{
const QStringList lines = output.split('\n');
Translation result;
if (lines.count() < 1) {
result.m_error = true;
return result;
}
// magic line parsing, since current version of translate-shell does not support stable CLI API
result.m_suggestions = QStringList() << lines.at(0).trimmed();
for (int i = 5; i < lines.count(); ++i) {
result.m_synonyms << lines.at(i).trimmed();
}
return result;
}
/*
SPDX-FileCopyrightText: 2021 Andreas Cord-Landwehr <cordlandwehr@kde.org>
SPDX-License-Identifier: LGPL-2.1-or-later
*/
#ifndef TRANSLATESHELLADAPTER_H
#define TRANSLATESHELLADAPTER_H
#include <QFuture>
#include <QProcessEnvironment>
#include <QStringList>
/**
* @brief Adapter to online translation services
*
* This class can be used to obtain translations of words via translatorshell.
* It is important to keep in mind that the whole integration is a best-effort integration. Always check first that
* translateshell is available and give reasonable information to the user. Moreover, some of the translateshell
* online APIs stop answering translation requests after a short time, which then leads to empty translation results,
* because no error responses are available.
*/
class TranslateShellAdapter
{
public:
/**
* @brief Result object of a translation request
*/
struct Translation {
QStringList m_suggestions;
QStringList m_synonyms;
QString phonetic;
bool m_error{false};
};
/**
* @brief Check if translateshell is supported
*
* Note that this call starts the translateshell --version option and thus you might not run this call in UI thread.
*
* @return true if application can be found, otherwise false
*/
bool isTranslateShellAvailable() const;
static TranslateShellAdapter::Translation parseTranslateShellResult(const QString &output);
/**
* @brief Asynchronous call to translate method
*
* This should be the preferred way to request translations. By using a watcher object the future object can be
* monitored in the main thread and it can be acted upon the availability of the translation without blocking UI.
*
* It is expected to check first if translateshell is available and supported with isTranslateShellAvailable().
* If process cannout be found or aborts, the result just provides an error flag.
*
* @param word the string that shall be translated, can also be a sequence of words
* @param sourceLanguage the source language identifier as ISO-639-1 language code
* @param targetLanguage the target language identifier as ISO-639-1 language code
* @return returns a list of available translations
*/
static QFuture<Translation> translateAsync(const QString &word, const QString &sourceLanguage, const QString &targetLanguage);
/**
* @brief Synchronous call to translate method
*
* Note that calling this method blocks the calling thread.
*
* It is expected to check first if translateshell is available and supported with isTranslateShellAvailable().
* If process cannout be found or aborts, the result just provides an error flag.
*
* @param word the string that shall be translated, can also be a sequence of words
* @param sourceLanguage the source language identifier as ISO-639-1 language code
* @param targetLanguage the target language identifier as ISO-639-1 language code
* @return returns a list of available translations
*/
static TranslateShellAdapter::Translation translate(const QString &word, const QString &sourceLanguage, const QString &targetLanguage);
private:
mutable bool m_translateShellAvailable{false};
};
#endif
......@@ -5,18 +5,19 @@
*/
#include "vocabularydelegate.h"
#include "vocabularyfilter.h"
#include "vocabularymodel.h"
#include "languagesettings.h"
#include "prefs.h"
#include "translateshelladapter.h"
#include "vocabularyfilter.h"
#include "vocabularymodel.h"
#include <KComboBox>
#include <KEduVocExpression>
#include <KEduVocWordtype>
#include <KLocalizedString>
#include <QCompleter>
#include <QDBusInterface>
#include <QDebug>
#include <QFutureWatcher>
#include <QHeaderView>
#include <QKeyEvent>
#include <QLineEdit>
......@@ -29,51 +30,22 @@ using namespace Editor;
VocabularyDelegate::VocabularyDelegate(QObject *parent)
: QItemDelegate(parent)
, m_translator(nullptr)
{
}
QSet<QString> VocabularyDelegate::getTranslations(const QModelIndex &index) const
{
if (Prefs::automaticTranslation() == false)
return QSet<QString>();
QSet<QString> translations; // translations of this column from all the other languages
int language = index.column() / VocabularyModel::EntryColumnsMAX;
QString toLanguage = m_doc->identifier(language).locale();
// iterate through all the Translation columns
for (int i = 0; i < index.model()->columnCount(index.parent()); i++) {
if (VocabularyModel::columnType(i) == VocabularyModel::Translation) { // translation column
QString fromLanguage = m_doc->identifier(VocabularyModel::translation(i)).locale();
QString word = index.model()->index(index.row(), i, QModelIndex()).data().toString();
if (fromLanguage != toLanguage) {
// qDebug() << fromLanguage << toLanguage << word;
// get the word translations and add them to the translations set
QSet<QString> *tr = m_translator->getTranslation(word, fromLanguage, toLanguage);
if (tr)
translations.unite(*(tr));
}
}
}
return translations;
}
QWidget *VocabularyDelegate::createEditor(QWidget *parent, const QStyleOptionViewItem &option, const QModelIndex &index) const
{
Q_UNUSED(option); /// as long as it's unused
Q_UNUSED(option) /// as long as it's unused
if (!index.isValid()) {
return 0;
return nullptr;
}
switch (VocabularyModel::columnType(index.column())) {
case VocabularyModel::WordClass: {
if (!m_doc)
return 0;
if (!m_doc) {
return nullptr;
}
KComboBox *wordTypeCombo = new KComboBox(parent);
WordTypeBasicModel *basicWordTypeModel = new WordTypeBasicModel(parent);
......@@ -95,28 +67,45 @@ QWidget *VocabularyDelegate::createEditor(QWidget *parent, const QStyleOptionVie
return wordTypeCombo;
}
case VocabularyModel::Translation:
if (!m_doc || !m_translator)
return 0;
if (VocabularyModel::columnType(index.column()) == VocabularyModel::Translation) {
// get the translations of this word (fetch only with the help of scripts, if enabled)
QSet<QString> translations = getTranslations(index);
// create combo box
// if there is only one word and that is the suggestion word (in translations) then don't create the combobox
if (!translations.isEmpty() && !(translations.size() == 1 && (*translations.begin()) == index.model()->data(index, Qt::DisplayRole).toString())) {
KComboBox *translationCombo = new KComboBox(parent);
translationCombo->setFrame(false);
translationCombo->addItems(translations.values());
translationCombo->setEditable(true);
translationCombo->setFont(index.model()->data(index, Qt::FontRole).value<QFont>());
translationCombo->setEditText(index.model()->data(index, Qt::DisplayRole).toString());
translationCombo->completionObject()->setItems(translations.values());
return translationCombo;
case VocabularyModel::Translation: {
if (!m_doc) {
return nullptr;
}