Commit 585e3a55 authored by Jean-Baptiste Mardelle's avatar Jean-Baptiste Mardelle
Browse files

Speech to text: set clip monitor zone when a sentence is selected in text editor

parent f4ad299c
Pipeline #50185 passed with stage
in 10 minutes and 28 seconds
......@@ -8,7 +8,6 @@ import sys
import os
import wave
import subprocess
import srt
import json
import datetime
......@@ -23,7 +22,6 @@ if not os.path.exists(sys.argv[2]):
sample_rate=16000
model = Model(sys.argv[2])
rec = KaldiRecognizer(model, sample_rate)
process = subprocess.Popen(['ffmpeg', '-loglevel', 'quiet', '-i',
sys.argv[3],
'-ar', str(sample_rate) , '-ac', '1', '-f', 's16le', '-'],
......@@ -34,7 +32,7 @@ def transcribe():
results = []
subs = []
while True:
data = process.stdout.read(2000)
data = process.stdout.read(4000)
if len(data) == 0:
break
if rec.AcceptWaveform(data):
......
......@@ -41,6 +41,8 @@
#include <KIO/FileCopyJob>
#if KIO_VERSION > QT_VERSION_CHECK(5, 70, 0)
#include <KIO/OpenUrlJob>
#else
#include <KRun>
#endif
#include <KIO/JobUiDelegate>
#include <KArchiveDirectory>
......@@ -53,7 +55,6 @@ SpeechDialog::SpeechDialog(const std::shared_ptr<TimelineItemModel> &timeline, Q
setFont(QFontDatabase::systemFont(QFontDatabase::SmallestReadableFont));
setupUi(this);
buttonBox->button(QDialogButtonBox::Apply)->setText(i18n("Process"));
dict_info->hide();
speech_info->hide();
slotParseDictionaries();
button_add->setIcon(QIcon::fromTheme(QStringLiteral("list-add")));
......@@ -62,7 +63,7 @@ SpeechDialog::SpeechDialog(const std::shared_ptr<TimelineItemModel> &timeline, Q
connect(buttonBox->button(QDialogButtonBox::Apply), &QPushButton::clicked, [this, timeline, zone]() {
slotProcessSpeech(timeline, zone);
});
connect(dict_info, &KMessageWidget::linkActivated, [&](const QString &contents) {
connect(speech_info, &KMessageWidget::linkActivated, [&](const QString &contents) {
qDebug()<<"=== LINK CLICKED: "<<contents;
#if KIO_VERSION > QT_VERSION_CHECK(5, 70, 0)
auto *job = new KIO::OpenUrlJob(QUrl(contents));
......@@ -70,6 +71,8 @@ SpeechDialog::SpeechDialog(const std::shared_ptr<TimelineItemModel> &timeline, Q
// methods like setRunExecutables, setSuggestedFilename, setEnableExternalBrowser, setFollowRedirections
// exist in both classes
job->start();
#else
new KRun(QUrl(contents), this);
#endif
});
//TODO: check for the python scripts vosk and srt
......@@ -170,9 +173,9 @@ void SpeechDialog::getDictionary()
QString tmpFile;
if (!url.isLocalFile()) {
KIO::FileCopyJob *copyjob = KIO::file_copy(url, QUrl::fromLocalFile(QDir::temp().absoluteFilePath(url.fileName())));
dict_info->setMessageType(KMessageWidget::Information);
dict_info->setText(i18n("Downloading model..."));
dict_info->animatedShow();
speech_info->setMessageType(KMessageWidget::Information);
speech_info->setText(i18n("Downloading model..."));
speech_info->animatedShow();
connect(copyjob, &KIO::FileCopyJob::result, this, &SpeechDialog::processArchive);
/*if (copyjob->exec()) {
qDebug()<<"=== GOT REST: "<<copyjob->destUrl();
......@@ -209,33 +212,33 @@ void SpeechDialog::processArchive(KJob* job)
dir.mkdir(QStringLiteral("speechmodels"));
if (!dir.cd(QStringLiteral("speechmodels"))) {
qDebug()<<"=== /// CANNOT ACCESS SPEECH DICTIONARIES FOLDER";
dict_info->setMessageType(KMessageWidget::Warning);
dict_info->setText(i18n("Cannot access dictionary folder"));
speech_info->setMessageType(KMessageWidget::Warning);
speech_info->setText(i18n("Cannot access dictionary folder"));
return;
}
if (archive->open(QIODevice::ReadOnly)) {
dict_info->setText(i18n("Extracting archive..."));
speech_info->setText(i18n("Extracting archive..."));
const KArchiveDirectory *archiveDir = archive->directory();
if (!archiveDir->copyTo(dir.absolutePath())) {
qDebug()<<"=== Error extracting archive!!";
} else {
QFile::remove(archiveFile);
emit parseDictionaries();
dict_info->setMessageType(KMessageWidget::Positive);
dict_info->setText(i18n("New dictionary installed"));
speech_info->setMessageType(KMessageWidget::Positive);
speech_info->setText(i18n("New dictionary installed"));
}
} else {
qDebug()<<"=== CANNOT OPEN ARCHIVE!!";
}
} else {
qDebug()<<"=== JOB NOT FOUND!!";
dict_info->setMessageType(KMessageWidget::Warning);
dict_info->setText(i18n("Download error"));
speech_info->setMessageType(KMessageWidget::Warning);
speech_info->setText(i18n("Download error"));
}
} else {
qDebug()<<"=== GOT JOB ERROR: "<<job->error();
dict_info->setMessageType(KMessageWidget::Warning);
dict_info->setText(i18n("Download error %1", job->errorString()));
speech_info->setMessageType(KMessageWidget::Warning);
speech_info->setText(i18n("Download error %1", job->errorString()));
}
}
......@@ -249,9 +252,9 @@ void SpeechDialog::slotParseDictionaries()
if (!dir.cd(QStringLiteral("speechmodels"))) {
qDebug()<<"=== /// CANNOT ACCESS SPEECH DICTIONARIES FOLDER";
tabWidget->setCurrentIndex(1);
dict_info->setMessageType(KMessageWidget::Information);
dict_info->setText(i18n("Download dictionaries from: <a href=\"https://alphacephei.com/vosk/models\">https://alphacephei.com/vosk/models</a>"));
dict_info->animatedShow();
speech_info->setMessageType(KMessageWidget::Information);
speech_info->setText(i18n("Download dictionaries from: <a href=\"https://alphacephei.com/vosk/models\">https://alphacephei.com/vosk/models</a>"));
speech_info->animatedShow();
return;
}
QStringList dicts = dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot);
......@@ -259,11 +262,11 @@ void SpeechDialog::slotParseDictionaries()
language_box->addItems(dicts);
if (!dicts.isEmpty()) {
buttonBox->button(QDialogButtonBox::Apply)->setEnabled(true);
dict_info->animatedHide();
speech_info->animatedHide();
} else {
tabWidget->setCurrentIndex(1);
dict_info->setMessageType(KMessageWidget::Information);
dict_info->setText(i18n("Download dictionaries from: <a href=\"https://alphacephei.com/vosk/models\">https://alphacephei.com/vosk/models</a>"));
dict_info->animatedShow();
speech_info->setMessageType(KMessageWidget::Information);
speech_info->setText(i18n("Download dictionaries from: <a href=\"https://alphacephei.com/vosk/models\">https://alphacephei.com/vosk/models</a>"));
speech_info->animatedShow();
}
}
......@@ -47,15 +47,17 @@ TextBasedEdit::TextBasedEdit(QWidget *parent)
});
connect(button_start, &QPushButton::clicked, this, &TextBasedEdit::startRecognition);
listWidget->setWordWrap(true);
search_frame->setVisible(false);
connect(listWidget, &QListWidget::currentRowChanged, [this] (int ix) {
if (ix > -1) {
QListWidgetItem *item = listWidget->item(ix);
if (!item) {
return;
}
double ms = item->data(Qt::UserRole).toDouble();
qDebug()<<"=== SEEKING TO: "<<ms;
pCore->getMonitor(Kdenlive::ClipMonitor)->requestSeek(GenTime(ms).frames(pCore->getCurrentFps()));
double startMs = item->data(Qt::UserRole).toDouble();
double endMs = item->data(Qt::UserRole + 1).toDouble();
pCore->getMonitor(Kdenlive::ClipMonitor)->requestSeek(GenTime(startMs).frames(pCore->getCurrentFps()));
pCore->getMonitor(Kdenlive::ClipMonitor)->slotLoadClipZone(QPoint(GenTime(startMs).frames(pCore->getCurrentFps()), GenTime(endMs).frames(pCore->getCurrentFps())));
}
});
info_message->hide();
......@@ -189,18 +191,26 @@ void TextBasedEdit::slotProcessSpeech()
QJsonObject obj = loadDoc.object();
qDebug()<<"==== ITEM IS OBJECT";
if (!obj.isEmpty()) {
QListWidgetItem *item = new QListWidgetItem(obj["text"].toString(), listWidget);
QString itemText = obj["text"].toString();
QListWidgetItem *item = new QListWidgetItem(listWidget);
if (obj["result"].isObject()) {
qDebug()<<"==== RESULT IS OBJECT";
} else if (obj["result"].isArray()) {
qDebug()<<"==== RESULT IS ARRAY";
QJsonArray obj2 = obj["result"].toArray();
QJsonValue val = obj2.at(0);
QJsonValue val = obj2.first();
if (val.isObject() && val.toObject().keys().contains("start")) {
double ms = val.toObject().value("start").toDouble();
itemText.prepend(QString("%1: ").arg(pCore->timecode().getDisplayTimecode(GenTime(ms), false)));
item->setData(Qt::UserRole, ms);
}
val = obj2.last();
if (val.isObject() && val.toObject().keys().contains("end")) {
double ms = val.toObject().value("end").toDouble();
item->setData(Qt::UserRole + 1, ms);
}
}
item->setText(itemText);
}
} else if (loadDoc.isEmpty()) {
qDebug()<<"==== EMPTY OBJEC DOC";
......
......@@ -71,9 +71,6 @@
</property>
</spacer>
</item>
<item row="5" column="0" colspan="2">
<widget class="KMessageWidget" name="speech_info"/>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_2">
......@@ -81,14 +78,14 @@
<string>Dictionaries</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_3">
<item row="2" column="0">
<widget class="QToolButton" name="button_add">
<property name="text">
<string>...</string>
<item row="0" column="0" colspan="3">
<widget class="QListWidget" name="listWidget">
<property name="alternatingRowColors">
<bool>true</bool>
</property>
</widget>
</item>
<item row="2" column="2">
<item row="1" column="2">
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
......@@ -101,28 +98,25 @@
</property>
</spacer>
</item>
<item row="2" column="1">
<item row="1" column="1">
<widget class="QToolButton" name="button_delete">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item row="0" column="0" colspan="3">
<widget class="QListWidget" name="listWidget">
<property name="alternatingRowColors">
<bool>true</bool>
<item row="1" column="0">
<widget class="QToolButton" name="button_add">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item row="1" column="0" colspan="3">
<widget class="KMessageWidget" name="dict_info"/>
</item>
</layout>
</widget>
</widget>
</item>
<item row="1" column="0">
<item row="2" column="0">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
......@@ -132,6 +126,9 @@
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="KMessageWidget" name="speech_info"/>
</item>
</layout>
</widget>
<customwidgets>
......
......@@ -11,55 +11,102 @@
</rect>
</property>
<layout class="QGridLayout" name="gridLayout_2">
<item row="3" column="0">
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QComboBox" name="language_box"/>
</item>
</layout>
</item>
<item row="0" column="0" colspan="3">
<widget class="QListWidget" name="listWidget">
<property name="alternatingRowColors">
<bool>true</bool>
<item row="2" column="0" colspan="3">
<widget class="QFrame" name="edit_frame">
<property name="frameShape">
<enum>QFrame::StyledPanel</enum>
</property>
<property name="frameShadow">
<enum>QFrame::Raised</enum>
</property>
<layout class="QGridLayout" name="gridLayout_4">
<item row="0" column="0">
<widget class="QToolButton" name="button_delete">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QToolButton" name="button_up">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item row="0" column="2">
<widget class="QToolButton" name="button_down">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item row="0" column="3">
<spacer name="horizontalSpacer_2">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>244</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</widget>
</item>
<item row="1" column="0" colspan="4">
<widget class="QFrame" name="frame_position">
<item row="3" column="0" colspan="3">
<widget class="QFrame" name="search_frame">
<property name="frameShape">
<enum>QFrame::NoFrame</enum>
<enum>QFrame::StyledPanel</enum>
</property>
<property name="frameShadow">
<enum>QFrame::Raised</enum>
</property>
<layout class="QGridLayout" name="gridLayout">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>0</number>
</property>
<layout class="QGridLayout" name="gridLayout_3">
<item row="0" column="1">
<widget class="QToolButton" name="search_prev">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
<item row="0" column="0">
<widget class="QLineEdit" name="search_line"/>
</item>
<item row="0" column="2">
<widget class="QToolButton" name="search_next">
<property name="text">
<string>...</string>
</property>
</widget>
</item>
</layout>
</widget>
</item>
<item row="2" column="0" colspan="3">
<widget class="KMessageWidget" name="info_message"/>
<item row="4" column="0">
<widget class="QComboBox" name="language_box"/>
</item>
<item row="3" column="2">
<item row="4" column="2">
<widget class="QPushButton" name="button_start">
<property name="text">
<string>Start recognition</string>
</property>
</widget>
</item>
<item row="3" column="1">
<item row="0" column="0" colspan="3">
<widget class="QListWidget" name="listWidget">
<property name="alternatingRowColors">
<bool>true</bool>
</property>
</widget>
</item>
<item row="1" column="0" colspan="3">
<widget class="KMessageWidget" name="info_message"/>
</item>
<item row="4" column="1">
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment