Add IPC support for communication with Glaxnimate

Fixes #1526

Double click on an animation clip in the timeline and this will open
Glaxnimate. The background of the animation in Kdenlive will also be
shown in Glaxnimate.

This requires Glaxnimate version >= 0.5.1

src/CMakeLists.txt

@@ -78,6 +78,7 @@ list(APPEND kdenlive_SRCS
     mltconnection.cpp
     statusbarmessagelabel.cpp
     undohelper.cpp
+    glaxnimateluncher.cpp
 )
 
 if(CRASH_AUTO_TEST)

src/bin/bin.cpp

@@ -19,6 +19,7 @@ SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
 #include "doc/kdenlivedoc.h"
 #include "doc/kthumb.h"
 #include "effects/effectstack/model/effectstackmodel.hpp"
+#include "glaxnimateluncher.h"
 #include "jobs/abstracttask.h"
 #include "jobs/audiolevelstask.h"
 #include "jobs/cliploadtask.h"
@@ -4126,7 +4127,7 @@ void Bin::slotOpenClipExtern()
             }
         }
         if (!KdenliveSettings::defaultimageapp().isEmpty()) {
-            openExternalApp(KdenliveSettings::defaultimageapp(), clip->url());
+            pCore->openExternalApp(KdenliveSettings::defaultimageapp(), {clip->url()});
         } else {
             KMessageBox::error(QApplication::activeWindow(), i18n("Please set a default application to open image files"));
         }
@@ -4143,44 +4144,19 @@ void Bin::slotOpenClipExtern()
             }
         }
         if (!KdenliveSettings::defaultaudioapp().isEmpty()) {
-            openExternalApp(KdenliveSettings::defaultaudioapp(), clip->url());
+            pCore->openExternalApp(KdenliveSettings::defaultaudioapp(), {clip->url()});
         } else {
             KMessageBox::error(QApplication::activeWindow(), i18n("Please set a default application to open audio files"));
         }
     } break;
     case ClipType::Animation: {
-        if (KdenliveSettings::glaxnimatePath().isEmpty()) {
-            QUrl url = KUrlRequesterDialog::getUrl(QUrl(), this, i18n("Enter path to the Glaxnimate application"));
-            if (!url.isEmpty()) {
-                KdenliveSettings::setGlaxnimatePath(url.toLocalFile());
-                KdenliveSettingsDialog *d = static_cast<KdenliveSettingsDialog *>(KConfigDialog::exists(QStringLiteral("settings")));
-                if (d) {
-                    d->updateExternalApps();
-                }
-            }
-        }
-        if (!KdenliveSettings::glaxnimatePath().isEmpty()) {
-            openExternalApp(KdenliveSettings::glaxnimatePath(), clip->url());
-        } else {
-            KMessageBox::error(QApplication::activeWindow(), i18n("Please set a path for the Glaxnimate application"));
-        }
+        GlaxnimateLuncher::instance().openFile(clip->url());
     } break;
     default:
         break;
     }
 }
 
-void Bin::openExternalApp(QString appPath, QString url)
-{
-    QStringList args;
-#if defined(Q_OS_MACOS)
-    args << QStringLiteral("-a") << appPath << QStringLiteral("--args");
-    appPath = QStringLiteral("open");
-#endif
-    args << url;
-    QProcess::startDetached(appPath, args);
-}
-
 /*
 void Bin::slotGotFilterJobResults(const QString &id, int startPos, int track, const stringMap &results, const stringMap &filterInfo)
 {

src/bin/bin.h

@@ -586,8 +586,6 @@ private:
     void showBinInfo();
     /** @brief Find all clip Ids that have a specific tag. */
     const QList<QString> getAllClipsWithTag(const QString &tag);
-    /** @brief Open a file using an external app. */
-    void openExternalApp(const QString appPath, const QString url);
 
 signals:
     void itemUpdated(std::shared_ptr<AbstractProjectItem>);

src/core.cpp

@@ -259,6 +259,20 @@ void Core::buildLumaThumbs(const QStringList &values)
     }
 }
 
+QString Core::openExternalApp(const QString &appPath, QStringList args)
+{
+#if defined(Q_OS_MACOS)
+    args.prepend({QStringLiteral("-a"), appPath, QStringLiteral("--args")});
+    appPath = QStringLiteral("open");
+#endif
+    QProcess process;
+    qDebug() << "Starting external app" << appPath << "with arguments" << args;
+    if (!process.startDetached(appPath, args)) {
+        return process.errorString();
+    }
+    return QString();
+}
+
 const QString Core::nameForLumaFile(const QString &filename)
 {
     static QMap<QString, QString> names;

src/core.h

@@ -101,6 +101,9 @@ public:
     /** @brief Returns a pointer to the main window. */
     MainWindow *window();
 
+    /** @brief Open a file using an external app. */
+    QString openExternalApp(const QString &appPath, const QStringList args);
+
     /** @brief Returns a pointer to the project manager. */
     ProjectManager *projectManager();
     /** @brief Returns a pointer to the current project. */

src/dialogs/clipcreationdialog.cpp

@@ -14,6 +14,7 @@ SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
 #include "core.h"
 #include "doc/docundostack.hpp"
 #include "doc/kdenlivedoc.h"
+#include "glaxnimateluncher.h"
 #include "kdenlive_debug.h"
 #include "kdenlivesettings.h"
 #include "project/dialogs/slideshowclip.h"
@@ -156,8 +157,7 @@ void ClipCreationDialog::createColorClip(KdenliveDoc *doc, const QString &parent
 
 void ClipCreationDialog::createAnimationClip(KdenliveDoc *doc, const QString &parentId)
 {
-    if (KdenliveSettings::glaxnimatePath().isEmpty()) {
-        KMessageBox::error(QApplication::activeWindow(), i18n("Please install Glaxnimate to edit Lottie animations."));
+    if (!GlaxnimateLuncher::instance().checkInstalled()) {
         return;
     }
     QDir dir(doc->projectDataFolder());
@@ -227,7 +227,7 @@ void ClipCreationDialog::createAnimationClip(KdenliveDoc *doc, const QString &pa
     QTextStream out(&file);
     out << templateJson;
     file.close();
-    QProcess::startDetached(KdenliveSettings::glaxnimatePath(), {fileName});
+    GlaxnimateLuncher::instance().openFile(fileName);
     // Add clip to project
     QDomDocument xml;
     QDomElement prod = xml.createElement(QStringLiteral("producer"));

src/glaxnimateluncher.cpp

+/*
+    SPDX-FileCopyrightText: 2022 Meltytech, LLC
+    SPDX-FileCopyrightText: 2022 Julius Künzel <jk.kdedev@smartlab.uber.space>
+
+    SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
+*/
+
+#include "glaxnimateluncher.h"
+#include "bin/projectclip.h"
+#include "bin/projectitemmodel.h"
+#include "core.h"
+#include "dialogs/kdenlivesettingsdialog.h"
+#include "doc/kthumb.h"
+#include "kdenlivesettings.h"
+#include "mainwindow.h"
+#include "timeline2/view/timelinewidget.h"
+
+#include <KConfigDialog>
+#include <KMessageBox>
+#include <KUrlRequesterDialog>
+#include <QLocalServer>
+#include <QLocalSocket>
+#include <QSharedMemory>
+
+bool GlaxnimateLuncher::checkInstalled()
+{
+    if (!KdenliveSettings::glaxnimatePath().isEmpty() && QFile(KdenliveSettings::glaxnimatePath()).exists()) {
+        return true;
+    }
+    QUrl url = KUrlRequesterDialog::getUrl(QUrl(), nullptr, i18n("Enter path to the Glaxnimate application"));
+    if (url.isEmpty() || !QFile(url.toLocalFile()).exists()) {
+        KMessageBox::error(QApplication::activeWindow(), i18n("You need enter a valid path to be able to edit Lottie animations."));
+        return false;
+    }
+    KdenliveSettings::setGlaxnimatePath(url.toLocalFile());
+    KdenliveSettingsDialog *d = static_cast<KdenliveSettingsDialog *>(KConfigDialog::exists(QStringLiteral("settings")));
+    if (d) {
+        d->updateExternalApps();
+    }
+    return true;
+}
+
+GlaxnimateLuncher &GlaxnimateLuncher::instance()
+{
+    static GlaxnimateLuncher instance;
+    return instance;
+}
+
+void GlaxnimateLuncher::reset()
+{
+    if (m_stream && m_socket && m_stream && QLocalSocket::ConnectedState == m_socket->state()) {
+        *m_stream << QString("clear");
+        m_socket->flush();
+    }
+    m_parent.reset();
+}
+
+void GlaxnimateLuncher::openFile(const QString &filename)
+{
+    QString error = pCore->openExternalApp(KdenliveSettings::glaxnimatePath(), {filename});
+    if (!error.isEmpty()) {
+        KMessageBox::detailedError(QApplication::activeWindow(), i18n("Failed to lunch Glaxnimate application"), error);
+        return;
+    }
+}
+
+void GlaxnimateLuncher::openClip(int clipId)
+{
+    if (!checkInstalled()) {
+        return;
+    }
+    m_parent.reset(new ParentResources);
+
+    m_parent->m_binClip = pCore->projectItemModel()->getClipByBinID(pCore->window()->getCurrentTimeline()->model()->getClipBinId(clipId));
+    if (m_parent->m_binClip->clipType() != ClipType::Animation) {
+        pCore->displayMessage(i18n("Item is not an animation clip"), ErrorMessage, 500);
+        return;
+    }
+
+    QString filename = m_parent->m_binClip->clipUrl();
+
+    if (m_server && m_socket && m_stream && QLocalSocket::ConnectedState == m_socket->state()) {
+        auto s = QString("open ").append(filename);
+        qDebug() << s;
+        *m_stream << s;
+        m_socket->flush();
+        m_parent->m_frameNum = -1;
+        return;
+    }
+    m_parent->m_clipId = clipId;
+    m_server.reset(new QLocalServer);
+    connect(m_server.get(), &QLocalServer::newConnection, this, &GlaxnimateLuncher::onConnect);
+    QString name = QString("kdenlive-%1").arg(QCoreApplication::applicationPid());
+    QStringList args = {"--ipc", name, filename};
+    /*QProcess childProcess;
+    QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
+    env.remove("LC_ALL");
+    childProcess.setProcessEnvironment(env);*/
+    QString error;
+    if (!m_server->listen(name)) {
+        qDebug() << "failed to start the IPC server:" << m_server->errorString();
+        m_server.reset();
+        args.clear();
+        args << filename;
+        qDebug() << "Run without --ipc";
+        error = pCore->openExternalApp(KdenliveSettings::glaxnimatePath(), args);
+        if (error.isEmpty()) {
+            m_sharedMemory.reset(new QSharedMemory(name));
+            return;
+        }
+    } else {
+        if (pCore->openExternalApp(KdenliveSettings::glaxnimatePath(), args).isEmpty()) {
+            m_sharedMemory.reset(new QSharedMemory(name));
+            return;
+        } else {
+            // This glaxnimate executable may not support --ipc
+            // XXX startDetached is not failing in this case, need something better
+            qDebug() << "Failed to start glaxnimate with the --ipc, trying without now";
+            m_server.reset();
+            args.clear();
+            args << filename;
+            qDebug() << "Run without --ipc";
+            error = pCore->openExternalApp(KdenliveSettings::glaxnimatePath(), args);
+        }
+    }
+    if (!error.isEmpty()) {
+        KMessageBox::detailedError(QApplication::activeWindow(), i18n("Failed to lunch Glaxnimate application"), error);
+        return;
+    }
+}
+
+void GlaxnimateLuncher::onConnect()
+{
+    m_socket = m_server->nextPendingConnection();
+    connect(m_socket, &QLocalSocket::readyRead, this, &GlaxnimateLuncher::onReadyRead);
+    connect(m_socket, &QLocalSocket::errorOccurred, this, &GlaxnimateLuncher::onSocketError);
+    m_stream.reset(new QDataStream(m_socket));
+    m_stream->setVersion(QDataStream::Qt_5_15);
+    *m_stream << QString("hello");
+    m_socket->flush();
+    m_server->close();
+    m_isProtocolValid = false;
+}
+
+void GlaxnimateLuncher::onReadyRead()
+{
+    if (!m_isProtocolValid) {
+        QString message;
+        *m_stream >> message;
+        qDebug() << message;
+        if (message.startsWith("version ") && message != "version 1") {
+            *m_stream << QString("bye");
+            m_socket->flush();
+            m_server->close();
+        } else {
+            m_isProtocolValid = true;
+        }
+    } else {
+        qreal time = -1.0;
+        for (int i = 0; i < 1000 && !m_stream->atEnd(); i++) {
+            *m_stream >> time;
+        }
+
+        // Only if the frame number is different
+        int frameNum = pCore->window()->getCurrentTimeline()->model()->getClipPosition(m_parent->m_clipId) + time -
+                       pCore->window()->getCurrentTimeline()->model()->getClipIn(m_parent->m_clipId);
+        if (frameNum != m_parent->m_frameNum) {
+            qDebug() << "glaxnimate time =" << time << "=> Kdenlive frameNum =" << frameNum;
+
+            // Get the image from MLT
+            pCore->window()->getCurrentTimeline()->model()->producer().get()->seek(frameNum);
+            QList<int> clips = m_parent->m_binClip->timelineInstances();
+            // Temporarily hide this title clip in timeline so that it does not appear when requesting background frame
+            pCore->temporaryUnplug(clips, true);
+            std::unique_ptr<Mlt::Frame> frame(pCore->window()->getCurrentTimeline()->model()->producer().get()->get_frame());
+            QImage temp = KThumb::getFrame(frame.get(), pCore->getCurrentFrameSize().width(), pCore->getCurrentFrameSize().height());
+            pCore->temporaryUnplug(clips, false);
+            if (copyToShared(temp)) {
+                m_parent->m_frameNum = frameNum;
+            }
+        }
+    }
+}
+
+void GlaxnimateLuncher::onSocketError(QLocalSocket::LocalSocketError socketError)
+{
+    switch (socketError) {
+    case QLocalSocket::PeerClosedError:
+        qDebug() << "Glaxnimate closed the connection";
+        m_stream.reset();
+        m_sharedMemory.reset();
+        break;
+    default:
+        qDebug() << "Glaxnimate IPC error:" << m_socket->errorString();
+    }
+}
+
+bool GlaxnimateLuncher::copyToShared(const QImage &image)
+{
+    if (!m_sharedMemory) {
+        return false;
+    }
+    qint32 sizeInBytes = image.sizeInBytes() + 4 * sizeof(qint32);
+    if (sizeInBytes > m_sharedMemory->size()) {
+        if (m_sharedMemory->isAttached()) {
+            m_sharedMemory->lock();
+            m_sharedMemory->detach();
+            m_sharedMemory->unlock();
+        }
+        // over-allocate to avoid recreating
+        if (!m_sharedMemory->create(sizeInBytes)) {
+            qDebug() << m_sharedMemory->errorString();
+            return false;
+        }
+    }
+    if (m_sharedMemory->isAttached()) {
+        m_sharedMemory->lock();
+
+        uchar *to = (uchar *)m_sharedMemory->data();
+        // Write the width of the image and move the pointer forward
+        qint32 width = image.width();
+        ::memcpy(to, &width, sizeof(width));
+        to += sizeof(width);
+
+        // Write the height of the image and move the pointer forward
+        qint32 height = image.height();
+        ::memcpy(to, &height, sizeof(height));
+        to += sizeof(height);
+
+        // Write the image format of the image and move the pointer forward
+        qint32 imageFormat = image.format();
+        ::memcpy(to, &imageFormat, sizeof(imageFormat));
+        to += sizeof(imageFormat);
+
+        // Write the bytes per line of the image and move the pointer forward
+        qint32 bytesPerLine = image.bytesPerLine();
+        ::memcpy(to, &bytesPerLine, sizeof(bytesPerLine));
+        to += sizeof(bytesPerLine);
+
+        // Write the raw data of the image and move the pointer forward
+        ::memcpy(to, image.constBits(), image.sizeInBytes());
+
+        m_sharedMemory->unlock();
+        if (m_stream && m_socket) {
+            *m_stream << QString("redraw");
+            m_socket->flush();
+        }
+        return true;
+    }
+    return false;
+}

src/glaxnimateluncher.h

+/*
+    SPDX-FileCopyrightText: 2022 Meltytech, LLC
+    SPDX-FileCopyrightText: 2022 Julius Künzel <jk.kdedev@smartlab.uber.space>
+
+    SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
+*/
+
+#pragma once
+
+#include <QLocalServer>
+#include <QLocalSocket>
+#include <QObject>
+#include <QPoint>
+#include <QSharedMemory>
+#include <mlt++/MltProducer.h>
+
+class ProjectClip;
+
+class GlaxnimateLuncher : public QObject
+{
+    Q_OBJECT
+
+    class ParentResources
+    {
+    public:
+        std::shared_ptr<ProjectClip> m_binClip;
+        int m_frameNum = -1;
+        int m_clipId = -1;
+    };
+
+public:
+    static GlaxnimateLuncher &instance();
+    bool checkInstalled();
+    void openFile(const QString &url);
+    void openClip(int clipId);
+
+private:
+    std::unique_ptr<ParentResources> m_parent;
+    std::unique_ptr<QDataStream> m_stream;
+    std::unique_ptr<QLocalServer> m_server;
+    std::unique_ptr<QSharedMemory> m_sharedMemory;
+    bool m_isProtocolValid = false;
+    QLocalSocket *m_socket;
+
+    bool copyToShared(const QImage &image);
+    void reset();
+
+private slots:
+    void onConnect();
+    void onReadyRead();
+    void onSocketError(QLocalSocket::LocalSocketError socketError);
+};

src/monitor/monitorproxy.cpp

@@ -224,7 +224,6 @@ void MonitorProxy::extractFrameToFile(int frame_position, const QStringList &pat
     const QString folderInfo = pathInfo.at(2);
     QSize finalSize = pCore->getCurrentFrameDisplaySize();
     QSize size = pCore->getCurrentFrameSize();
-    QImage img;
     int height = size.height();
     int width = size.width();
     if (path.isEmpty()) {

src/timeline2/view/qml/timeline.qml

@@ -1910,6 +1910,8 @@ Rectangle {
                                             timeline.ungrabHack()
                                             if(dragProxy.masterObject.itemType === ProducerType.Text || dragProxy.masterObject.itemType === ProducerType.TextTemplate) {
                                                 timeline.editTitleClip(dragProxy.draggedItem)
+                                            } else if (dragProxy.masterObject.itemType === ProducerType.Animation) {
+                                                timeline.editAnimationClip(dragProxy.draggedItem)
                                             } else {
                                                 timeline.editItemDuration(dragProxy.draggedItem)
                                             }

src/timeline2/view/timelinecontroller.cpp

@@ -23,6 +23,7 @@
 #include "doc/kdenlivedoc.h"
 #include "effects/effectsrepository.hpp"
 #include "effects/effectstack/model/effectstackmodel.hpp"
+#include "glaxnimateluncher.h"
 #include "kdenlivesettings.h"
 #include "lib/audio/audioEnvelope.h"
 #include "mainwindow.h"
@@ -3811,6 +3812,24 @@ void TimelineController::editTitleClip(int id)
     pCore->bin()->showTitleWidget(binClip);
 }
 
+void TimelineController::editAnimationClip(int id)
+{
+    if (id == -1) {
+        id = m_root->property("mainItemId").toInt();
+        if (id == -1) {
+            std::unordered_set<int> sel = m_model->getCurrentSelection();
+            if (!sel.empty()) {
+                id = *sel.begin();
+            }
+            if (id == -1 || !m_model->isItem(id) || !m_model->isClip(id)) {
+                pCore->displayMessage(i18n("No clip selected"), ErrorMessage, 500);
+                return;
+            }
+        }
+    }
+    GlaxnimateLuncher::instance().openClip(id);
+}
+
 QPoint TimelineController::selectionInOut() const
 {
     std::unordered_set<int> ids = m_model->getCurrentSelection();

src/timeline2/view/timelinecontroller.h

@@ -92,6 +92,9 @@ public:
     /** @brief Edit a title clip with a title widget
      */
     Q_INVOKABLE void editTitleClip(int itemId = -1);
+    /** @brief Edit an animation with Glaxnimate
+     */
+    Q_INVOKABLE void editAnimationClip(int itemId = -1);
 
     /** @brief Returns the topmost track containing a selected item (-1 if selection is embty) */
     Q_INVOKABLE int selectedTrack() const;