Rewrite audio thumbnails to only use FFmpeg's data and optimize memory usage on creation.

Related to #102

src/bin/bin.cpp

@@ -2517,13 +2517,13 @@ void Bin::slotEditClipCommand(const QString &id, const QMap<QString, QString> &o
     m_doc->commandStack()->push(command);
 }
 
-void Bin::reloadClip(const QString &id, bool reloadAudio)
+void Bin::reloadClip(const QString &id)
 {
     std::shared_ptr<ProjectClip> clip = m_itemModel->getClipByBinID(id);
     if (!clip) {
         return;
     }
-    clip->reloadProducer(false, false, reloadAudio);
+    clip->reloadProducer(false, false);
 }
 
 void Bin::reloadMonitorIfActive(const QString &id)
@@ -3947,11 +3947,11 @@ void Bin::reloadAllProducers(bool reloadThumbs)
         if (!xml.isNull()) {
             clip->setClipStatus(AbstractProjectItem::StatusWaiting);
             pCore->jobManager()->slotDiscardClipJobs(clip->clipId());
-            clip->discardAudioThumb(false);
+            clip->discardAudioThumb();
             // We need to set a temporary id before all outdated producers are replaced;
             int jobId = pCore->jobManager()->startJob<LoadJob>({clip->clipId()}, -1, QString(), xml);
             if (reloadThumbs) {
-                ThumbnailCache::get()->invalidateThumbsForClip(clip->clipId(), true);
+                ThumbnailCache::get()->invalidateThumbsForClip(clip->clipId());
             }
             pCore->jobManager()->startJob<ThumbJob>({clip->clipId()}, jobId, QString(), -1, true, true);
             if (KdenliveSettings::audiothumbnails()) {

src/bin/bin.h

@@ -226,7 +226,7 @@ public:
     const QString getDocumentProperty(const QString &key);
 
     /** @brief Ask MLT to reload this clip's producer  */
-    void reloadClip(const QString &id, bool reloadAudio = true);
+    void reloadClip(const QString &id);
 
     /** @brief refresh monitor (if clip changed)  */
     void reloadMonitorIfActive(const QString &id);

src/bin/projectclip.cpp

@@ -207,6 +207,50 @@ QString ProjectClip::getXmlProperty(const QDomElement &producer, const QString &
 void ProjectClip::updateAudioThumbnail()
 {
     emit audioThumbReady();
+    if (m_clipType == ClipType::Audio) {
+        QImage thumb = ThumbnailCache::get()->getThumbnail(m_binId, 0);
+        if (thumb.isNull()) {
+            int iconHeight = QFontInfo(qApp->font()).pixelSize() * 3.5;
+            QImage img(QSize(iconHeight * pCore->getCurrentDar(), iconHeight), QImage::Format_ARGB32);
+            img.fill(Qt::darkGray);
+            QMap <int, QString> streams = audioInfo()->streams();
+            QMap <int, int> channelsList = audioInfo()->streamChannels();
+            QPainter painter(&img);
+            QPen pen = painter.pen();
+            pen.setColor(Qt::white);
+            painter.setPen(pen);
+            int streamCount = 0;
+            qreal indicesPrPixel = qreal(getFramePlaytime()) / img.width();
+            double increment = qMax(1., 1. / qAbs(indicesPrPixel));
+            if (streams.count() > 0) {
+                double streamHeight = iconHeight / streams.count();
+                QMapIterator<int, QString> st(streams);
+                while (st.hasNext()) {
+                    st.next();
+                    int channels = channelsList.value(st.key());
+                    double channelHeight = (double) streamHeight / channels;
+                    const QVector <uint8_t> audioLevels = audioFrameCache(st.key());
+                    for (int channel = 0; channel < channels; channel++) {
+                        double y = (streamHeight * streamCount) + (channel * channelHeight) + channelHeight / 2;
+                        for (int i = 0; i <= img.width(); i++) {
+                            double j = i * increment;
+                            int idx = j * indicesPrPixel;
+                            idx += idx % channels;
+                            idx += channel;
+                            if (idx >= audioLevels.length() || idx < 0) break;
+                                double level = audioLevels.at(idx) * channelHeight / 510.; // divide height by 510 (2*255) to get height
+                                painter.drawLine(i, y - level, i, y + level);
+                            }
+                    }
+                    streamCount++;
+                }
+            }
+            thumb = img;
+            // Cache thumbnail
+            ThumbnailCache::get()->storeThumbnail(m_binId, 0, thumb, true);
+        }
+        setThumbnail(thumb);
+    }
     if (!KdenliveSettings::audiothumbnails()) {
         return;
     }
@@ -309,7 +353,7 @@ size_t ProjectClip::frameDuration() const
     return (size_t)getFramePlaytime();
 }
 
-void ProjectClip::reloadProducer(bool refreshOnly, bool audioStreamChanged, bool reloadAudio)
+void ProjectClip::reloadProducer(bool refreshOnly, bool isProxy)
 {
     // we find if there are some loading job on that clip
     int loadjobId = -1;
@@ -319,7 +363,7 @@ void ProjectClip::reloadProducer(bool refreshOnly, bool audioStreamChanged, bool
         // In that case, we only want a new thumbnail.
         // We thus set up a thumb job. We must make sure that there is no pending LOADJOB
         // Clear cache first
-        ThumbnailCache::get()->invalidateThumbsForClip(clipId(), false);
+        ThumbnailCache::get()->invalidateThumbsForClip(clipId());
         pCore->jobManager()->discardJobs(clipId(), AbstractClipJob::THUMBJOB);
         m_thumbsProducer.reset();
         emit pCore->jobManager()->startJob<ThumbJob>({clipId()}, loadjobId, QString(), -1, true, true);
@@ -352,14 +396,11 @@ void ProjectClip::reloadProducer(bool refreshOnly, bool audioStreamChanged, bool
                 }
                 
             }
-            ThumbnailCache::get()->invalidateThumbsForClip(clipId(), reloadAudio);
+            ThumbnailCache::get()->invalidateThumbsForClip(clipId());
             int loadJob = pCore->jobManager()->startJob<LoadJob>({clipId()}, loadjobId, QString(), xml);
             emit pCore->jobManager()->startJob<ThumbJob>({clipId()}, loadJob, QString(), -1, true, true);
-            if (audioStreamChanged || hashChanged) {
+            if (!isProxy && hashChanged) {
                 discardAudioThumb();
-            } else {
-                // refresh bin/monitor mini thumb only
-                discardAudioThumb(true);
             }
             if (KdenliveSettings::audiothumbnails()) {
                 emit pCore->jobManager()->startJob<AudioThumbJob>({clipId()}, loadjobId, QString());
@@ -1124,7 +1165,7 @@ void ProjectClip::setProperties(const QMap<QString, QString> &properties, bool r
             setProducerProperty(QStringLiteral("_overwriteproxy"), 1);
             emit pCore->jobManager()->startJob<ProxyJob>({clipId()}, -1, QString());
         } else {
-            reloadProducer(refreshOnly, audioStreamChanged, audioStreamChanged || (!refreshOnly && !properties.contains(QStringLiteral("kdenlive:proxy"))));
+            reloadProducer(refreshOnly, properties.contains(QStringLiteral("kdenlive:proxy")));
         }
         if (refreshOnly) {
             if (auto ptr = m_model.lock()) {
@@ -1292,7 +1333,7 @@ int ProjectClip::audioChannels() const
     return audioInfo()->channels();
 }
 
-void ProjectClip::discardAudioThumb(bool miniThumbOnly)
+void ProjectClip::discardAudioThumb()
 {
     if (!m_audioInfo) {
         return;
@@ -1300,18 +1341,16 @@ void ProjectClip::discardAudioThumb(bool miniThumbOnly)
     pCore->jobManager()->discardJobs(clipId(), AbstractClipJob::AUDIOTHUMBJOB);
     QString audioThumbPath;
     QList <int> streams = m_audioInfo->streams().keys();
-    if (!miniThumbOnly) {
-        // Delete audio thumbnail data
-        for (int &st : streams) {
-            audioThumbPath = getAudioThumbPath(st);
-            if (!audioThumbPath.isEmpty()) {
-                QFile::remove(audioThumbPath);
-            }
+    // Delete audio thumbnail data
+    for (int &st : streams) {
+        audioThumbPath = getAudioThumbPath(st);
+        if (!audioThumbPath.isEmpty()) {
+            QFile::remove(audioThumbPath);
         }
     }
-    // Delete mini thumb
+    // Delete thumbnail
     for (int &st : streams) {
-        audioThumbPath = getAudioThumbPath(st, true);
+        audioThumbPath = getAudioThumbPath(st);
         if (!audioThumbPath.isEmpty()) {
             QFile::remove(audioThumbPath);
         }
@@ -1332,9 +1371,9 @@ int ProjectClip::getAudioStreamFfmpegIndex(int mltStream)
     return -1;
 }
 
-const QString ProjectClip::getAudioThumbPath(int stream, bool miniThumb)
+const QString ProjectClip::getAudioThumbPath(int stream)
 {
-    if (audioInfo() == nullptr && !miniThumb) {
+    if (audioInfo() == nullptr) {
         return QString();
     }
     bool ok = false;
@@ -1348,10 +1387,6 @@ const QString ProjectClip::getAudioThumbPath(int stream, bool miniThumb)
     }
     QString audioPath = thumbFolder.absoluteFilePath(clipHash);
     audioPath.append(QLatin1Char('_') + QString::number(stream));
-    if (miniThumb) {
-        audioPath.append(QStringLiteral(".png"));
-        return audioPath;
-    }
     int roundedFps = (int)pCore->getCurrentFps();
     audioPath.append(QStringLiteral("_%1_audio.png").arg(roundedFps));
     return audioPath;

src/bin/projectclip.h

@@ -81,7 +81,7 @@ protected:
 public:
     ~ProjectClip() override;
 
-    void reloadProducer(bool refreshOnly = false, bool audioStreamChanged = false, bool reloadAudio = true);
+    void reloadProducer(bool refreshOnly = false, bool isProxy = false);
 
     /** @brief Returns a unique hash identifier used to store clip thumbnails. */
     // virtual void hash() = 0;
@@ -185,9 +185,9 @@ public:
     /** @brief Returns the list of this clip's subclip's ids. */
     QStringList subClipIds() const;
     /** @brief Delete cached audio thumb - needs to be recreated */
-    void discardAudioThumb(bool miniThumbOnly = false);
+    void discardAudioThumb();
     /** @brief Get path for this clip's audio thumbnail */
-    const QString getAudioThumbPath(int stream, bool miniThumb = false);
+    const QString getAudioThumbPath(int stream);
     /** @brief Returns true if this producer has audio and can be splitted on timeline*/
     bool isSplittable() const;
 

src/jobs/audiothumbjob.cpp

@@ -120,46 +120,13 @@ bool AudioThumbJob::computeWithFFMPEG()
     if (!QFile::exists(filePath)) {
         return false;
     }
-    m_ffmpegProcess.reset(new QProcess);
-    QString thumbPath = m_binClip->getAudioThumbPath(m_audioStream, true);
-    int audioStreamIndex = m_binClip->getAudioStreamFfmpegIndex(m_audioStream);
-    if (!QFile::exists(thumbPath)) {
-        // Generate thumbnail used in monitor overlay
-        QStringList args = {QStringLiteral("-hide_banner"), QStringLiteral("-y"), QStringLiteral("-i"), QUrl::fromLocalFile(filePath).toLocalFile(), QString("-filter_complex")};
-        if (m_audioStream >= 0) {
-            args << QString("[a:%1]showwavespic=s=%2x%3:split_channels=1:scale=cbrt:colors=%4|%5").arg(audioStreamIndex).arg(m_thumbSize.width()).arg(m_thumbSize.height()).arg(KdenliveSettings::thumbColor1().name(), KdenliveSettings::thumbColor2().name());
-        } else {
-            // Only 1 audio stream in clip
-            args << QString("[a]showwavespic=s=%2x%3:split_channels=1:scale=cbrt:colors=%4|%5").arg(m_thumbSize.width()).arg(m_thumbSize.height()).arg(KdenliveSettings::thumbColor1().name(), KdenliveSettings::thumbColor2().name());
-        }
-        args << QStringLiteral("-frames:v") << QStringLiteral("1");
-        args << thumbPath;
-        qDebug()<<"=== FFARGS: "<<args;
-        connect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress, Qt::UniqueConnection);
-        connect(this, &AudioThumbJob::jobCanceled, [&] () {
-            if (m_ffmpegProcess) {
-                disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
-                m_ffmpegProcess->kill();
-            }
-            m_done = true;
-            m_successful = false;
-        });
-        m_ffmpegProcess->start(KdenliveSettings::ffmpegpath(), args);
-        m_ffmpegProcess->waitForFinished(-1);
-        disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
-        if (m_ffmpegProcess->exitStatus() != QProcess::CrashExit) {
-            if (m_dataInCache || !KdenliveSettings::audiothumbnails()) {
-                m_done = true;
-            }
-        }
-    } else if (!KdenliveSettings::audiothumbnails()) {
-        m_done = true;
-    }
+
     if (!KdenliveSettings::audiothumbnails()) {
         // We only wanted the thumb generation
         return m_done;
     }
-    if (!QFile::exists(m_cachePath) && !m_dataInCache && !m_done) {
+    int audioStreamIndex = m_binClip->getAudioStreamFfmpegIndex(m_audioStream);
+    if (!QFile::exists(m_cachePath) && !m_dataInCache) {
         // Generate timeline audio thumbnail data
         m_audioLevels.clear();
         std::vector<std::unique_ptr<QTemporaryFile>> channelFiles;
@@ -184,9 +151,9 @@ bool AudioThumbJob::computeWithFFMPEG()
         args << QStringLiteral("-filter_complex");
         if (m_channels == 1) {
             if (m_audioStream >= 0) {
-                args << QStringLiteral("[a:%1]aformat=channel_layouts=mono,%2=100").arg(audioStreamIndex).arg(isFFmpeg ? "aresample=async" : "sample_rates");
+                args << QStringLiteral("[a:%1]aformat=channel_layouts=mono,%2=100").arg(audioStreamIndex).arg(isFFmpeg ? "aresample=1500:async" : "sample_rates");
             } else {
-                args << QStringLiteral("[a]aformat=channel_layouts=mono,%1=100").arg(isFFmpeg ? "aresample=async" : "sample_rates");
+                args << QStringLiteral("[a]aformat=channel_layouts=mono,%1=100").arg(isFFmpeg ? "aresample=1500:async" : "sample_rates");
             }
             /*args << QStringLiteral("-map") << QStringLiteral("0:a%1").arg(m_audioStream > 0 ? ":" + QString::number(audioStreamIndex) : QString())*/
             args << QStringLiteral("-c:a") << QStringLiteral("pcm_s16le") << QStringLiteral("-frames:v") 
@@ -195,7 +162,7 @@ bool AudioThumbJob::computeWithFFMPEG()
         } else {
             QString aformat = QStringLiteral("[a%1]%2=100,channelsplit=channel_layout=%3")
                               .arg(audioStreamIndex >= 0 ? ":" + QString::number(audioStreamIndex) : QString(),
-                                   isFFmpeg ? "aresample=async" : "aformat=sample_rates",
+                                   isFFmpeg ? "aresample=1500:async" : "aformat=sample_rates",
                                    m_channels > 2 ? "5.1" : "stereo");
             for (int i = 0; i < m_channels; ++i) {
                 aformat.append(QStringLiteral("[0:%1]").arg(i));
@@ -279,15 +246,16 @@ bool AudioThumbJob::computeWithFFMPEG()
             }
             m_done = true;
             return true;
+        } else {
+            QString err = m_ffmpegProcess->readAllStandardError();
+            m_ffmpegProcess.reset();
+            // m_errorMessage += err;
+            // m_errorMessage.append(i18n("Failed to create FFmpeg audio thumbnails, we now try to use MLT"));
+            qWarning() << "Failed to create FFmpeg audio thumbs:\n" << err << "\n---------------------";
         }
     } else {
         m_done = true;
     }
-    QString err = m_ffmpegProcess->readAllStandardError();
-    m_ffmpegProcess.reset();
-    // m_errorMessage += err;
-    // m_errorMessage.append(i18n("Failed to create FFmpeg audio thumbnails, we now try to use MLT"));
-    qWarning() << "Failed to create FFmpeg audio thumbs:\n" << err << "\n---------------------";
     return m_done;
 }
 
@@ -333,20 +301,6 @@ bool AudioThumbJob::startJob()
         return false;
     }
     m_lengthInFrames = m_prod->get_length(); // Multiply this if we want more than 1 sample per frame
-    int thumbResolution = 3000;
-    
-    // Increase audio thumb resolution for longer clips to get a better resolution
-    if (m_lengthInFrames > 10000) {
-        // More than 10 minutes at 25fps
-        if (m_lengthInFrames > 90000) {
-            // More than 1 hour at 25fps
-            thumbResolution = 10000;
-        } else {
-            thumbResolution = 6000;
-        }
-    }
-    m_thumbSize = QSize(thumbResolution, 1000 / pCore->getCurrentDar());
-
     m_frequency = m_binClip->audioInfo()->samplingRate();
     m_frequency = m_frequency <= 0 ? 48000 : m_frequency;
 
@@ -354,12 +308,16 @@ bool AudioThumbJob::startJob()
     m_channels = m_channels <= 0 ? 2 : m_channels;
 
     QMap <int, QString> streams = m_binClip->audioInfo()->streams();
+    QMap <int, int> audioChannels = m_binClip->audioInfo()->streamChannels();
     QMapIterator<int, QString> st(streams);
     m_done = true;
     ClipType::ProducerType type = m_binClip->clipType();
     while (st.hasNext()) {
         st.next();
         int stream = st.key();
+        if (audioChannels.contains(stream)) {
+            m_channels = audioChannels.value(stream);
+        }
         // Generate one thumb per stream
         m_audioStream = stream;
         m_cachePath = m_binClip->getAudioThumbPath(stream);
@@ -427,26 +385,13 @@ bool AudioThumbJob::commitResult(Fun &undo, Fun &redo)
     if (!m_successful) {
         return false;
     }
-    QImage oldImage;
-    QImage result;
-    if (m_binClip->clipType() == ClipType::Audio) {
-        oldImage = m_binClip->thumbnail(200, 200 / pCore->getCurrentDar()).toImage();
-        result = ThumbnailCache::get()->getAudioThumbnail(m_clipId).scaled(200, 200 / pCore->getCurrentDar());
-    }
 
-    // note that the image is moved into lambda, it won't be available from this class anymore
-    auto operation = [clip = m_binClip, image = std::move(result)]() {
+    auto operation = [clip = m_binClip]() {
         clip->updateAudioThumbnail();
-        if (!image.isNull() && clip->clipType() == ClipType::Audio) {
-            clip->setThumbnail(image);
-        }
         return true;
     };
-    auto reverse = [clip = m_binClip, image = std::move(oldImage)]() {
+    auto reverse = [clip = m_binClip]() {
         clip->updateAudioThumbnail();
-        if (!image.isNull() && clip->clipType() == ClipType::Audio) {
-            clip->setThumbnail(image);
-        }
         return true;
     };
     bool ok = operation();

src/jobs/audiothumbjob.hpp

@@ -64,12 +64,9 @@ protected:
 private:
     std::shared_ptr<ProjectClip> m_binClip;
     std::shared_ptr<Mlt::Producer> m_prod;
-    QString m_miniThumbPath;
     QString m_cachePath;
-    QSize m_thumbSize;
     bool m_dataInCache;
     bool m_thumbInCache;
-
     bool m_done{false}, m_successful{false};
     int m_channels, m_frequency, m_lengthInFrames, m_audioStream;
     QVector <uint8_t>m_audioLevels;

src/jobs/proxyclipjob.cpp

@@ -377,14 +377,14 @@ bool ProxyJob::commitResult(Fun &undo, Fun &redo)
         binClip->setProducerProperty(QStringLiteral("_overwriteproxy"), QString());
         const QString dest = binClip->getProducerProperty(QStringLiteral("kdenlive:proxy"));
         binClip->setProducerProperty(QStringLiteral("resource"), dest);
-        pCore->bin()->reloadClip(clipId, false);
+        pCore->bin()->reloadClip(clipId);
         return true;
     };
     auto reverse = [clipId = m_clipId]() {
         auto binClip = pCore->projectItemModel()->getClipByBinID(clipId);
         const QString dest = binClip->getProducerProperty(QStringLiteral("kdenlive:originalurl"));
         binClip->setProducerProperty(QStringLiteral("resource"), dest);
-        pCore->bin()->reloadClip(clipId, false);
+        pCore->bin()->reloadClip(clipId);
         return true;
     };
     bool ok = operation();

src/lib/audio/audioStreamInfo.cpp

@@ -116,7 +116,12 @@ QMap <int, QString> AudioStreamInfo::streams() const
     return m_audioStreams;
 }
 
-QList <int> AudioStreamInfo::streamChannels() const
+QMap <int, int> AudioStreamInfo::streamChannels() const
+{
+    return m_audioChannels;
+}
+
+QList <int> AudioStreamInfo::activeStreamChannels() const
 {
     if (m_activeStreams.size() == 1 && m_activeStreams.contains(INT_MAX)) {
         return m_audioChannels.values();

src/lib/audio/audioStreamInfo.h

@@ -31,7 +31,9 @@ public:
     /** @brief returns a list of audio stream index > stream description */
     QMap <int, QString> streams() const;
     /** @brief returns a list of audio stream index > channels per stream */
-    QList <int> streamChannels() const;
+    QMap <int, int> streamChannels() const;
+    /** @brief returns a list of audio channels per active stream */
+    QList <int> activeStreamChannels() const;
     /** @brief returns a list of enabled audio stream indexes > stream description */
     QMap <int, QString> activeStreams() const;
     int bitrate() const;

src/mltcontroller/clipcontroller.cpp

@@ -1049,7 +1049,7 @@ QList <int> ClipController::activeStreamChannels() const
     if (!audioInfo()) {
         return QList <int>();
     }
-    return audioInfo()->streamChannels();
+    return audioInfo()->activeStreamChannels();
 }
 
 QMap <int, QString> ClipController::activeStreams() const

src/utils/thumbnailcache.cpp

@@ -223,7 +223,7 @@ void ThumbnailCache::saveCachedThumbs(QStringList keys)
     }
 }
 
-void ThumbnailCache::invalidateThumbsForClip(const QString &binId, bool reloadAudio)
+void ThumbnailCache::invalidateThumbsForClip(const QString &binId)
 {
     QMutexLocker locker(&m_mutex);
     if (m_storedVolatile.find(binId) != m_storedVolatile.end()) {
@@ -243,16 +243,7 @@ void ThumbnailCache::invalidateThumbsForClip(const QString &binId, bool reloadAu
     if (ok && m_storedOnDisk.find(binId) != m_storedOnDisk.end()) {
         // Remove persistent cache
         for (int pos : m_storedOnDisk.at(binId)) {
-            if (pos < 0) {
-                if (reloadAudio) {
-                    auto key = getAudioKey(binId, &ok);
-                    if (ok) {
-                        for (const QString &p : qAsConst(key)) {
-                            QFile::remove(audioThumbFolder.absoluteFilePath(p));
-                        }
-                    }
-                }
-            } else {
+            if (pos >= 0) {
                 auto key = getKey(binId, pos, &ok);
                 if (ok) {
                     QFile::remove(thumbFolder.absoluteFilePath(key));

src/utils/thumbnailcache.hpp

@@ -71,7 +71,7 @@ public:
     void storeThumbnail(const QString &binId, int pos, const QImage &img, bool persistent = false);
 
     /* @brief Removes all the thumbnails for a given clip */
-    void invalidateThumbsForClip(const QString &binId, bool reloadAudio);
+    void invalidateThumbsForClip(const QString &binId);
 
     /* @brief Save all cached thumbs to disk */
     void saveCachedThumbs(QStringList keys);