audiothumbjob.cpp 18.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
/***************************************************************************
 *   Copyright (C) 2017 by Nicolas Carion                                  *
 *   This file is part of Kdenlive. See www.kdenlive.org.                  *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) version 3 or any later version accepted by the       *
 *   membership of KDE e.V. (or its successor approved  by the membership  *
 *   of KDE e.V.), which shall act as a proxy defined in Section 14 of     *
 *   version 3 of the license.                                             *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>. *
 ***************************************************************************/

#include "audiothumbjob.hpp"
#include "bin/projectclip.h"
#include "bin/projectitemmodel.h"
#include "core.h"
#include "doc/kdenlivedoc.h"
#include "doc/kthumb.h"
#include "kdenlivesettings.h"
#include "klocalizedstring.h"
#include "lib/audio/audioStreamInfo.h"
#include "macros.hpp"
#include "utils/thumbnailcache.hpp"
33

34
35
#include <QScopedPointer>
#include <QTemporaryFile>
Laurent Montel's avatar
Laurent Montel committed
36
#include <QProcess>
37
38
39
40
#include <memory>
#include <mlt++/MltProducer.h>

AudioThumbJob::AudioThumbJob(const QString &binId)
41
    : AbstractClipJob(AUDIOTHUMBJOB, binId, {ObjectType::BinClip, binId.toInt()})
42
    , m_ffmpegProcess(nullptr)
43
44
45
46
47
48
49
50
51
52
53
{
}

const QString AudioThumbJob::getDescription() const
{
    return i18n("Extracting audio thumb from clip %1", m_clipId);
}

bool AudioThumbJob::computeWithMlt()
{
    m_audioLevels.clear();
54
    m_errorMessage.clear();
55
56
57
58
59
60
61
62
63
    // MLT audio thumbs: slower but safer
    QString service = m_prod->get("mlt_service");
    if (service == QLatin1String("avformat-novalidate")) {
        service = QStringLiteral("avformat");
    } else if (service.startsWith(QLatin1String("xml"))) {
        service = QStringLiteral("xml-nogl");
    }
    QScopedPointer<Mlt::Producer> audioProducer(new Mlt::Producer(*m_prod->profile(), service.toUtf8().constData(), m_prod->get("resource")));
    if (!audioProducer->is_valid()) {
64
        m_errorMessage.append(i18n("Audio thumbs: cannot open file %1", m_prod->get("resource")));
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
        return false;
    }
    audioProducer->set("video_index", "-1");
    Mlt::Filter chans(*m_prod->profile(), "audiochannels");
    Mlt::Filter converter(*m_prod->profile(), "audioconvert");
    Mlt::Filter levels(*m_prod->profile(), "audiolevel");
    audioProducer->attach(chans);
    audioProducer->attach(converter);
    audioProducer->attach(levels);

    int last_val = 0;
    double framesPerSecond = audioProducer->get_fps();
    mlt_audio_format audioFormat = mlt_audio_s16;
    QStringList keys;
    keys.reserve(m_channels);
    for (int i = 0; i < m_channels; i++) {
        keys << "meta.media.audio_level." + QString::number(i);
    }
83
84
    double maxLevel = 1;
    QVector <double> mltLevels;
85
    for (int z = 0; z < m_lengthInFrames; ++z) {
86
        int val = int(100.0 * z / m_lengthInFrames);
87
88
89
90
91
92
        if (last_val != val) {
            emit jobProgress(val);
            last_val = val;
        }
        QScopedPointer<Mlt::Frame> mltFrame(audioProducer->get_frame());
        if ((mltFrame != nullptr) && mltFrame->is_valid() && (mltFrame->get_int("test_audio") == 0)) {
93
            int samples = mlt_audio_calculate_frame_samples(float(framesPerSecond), m_frequency, z);
94
95
            mltFrame->get_audio(audioFormat, m_frequency, m_channels, samples);
            for (int channel = 0; channel < m_channels; ++channel) {
96
97
98
                double lev = mltFrame->get_double(keys.at(channel).toUtf8().constData());
                mltLevels << lev;
                maxLevel = qMax(lev, maxLevel);
99
            }
100
        } else if (!mltLevels.isEmpty()) {
101
            for (int channel = 0; channel < m_channels; channel++) {
102
                mltLevels << mltLevels.last();
103
104
105
            }
        }
    }
106
107
    // Normalize
    for (double &v : mltLevels) {
Vincent Pinon's avatar
Vincent Pinon committed
108
        m_audioLevels << uchar(255 * v / maxLevel);
109
    }
110

111
112
113
114
115
116
    m_done = true;
    return true;
}

bool AudioThumbJob::computeWithFFMPEG()
{
117
118
119
120
    if (!KdenliveSettings::audiothumbnails()) {
        // We only wanted the thumb generation
        return m_done;
    }
121
    QString filePath = m_binClip->getOriginalUrl();
122
123
124
    if (!QFile::exists(filePath)) {
        return false;
    }
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    int audioMax = m_binClip->getProducerIntProperty(QStringLiteral("kdenlive:audio_max"));
    if (audioMax == 0) {
        // Calculate max audio level with ffmpeg
        QProcess ffmpeg;
        QStringList args = {QStringLiteral("-i"), filePath, QStringLiteral("-vn"), QStringLiteral("-af"), QStringLiteral("volumedetect"), QStringLiteral("-f"), QStringLiteral("null")};
#ifdef Q_OS_WIN
        args << QStringLiteral("-");
#else
        args << QStringLiteral("/dev/stdout");
#endif
        QObject::connect(&ffmpeg, &QProcess::readyReadStandardOutput, [&ffmpeg, this]() {
            QString output = ffmpeg.readAllStandardOutput();
            if (output.contains(QLatin1String("max_volume"))) {
                output = output.section(QLatin1String("max_volume:"), 1).simplified();
                output = output.section(QLatin1Char(' '), 0, 0);
                bool ok;
                double maxVolume = output.toDouble(&ok);
                if (ok) {
                    int aMax = qMax(1, qAbs(qRound(maxVolume)));
                    m_binClip->setProducerProperty(QStringLiteral("kdenlive:audio_max"), aMax);
                    QMetaObject::invokeMethod(pCore.get(), "setDocumentModified", Qt::QueuedConnection);
                } else {
                    m_binClip->setProducerProperty(QStringLiteral("kdenlive:audio_max"), -1);
                }
            }
        });
        ffmpeg.setProcessChannelMode(QProcess::MergedChannels);
        ffmpeg.start(KdenliveSettings::ffmpegpath(), args);
        ffmpeg.waitForFinished(-1);
    }
155
156
157

    int audioStreamIndex = m_binClip->getAudioStreamFfmpegIndex(m_audioStream);
    if (!QFile::exists(m_cachePath) && !m_dataInCache) {
158
        // Generate timeline audio thumbnail data
159
160
161
162
163
164
165
166
167
168
169
170
171
172
        m_audioLevels.clear();
        std::vector<std::unique_ptr<QTemporaryFile>> channelFiles;
        for (int i = 0; i < m_channels; i++) {
            std::unique_ptr<QTemporaryFile> channelTmpfile(new QTemporaryFile());
            if (!channelTmpfile->open()) {
                m_errorMessage.append(i18n("Audio thumbs: cannot create temporary file, check disk space and permissions\n"));
                return false;
            }
            channelTmpfile->close();
            channelFiles.emplace_back(std::move(channelTmpfile));
        }
        // Always create audio thumbs from the original source file, because proxy 
        // can have a different audio config (channels / mono/ stereo)
        QStringList args {QStringLiteral("-hide_banner"), QStringLiteral("-i"), QUrl::fromLocalFile(filePath).toLocalFile(), QStringLiteral("-progress")};
173
#ifdef Q_OS_WIN
174
        args << QStringLiteral("-");
175
#else
176
        args << QStringLiteral("/dev/stdout");
177
#endif
178
        bool isFFmpeg = KdenliveSettings::ffmpegpath().contains(QLatin1String("ffmpeg"));
179
        args << QStringLiteral("-filter_complex");
180
        if (m_channels == 1) {
181
            if (m_audioStream >= 0) {
182
                args << QStringLiteral("[a:%1]aformat=channel_layouts=mono,%2=100").arg(audioStreamIndex).arg(isFFmpeg ? "aresample=1500:async" : "sample_rates");
183
            } else {
184
                args << QStringLiteral("[a]aformat=channel_layouts=mono,%1=100").arg(isFFmpeg ? "aresample=1500:async" : "sample_rates");
185
186
187
            }
            /*args << QStringLiteral("-map") << QStringLiteral("0:a%1").arg(m_audioStream > 0 ? ":" + QString::number(audioStreamIndex) : QString())*/
            args << QStringLiteral("-c:a") << QStringLiteral("pcm_s16le") << QStringLiteral("-frames:v") 
188
             << QStringLiteral("1") << QStringLiteral("-y") << QStringLiteral("-f") << QStringLiteral("data")
189
             << channelFiles[0]->fileName();
190
        } else {
191
            QString aformat = QStringLiteral("[a%1]%2=100,channelsplit=channel_layout=%3")
Vincent Pinon's avatar
Vincent Pinon committed
192
                              .arg(audioStreamIndex >= 0 ? ":" + QString::number(audioStreamIndex) : QString(),
193
                                   isFFmpeg ? "aresample=1500:async" : "aformat=sample_rates",
Vincent Pinon's avatar
Vincent Pinon committed
194
                                   m_channels > 2 ? "5.1" : "stereo");
195
196
197
198
199
200
201
202
            for (int i = 0; i < m_channels; ++i) {
                aformat.append(QStringLiteral("[0:%1]").arg(i));
            }
            args << aformat;
            args << QStringLiteral("-frames:v") << QStringLiteral("1");
            for (int i = 0; i < m_channels; i++) {
                // Channel 1
                args << QStringLiteral("-map") << QStringLiteral("[0:%1]").arg(i) << QStringLiteral("-c:a") << QStringLiteral("pcm_s16le") << QStringLiteral("-y")
203
                 << QStringLiteral("-f") << QStringLiteral("data") << channelFiles[size_t(i)]->fileName();
204
            }
205
        }
206
        m_ffmpegProcess = std::make_unique<QProcess>();
207
        connect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress, Qt::UniqueConnection);
208
209
        connect(this, &AudioThumbJob::jobCanceled, [&]() {
            if (m_ffmpegProcess) {
210
                disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
211
212
                m_ffmpegProcess->kill();
            }
213
214
215
            m_audioLevels.clear();
            m_done = true;
            m_successful = false;
216
        });
217
218
        m_ffmpegProcess->start(KdenliveSettings::ffmpegpath(), args);
        m_ffmpegProcess->waitForFinished(-1);
219
        disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
220
        if (m_successful && m_ffmpegProcess->exitStatus() != QProcess::CrashExit) {
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
            int dataSize = 0;
            std::vector<const qint16 *> rawChannels;
            std::vector<QByteArray> sourceChannels;
            for (auto &channelFile : channelFiles) {
                channelFile->open();
                sourceChannels.emplace_back(channelFile->readAll());
                QByteArray &res = sourceChannels.back();
                channelFile->close();
                if (dataSize == 0) {
                    dataSize = res.size();
                }
                if (res.isEmpty() || res.size() != dataSize) {
                    // Something went wrong, abort
                    m_errorMessage.append(i18n("Audio thumbs: error reading audio thumbnail created with FFmpeg\n"));
                    return false;
                }
237
                rawChannels.emplace_back(reinterpret_cast<const qint16 *>(res.constData()));
238
            }
239
240
            int progress = 0;
            std::vector<long> channelsData;
241
            double offset = double(dataSize) / (2.0 * m_lengthInFrames);
242
243
            int intraOffset = 1;
            if (offset > 1000) {
244
                intraOffset = int(offset / 60);
245
            } else if (offset > 250) {
246
                intraOffset = int(offset / 10);
247
            }
Jean-Baptiste Mardelle's avatar
Jean-Baptiste Mardelle committed
248

249
            long maxAudioLevel = 1;
250
251
252
253
254
            if (!m_successful) {
                m_done = true;
                return true;
            }
            std::vector <long> ffmpegLevels;
255
            for (int i = 0; i < m_lengthInFrames; i++) {
256
                channelsData.resize(size_t(rawChannels.size()));
257
                std::fill(channelsData.begin(), channelsData.end(), 0);
258
                int pos = int(i * offset);
259
                int steps = 0;
260
                for (int j = 0; j < int(offset) && (pos + j < dataSize); j += intraOffset) {
261
262
                    steps++;
                    for (size_t k = 0; k < rawChannels.size(); k++) {
263
                        channelsData[k] += abs(rawChannels[k][pos + j]);
264
                    }
265
                }
266
                steps = qMax(steps, 1);
267
                for (long &k : channelsData) {
268
269
                    if (!m_successful) {
                        break;
270
                    }
271
                    k /= steps;
272
                    maxAudioLevel = qMax(k, maxAudioLevel);
273
                }
274
                
275
276
277
278
                int p = 80 + (i * 20 / m_lengthInFrames);
                if (p != progress) {
                    emit jobProgress(p);
                    progress = p;
279
                }
280
281
282
283
284
                ffmpegLevels.insert(ffmpegLevels.end(), channelsData.begin(), channelsData.end());
            }
            if (!m_successful) {
                m_done = true;
                return true;
285
            }
286
            for (long &v : ffmpegLevels) {
287
                m_audioLevels << uint8_t(255 * v / maxAudioLevel);
288
            }
289
290
            m_done = true;
            return true;
291
        } else if (m_ffmpegProcess) {
292
293
294
295
            QString err = m_ffmpegProcess->readAllStandardError();
            // m_errorMessage += err;
            // m_errorMessage.append(i18n("Failed to create FFmpeg audio thumbnails, we now try to use MLT"));
            qWarning() << "Failed to create FFmpeg audio thumbs:\n" << err << "\n---------------------";
296
        }
297
298
    } else {
        m_done = true;
299
    }
300
    return m_done;
301
302
303
304
}

void AudioThumbJob::updateFfmpegProgress()
{
305
306
307
    if (m_ffmpegProcess == nullptr) {
        return;
    }
308
    const QString result = m_ffmpegProcess->readAllStandardOutput();
309
310
311
    const QStringList lines = result.split(QLatin1Char('\n'));
    for (const QString &data : lines) {
        if (data.startsWith(QStringLiteral("out_time_ms"))) {
Vincent Pinon's avatar
Vincent Pinon committed
312
            double ms = data.section(QLatin1Char('='), 1).toDouble();
313
            emit jobProgress(int(ms / m_binClip->duration().ms() / 10));
314
315
316
317
318
319
320
321
322
323
324
        } else {
            m_logDetails += data + QStringLiteral("\n");
        }
    }
}

bool AudioThumbJob::startJob()
{
    if (m_done) {
        return true;
    }
325
    m_dataInCache = false;
326
    m_binClip = pCore->projectItemModel()->getClipByBinID(m_clipId);
327
328
329
330
    if (m_binClip == nullptr) {
        // Clip was deleted
        return false;
    }
331
    m_successful = true;
332
    if (m_binClip->audioChannels() == 0 || m_binClip->audioThumbCreated()) {
333
334
335
336
337
        // nothing to do
        m_done = true;
        return true;
    }
    m_prod = m_binClip->originalProducer();
338
339
340
341
342
343
344
    if ((m_prod == nullptr) || !m_prod->is_valid()) {
        m_errorMessage.append(i18n("Audio thumbs: cannot open project file %1", m_binClip->url()));
        m_done = true;
        m_successful = false;
        return false;
    }
    m_lengthInFrames = m_prod->get_length(); // Multiply this if we want more than 1 sample per frame
345
346
347
348
349
350
    if (m_lengthInFrames == INT_MAX) {
        // This is a broken file or live feed, don't attempt to generate audio thumbnails
        m_done = true;
        m_successful = false;
        return false;
    }
351
352
353
354
355
356
    m_frequency = m_binClip->audioInfo()->samplingRate();
    m_frequency = m_frequency <= 0 ? 48000 : m_frequency;

    m_channels = m_binClip->audioInfo()->channels();
    m_channels = m_channels <= 0 ? 2 : m_channels;

357
    QMap <int, QString> streams = m_binClip->audioInfo()->streams();
358
    QMap <int, int> audioChannels = m_binClip->audioInfo()->streamChannels();
359
    QMapIterator<int, QString> st(streams);
360
    m_done = true;
361
    ClipType::ProducerType type = m_binClip->clipType();
362
363
364
    while (st.hasNext()) {
        st.next();
        int stream = st.key();
365
366
367
        if (audioChannels.contains(stream)) {
            m_channels = audioChannels.value(stream);
        }
368
369
370
        // Generate one thumb per stream
        m_audioStream = stream;
        m_cachePath = m_binClip->getAudioThumbPath(stream);
371
372
373
374
        if (QFile::exists(m_cachePath)) {
            // Audio thumb already exists
            continue;
        }
375
        m_done = false;
376
        bool ok = false;
377
        if (type == ClipType::Playlist) {
378
379
380
381
382
383
384
385
            if (KdenliveSettings::audiothumbnails()) {
                ok = computeWithMlt();
            }
        } else {
            ok = computeWithFFMPEG();
            if (!ok && KdenliveSettings::audiothumbnails()) {
                ok = computeWithMlt();
            }
386
        }
387
        m_ffmpegProcess.reset();
388
        Q_ASSERT(ok == m_done);
389
390
391
        if (!m_successful) {
            // Job was aborted
            m_done = true;
392
            m_audioLevels.clear();
393
394
            return false;
        }
395

396
        if (ok && !QFile::exists(m_cachePath) && m_done && !m_audioLevels.isEmpty()) {
397
398
            // Put into an image for caching.
            int count = m_audioLevels.size();
399
            QImage image(int(lrint((count + 3) / 4.0 / m_channels)), m_channels, QImage::Format_ARGB32);
400
401
402
403
404
            int n = image.width() * image.height();
            for (int i = 0; i < n; i++) {
                QRgb p;
                if ((4 * i + 3) < count) {
                    p = qRgba(m_audioLevels.at(4 * i), m_audioLevels.at(4 * i + 1), m_audioLevels.at(4 * i + 2),
405
                          m_audioLevels.at(4 * i + 3));
406
407
408
409
410
411
412
413
414
                } else {
                    int last = m_audioLevels.last();
                    int r = (4 * i + 0) < count ? m_audioLevels.at(4 * i + 0) : last;
                    int g = (4 * i + 1) < count ? m_audioLevels.at(4 * i + 1) : last;
                    int b = (4 * i + 2) < count ? m_audioLevels.at(4 * i + 2) : last;
                    int a = last;
                    p = qRgba(r, g, b, a);
                }
                image.setPixel(i / m_channels, i % m_channels, p);
415
            }
416
            image.save(m_cachePath);
417
        }
418
419
420
        m_audioLevels.clear();
    }
    if (m_done || !KdenliveSettings::audiothumbnails()) {
421
422
        m_successful = true;
        return true;
423
424
425
426
427
428
    }
    m_done = true;
    m_successful = false;
    return false;
}

429
bool AudioThumbJob::commitResult(Fun &undo, Fun &redo)
430
431
432
433
434
435
436
437
438
439
{
    Q_ASSERT(!m_resultConsumed);
    if (!m_done) {
        qDebug() << "ERROR: Trying to consume invalid results";
        return false;
    }
    m_resultConsumed = true;
    if (!m_successful) {
        return false;
    }
440
    auto operation = [clip = m_binClip]() {
441
        clip->updateAudioThumbnail();
442
443
        return true;
    };
444
    auto reverse = [clip = m_binClip]() {
445
        clip->updateAudioThumbnail();
446
447
448
449
450
451
452
453
        return true;
    };
    bool ok = operation();
    if (ok) {
        UPDATE_UNDO_REDO_NOLOCK(operation, reverse, undo, redo);
    }
    return ok;
}