audioEnvelope.cpp 6.38 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/***************************************************************************
 *   Copyright (C) 2012 by Simon Andreas Eugster (simon.eu@gmail.com)      *
 *   This file is part of kdenlive. See www.kdenlive.org.                  *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 ***************************************************************************/

#include "audioEnvelope.h"
12
#include "audioStreamInfo.h"
13
14
#include "bin/bin.h"
#include "bin/projectclip.h"
15
#include "core.h"
Laurent Montel's avatar
Laurent Montel committed
16
#include "kdenlive_debug.h"
17
#include <QImage>
18
#include <QElapsedTimer>
19
#include <QtConcurrent>
20
#include <KLocalizedString>
21
#include <algorithm>
22
#include <cmath>
Nicolas Carion's avatar
Nicolas Carion committed
23
#include <memory>
24

25
26
27
28
AudioEnvelope::AudioEnvelope(const QString &binId, int clipId, size_t offset, size_t length, size_t startPos)
    : m_offset(offset)
    , m_clipId(clipId)
    , m_startpos(startPos)
29
{
30
31
    std::shared_ptr<ProjectClip> clip = pCore->bin()->getBinClip(binId);
    m_producer = clip->cloneProducer();
32
33
34
    if (length > 2000) {
        // Analyse on timeline clip zone only
        m_offset = 0;
35
        m_producer->set_in_and_out(int(offset), int(offset + length));
36
    }
37
    m_envelopeSize = size_t(m_producer->get_playtime());
38
39

    m_producer->set("set.test_image", 1);
Vincent Pinon's avatar
Vincent Pinon committed
40
    connect(&m_watcher, &QFutureWatcherBase::finished, this, [this] { emit envelopeReady(this); });
41
42
    if (!m_producer || !m_producer->is_valid()) {
        qCDebug(KDENLIVE_LOG) << "// Cannot create envelope for producer: " << binId;
43
44
    } else {
        m_info = std::make_unique<AudioInfo>(m_producer);
45
    }
46
47
48
49
}

AudioEnvelope::~AudioEnvelope()
{
50
51
52
53
54
55
56
    if (hasComputationStarted()) {
        // This is better than nothing, but does not seem enough to
        // guarantee safe deletion of the AudioEnvelope while the
        // computations are running: if the computations have just
        // finished, m_watcher might be finished, but the signal
        // 'envelopeReady' might still be pending while AudioEnvelope is
        // being deleted, which can cause a crash according to
Yuri Chornoivan's avatar
Yuri Chornoivan committed
57
        // https://doc.qt.io/qt-5/qobject.html#dtor.QObject.
58
59
60
        m_audioSummary.waitForFinished();
        m_watcher.waitForFinished();
    }
61
62
}

63
64
void AudioEnvelope::startComputeEnvelope()
{
65
66
    m_audioSummary = QtConcurrent::run(this, &AudioEnvelope::loadAndNormalizeEnvelope);
    m_watcher.setFuture(m_audioSummary);
67
}
68

69
70
bool AudioEnvelope::hasComputationStarted() const
{
71
    // An empty qFuture is canceled. QtConcurrent::run() returns a
72
    // future that does not support cancellation, so this is a good way
73
74
    // to check whether the computations have started.
    return !m_audioSummary.isCanceled();
75
76
}

77
78
const AudioEnvelope::AudioSummary &AudioEnvelope::audioSummary()
{
79
80
81
82
83
84
85
86
    Q_ASSERT(hasComputationStarted());
    m_audioSummary.waitForFinished();
    Q_ASSERT(m_audioSummary.constBegin() != m_audioSummary.constEnd());
    // We use this instead of m_audioSummary.result() in order to return
    // a const reference instead of a copy.
    return *m_audioSummary.constBegin();
}

87
size_t AudioEnvelope::offset()
88
89
90
91
{
    return m_offset;
}

92
const std::vector<qint64> &AudioEnvelope::envelope()
93
{
94
95
96
    // Blocks until the summary is available.
    return audioSummary().audioAmplitudes;
}
97

98
99
AudioEnvelope::AudioSummary AudioEnvelope::loadAndNormalizeEnvelope() const
{
Laurent Montel's avatar
Laurent Montel committed
100
    qCDebug(KDENLIVE_LOG) << "Loading envelope ...";
101
    AudioSummary summary(m_envelopeSize);
102
    if (!m_info || m_info->size() < 1) {
103
104
        return summary;
    }
105
106
107
108
    int samplingRate = m_info->info(0)->samplingRate();
    mlt_audio_format format_s16 = mlt_audio_s16;
    int channels = 1;

109
    QElapsedTimer t;
110
    t.start();
111
    m_producer->seek(0);
112
113
    size_t max = summary.audioAmplitudes.size();
    for (size_t i = 0; i < max; ++i) {
114
        std::unique_ptr<Mlt::Frame> frame(m_producer->get_frame(int(i)));
Vincent Pinon's avatar
Vincent Pinon committed
115
        qint64 position = mlt_frame_get_position(frame->get_frame());
116
        int samples = mlt_audio_calculate_frame_samples(float(m_producer->get_fps()), samplingRate, position);
Nicolas Carion's avatar
linting    
Nicolas Carion committed
117
        auto *data = static_cast<qint16 *>(frame->get_audio(format_s16, samplingRate, channels, samples));
118

119
        summary.audioAmplitudes[i] = 0;
120
        for (int k = 0; k < samples; ++k) {
121
            summary.audioAmplitudes[i] += abs(data[k]);
122
        }
123
        pCore->displayMessage(i18n("Processing data analysis"), ProcessingJobMessage, int(100 * i / max));
124
    }
Nicolas Carion's avatar
Nicolas Carion committed
125
    qCDebug(KDENLIVE_LOG) << "Calculating the envelope (" << m_envelopeSize << " frames) took " << t.elapsed() << " ms.";
126
    qCDebug(KDENLIVE_LOG) << "Normalizing envelope ...";
127
    const qint64 meanBeforeNormalization =
128
        std::accumulate(summary.audioAmplitudes.begin(), summary.audioAmplitudes.end(), 0LL) / qint64(summary.audioAmplitudes.size());
129
130
131

    // Normalize the envelope.
    summary.amplitudeMax = 0;
132
    for (size_t i = 0; i < max; ++i) {
133
134
        summary.audioAmplitudes[i] -= meanBeforeNormalization;
        summary.amplitudeMax = std::max(summary.amplitudeMax, qAbs(summary.audioAmplitudes[i]));
135
    }
136
    pCore->displayMessage(i18n("Audio analysis finished"), OperationCompletedMessage, 300);
137
    return summary;
138
139
}

140
int AudioEnvelope::clipId() const
141
{
142
    return m_clipId;
143
144
}

145
size_t AudioEnvelope::startPos() const
146
147
148
149
{
    return m_startpos;
}

150
151
QImage AudioEnvelope::drawEnvelope()
{
152
    const AudioSummary &summary = audioSummary();
153

154
    QImage img(int(m_envelopeSize), 400, QImage::Format_ARGB32);
Laurent Montel's avatar
Laurent Montel committed
155
    img.fill(qRgb(255, 255, 255));
156

157
    if (summary.amplitudeMax == 0) {
158
        return img;
Laurent Montel's avatar
Laurent Montel committed
159
    }
160
161

    for (int x = 0; x < img.width(); ++x) {
162
        double fy = double(summary.audioAmplitudes[size_t(x)]) / summary.amplitudeMax * img.height();
Laurent Montel's avatar
Laurent Montel committed
163
164
        for (int y = img.height() - 1; y > img.height() - 1 - fy; --y) {
            img.setPixel(x, y, qRgb(50, 50, 50));
165
166
167
168
169
        }
    }
    return img;
}

170
void AudioEnvelope::dumpInfo()
171
{
172
173
174
175
176
177
178
    if (!m_audioSummary.isFinished()) {
        qCDebug(KDENLIVE_LOG) << "Envelope not yet generated, no information available.";
    } else {
        const AudioSummary &summary = audioSummary();
        qCDebug(KDENLIVE_LOG) << "Envelope info"
                              << "\n* size = " << summary.audioAmplitudes.size() << "\n* max = " << summary.amplitudeMax;
    }
179
}