merge align audio fixes from:

https://phabricator.kde.org/D5205
parent d3e47040
......@@ -17,16 +17,17 @@ the Free Software Foundation, either version 3 of the License, or
#include <cmath>
#include <iostream>
AudioCorrelation::AudioCorrelation(AudioEnvelope *mainTrackEnvelope)
: m_mainTrackEnvelope(mainTrackEnvelope)
AudioCorrelation::AudioCorrelation(std::unique_ptr<AudioEnvelope> mainTrackEnvelope) :
m_mainTrackEnvelope(std::move(mainTrackEnvelope))
{
m_mainTrackEnvelope->normalizeEnvelope();
connect(m_mainTrackEnvelope, &AudioEnvelope::envelopeReady, this, &AudioCorrelation::slotAnnounceEnvelope);
//Q_ASSERT(!mainTrackEnvelope->hasComputationStarted());
connect(m_mainTrackEnvelope.get(), &AudioEnvelope::envelopeReady,
this, &AudioCorrelation::slotAnnounceEnvelope);
m_mainTrackEnvelope->startComputeEnvelope();
}
AudioCorrelation::~AudioCorrelation()
{
delete m_mainTrackEnvelope;
for (AudioEnvelope *envelope : m_children) {
delete envelope;
}
......@@ -44,26 +45,37 @@ void AudioCorrelation::slotAnnounceEnvelope()
void AudioCorrelation::addChild(AudioEnvelope *envelope)
{
envelope->normalizeEnvelope();
connect(envelope, &AudioEnvelope::envelopeReady, this, &AudioCorrelation::slotProcessChild);
// We need to connect before starting the computation, to make sure
// there is no race condition where the signal 'envelopeReady' is
// lost.
Q_ASSERT(!envelope->hasComputationStarted());
connect(envelope, &AudioEnvelope::envelopeReady,
this, &AudioCorrelation::slotProcessChild);
envelope->startComputeEnvelope();
}
void AudioCorrelation::slotProcessChild(AudioEnvelope *envelope)
{
const int sizeMain = m_mainTrackEnvelope->envelopeSize();
const int sizeSub = envelope->envelopeSize();
// Note that at this point the computation of the envelope of the
// main track might not be finished. envelope() will block until
// the computation is done.
const int sizeMain = m_mainTrackEnvelope->envelope().size();
const int sizeSub = envelope->envelope().size();
auto *info = new AudioCorrelationInfo(sizeMain, sizeSub);
AudioCorrelationInfo *info = new AudioCorrelationInfo(sizeMain, sizeSub);
qint64 *correlation = info->correlationVector();
const qint64 *envMain = m_mainTrackEnvelope->envelope();
const qint64 *envSub = envelope->envelope();
const std::vector<qint64>& envMain = m_mainTrackEnvelope->envelope();
const std::vector<qint64>& envSub = envelope->envelope();
qint64 max = 0;
if (sizeSub > 200) {
FFTCorrelation::correlate(envMain, sizeMain, envSub, sizeSub, correlation);
FFTCorrelation::correlate(&envMain[0], sizeMain,
&envSub[0], sizeSub,
correlation);
} else {
correlate(envMain, sizeMain, envSub, sizeSub, correlation, &max);
correlate(&envMain[0], sizeMain,
&envSub[0], sizeSub, correlation, &max);
info->setMax(max);
}
......@@ -82,7 +94,7 @@ int AudioCorrelation::getShift(int childIndex) const
Q_ASSERT(childIndex < m_correlations.size());
int indexOffset = m_correlations.at(childIndex)->maxIndex();
indexOffset -= m_children.at(childIndex)->envelopeSize();
indexOffset -= m_children.at(childIndex)->envelope().size();
return indexOffset;
}
......
......@@ -27,11 +27,28 @@ class AudioCorrelation : public QObject
{
Q_OBJECT
public:
/// AudioCorrelation will take ownership of mainTrackEnvelope
explicit AudioCorrelation(AudioEnvelope *mainTrackEnvelope);
/**
@param mainTrackEnvelope Envelope of the reference track. Its
actual computation will be started in
this contructor
(i.e. mainTrackEnvelope->StartComputeEnvelope()
will be called). The computation of the
envelop must not be started when passed
to this contructor
(i.e. mainTrackEnvelope->HasComputationStarted()
must return false).
*/
explicit AudioCorrelation(std::unique_ptr<AudioEnvelope> mainTrackEnvelope);
~AudioCorrelation();
/**
Adds a child envelope that will be aligned to the reference
envelope. This function returns immediately, the alignment
computation is done asynchronously. When done, the signal
gotAudioAlignData will be emitted. Similarly to the main
envelope, the computation of the envelope must not be started
when it is passed to this object.
This object will take ownership of the passed envelope.
*/
void addChild(AudioEnvelope *envelope);
......@@ -46,12 +63,19 @@ public:
static void correlate(const qint64 *envMain, int sizeMain, const qint64 *envSub, int sizeSub, qint64 *correlation, qint64 *out_max = nullptr);
private:
AudioEnvelope *m_mainTrackEnvelope;
std::unique_ptr<AudioEnvelope> m_mainTrackEnvelope;
QList<AudioEnvelope *> m_children;
QList<AudioCorrelationInfo *> m_correlations;
private slots:
/**
This is invoked when the child envelope is computed. This
triggers the actual computations of the cross-correlation for
aligning the envelope to the reference envelope.
Takes ownership of @p envelope.
*/
void slotProcessChild(AudioEnvelope *envelope);
void slotAnnounceEnvelope();
......
......@@ -18,104 +18,106 @@
#include <QTime>
#include <QtConcurrent>
#include <cmath>
#include <algorithm>
AudioEnvelope::AudioEnvelope(const QString &binId, int clipId, int offset, int length, int startPos) :
m_envelope(nullptr),
m_offset(offset),
m_length(length),
m_clipId(clipId),
m_startpos(startPos),
m_envelopeMax(0),
m_envelopeMean(0),
m_envelopeStdDev(0),
m_envelopeStdDevCalculated(false),
m_envelopeIsNormalized(false)
m_length(length)
{
std::shared_ptr<ProjectClip> clip = pCore->bin()->getBinClip(binId);
m_envelopeSize = clip->frameDuration();
m_producer = clip->cloneProducer();
connect(&m_watcher, &QFutureWatcherBase::finished, this, &AudioEnvelope::slotProcessEnveloppe);
connect(&m_watcher, &QFutureWatcherBase::finished, this, [this]{ envelopeReady(this); } );
if (!m_producer || !m_producer->is_valid()) {
qCDebug(KDENLIVE_LOG) << "// Cannot create envelope for producer: " << binId;
}
m_info = new AudioInfo(m_producer);
m_info.reset(new AudioInfo(m_producer));
Q_ASSERT(m_offset >= 0);
if (m_length > 0) {
Q_ASSERT(m_length + m_offset <= m_envelopeSize);
m_envelopeSize = m_length;
if (length > 0) {
Q_ASSERT(length + m_offset <= m_envelopeSize);
m_envelopeSize = length;
}
}
AudioEnvelope::~AudioEnvelope()
{
if (m_envelope != nullptr) {
delete[] m_envelope;
}
delete m_info;
if (hasComputationStarted()) {
// This is better than nothing, but does not seem enough to
// guarantee safe deletion of the AudioEnvelope while the
// computations are running: if the computations have just
// finished, m_watcher might be finished, but the signal
// 'envelopeReady' might still be pending while AudioEnvelope is
// being deleted, which can cause a crash according to
// http://doc.qt.io/qt-5/qobject.html#dtor.QObject.
m_audioSummary.waitForFinished();
m_watcher.waitForFinished();
}
}
const qint64 *AudioEnvelope::envelope()
{
if (m_envelope == nullptr) {
loadEnvelope();
}
return m_envelope;
void AudioEnvelope::startComputeEnvelope() {
m_audioSummary = QtConcurrent::run(this, &AudioEnvelope::loadAndNormalizeEnvelope);
m_watcher.setFuture(m_audioSummary);
}
int AudioEnvelope::envelopeSize() const
{
return m_envelopeSize;
bool AudioEnvelope::hasComputationStarted() const {
// An empty qFuture is canceled. QtConcurrent::run() returns a
// future that does not support cancelation, so this is a good way
// to check whether the computations have started.
return !m_audioSummary.isCanceled();
}
void AudioEnvelope::loadEnvelope()
const AudioEnvelope::AudioSummary& AudioEnvelope::audioSummary() {
Q_ASSERT(hasComputationStarted());
m_audioSummary.waitForFinished();
Q_ASSERT(m_audioSummary.constBegin() != m_audioSummary.constEnd());
// We use this instead of m_audioSummary.result() in order to return
// a const reference instead of a copy.
return *m_audioSummary.constBegin();
}
const std::vector<qint64>& AudioEnvelope::envelope()
{
Q_ASSERT(m_envelope == nullptr);
// Blocks until the summary is available.
return audioSummary().audioAmplitudes;
}
AudioEnvelope::AudioSummary AudioEnvelope::loadAndNormalizeEnvelope() const {
qCDebug(KDENLIVE_LOG) << "Loading envelope ...";
AudioSummary summary(m_envelopeSize);
int samplingRate = m_info->info(0)->samplingRate();
mlt_audio_format format_s16 = mlt_audio_s16;
int channels = 1;
m_envelope = new qint64[m_envelopeSize];
m_envelopeMax = 0;
m_envelopeMean = 0;
QTime t;
t.start();
int count = 0;
m_producer->seek(m_offset);
m_producer->set_speed(1.0); // This is necessary, otherwise we don't get any new frames in the 2nd run.
for (int i = 0; i < m_envelopeSize; ++i) {
Mlt::Frame *frame = m_producer->get_frame(i);
for (int i = 0; i < summary.audioAmplitudes.size() ; ++i) {
std::unique_ptr<Mlt::Frame> frame(m_producer->get_frame(i));
qint64 position = mlt_frame_get_position(frame->get_frame());
int samples = mlt_sample_calculator(m_producer->get_fps(), samplingRate, position);
auto *data = static_cast<qint16 *>(frame->get_audio(format_s16, samplingRate, channels, samples));
qint64 sum = 0;
summary.audioAmplitudes[i] = 0;
for (int k = 0; k < samples; ++k) {
sum += abs(data[k]);
}
m_envelope[i] = sum;
m_envelopeMean += sum;
if (sum > m_envelopeMax) {
m_envelopeMax = sum;
}
// qCDebug(KDENLIVE_LOG) << position << '|' << m_producer->get_playtime()
// << '-' << m_producer->get_in() << '+' << m_producer->get_out() << ' ';
delete frame;
count++;
if (m_length > 0 && count > m_length) {
break;
summary.audioAmplitudes[i] += abs(data[k]);
}
}
m_envelopeMean /= m_envelopeSize;
qCDebug(KDENLIVE_LOG) << "Calculating the envelope (" << m_envelopeSize << " frames) took " << t.elapsed() << " ms.";
qCDebug(KDENLIVE_LOG) << "Normalizing envelope ...";
const qint64 meanBeforeNormalization = std::accumulate(summary.audioAmplitudes.begin(),
summary.audioAmplitudes.end(), 0LL) /
summary.audioAmplitudes.size();
// Normalize the envelope.
summary.amplitudeMax = 0;
for (int i = 0; i < summary.audioAmplitudes.size(); ++i) {
summary.audioAmplitudes[i] -= meanBeforeNormalization;
summary.amplitudeMax = std::max(summary.amplitudeMax, abs(summary.audioAmplitudes[i]));
}
return summary;
}
int AudioEnvelope::clipId() const
......@@ -128,56 +130,19 @@ int AudioEnvelope::startPos() const
return m_startpos;
}
void AudioEnvelope::normalizeEnvelope(bool /*clampTo0*/)
{
if (m_envelope == nullptr && !m_future.isRunning()) {
m_future = QtConcurrent::run(this, &AudioEnvelope::loadEnvelope);
m_watcher.setFuture(m_future);
}
}
void AudioEnvelope::slotProcessEnveloppe()
{
if (!m_envelopeIsNormalized) {
m_envelopeMax = 0;
qint64 newMean = 0;
for (int i = 0; i < m_envelopeSize; ++i) {
m_envelope[i] -= m_envelopeMean;
/*if (clampTo0) {
if (m_envelope[i] < 0) { m_envelope[i] = 0; }
}*/
if (m_envelope[i] > m_envelopeMax) {
m_envelopeMax = m_envelope[i];
}
newMean += m_envelope[i];
}
m_envelopeMean = newMean / m_envelopeSize;
m_envelopeIsNormalized = true;
}
emit envelopeReady(this);
}
QImage AudioEnvelope::drawEnvelope()
{
if (m_envelope == nullptr) {
loadEnvelope();
}
const AudioSummary& summary = audioSummary();
QImage img(m_envelopeSize, 400, QImage::Format_ARGB32);
img.fill(qRgb(255, 255, 255));
if (m_envelopeMax == 0) {
if (summary.amplitudeMax == 0) {
return img;
}
for (int x = 0; x < img.width(); ++x) {
double fy = m_envelope[x] / double(m_envelopeMax) * img.height();
double fy = summary.audioAmplitudes[x] / double(summary.amplitudeMax) * img.height();
for (int y = img.height() - 1; y > img.height() - 1 - fy; --y) {
img.setPixel(x, y, qRgb(50, 50, 50));
}
......@@ -185,15 +150,14 @@ QImage AudioEnvelope::drawEnvelope()
return img;
}
void AudioEnvelope::dumpInfo() const
void AudioEnvelope::dumpInfo()
{
if (m_envelope == nullptr) {
qCDebug(KDENLIVE_LOG) << "Envelope not generated, no information available.";
} else {
qCDebug(KDENLIVE_LOG) << "Envelope info"
<< "\n* size = " << m_envelopeSize << "\n* max = " << m_envelopeMax << "\n* µ = " << m_envelopeMean;
if (m_envelopeStdDevCalculated) {
qCDebug(KDENLIVE_LOG) << "* s = " << m_envelopeStdDev;
}
}
if (!m_audioSummary.isFinished()) {
qCDebug(KDENLIVE_LOG) << "Envelope not yet generated, no information available.";
} else {
const AudioSummary& summary = audioSummary();
qCDebug(KDENLIVE_LOG) << "Envelope info"
<< "\n* size = " << summary.audioAmplitudes.size()
<< "\n* max = " << summary.amplitudeMax;
}
}
......@@ -13,7 +13,8 @@
#include "audioInfo.h"
#include <mlt++/Mlt.h>
#include <memory>
#include <vector>
#include <QFutureWatcher>
#include <QObject>
......@@ -33,43 +34,65 @@ class AudioEnvelope : public QObject
public:
explicit AudioEnvelope(const QString &binId, int clipId, int offset = 0, int length = 0, int startPos = 0);
virtual ~AudioEnvelope();
/// Returns the envelope, calculates it if necessary.
qint64 const *envelope();
int envelopeSize() const;
void loadEnvelope();
void normalizeEnvelope(bool clampTo0 = false);
/**
Starts the asynchronous computation that computes the
envelope. When the computations are done, the signal
'envelopeReady' will be emitted.
*/
void startComputeEnvelope();
/**
Returns whether startComputeEnvelope() has been called.
*/
bool hasComputationStarted() const;
/**
Returns the envelope data. Blocks until the computation of the
envelope is done.
REQUIRES: startComputeEnvelope() has been called.
*/
const std::vector<qint64>& envelope();
QImage drawEnvelope();
void dumpInfo() const;
void dumpInfo();
int clipId() const;
int startPos() const;
private:
qint64 *m_envelope;
std::shared_ptr<Mlt::Producer>m_producer;
AudioInfo *m_info;
QFutureWatcher<void> m_watcher;
QFuture<void> m_future;
int m_offset;
int m_length;
int m_clipId;
int m_startpos;
struct AudioSummary {
explicit AudioSummary(int size) : audioAmplitudes(size) {}
AudioSummary() {}
// This is the envelope data. There is one element for each
// frame, which contains the sum of the absolute amplitudes of
// the audio signal for that frame.
std::vector<qint64> audioAmplitudes;
// Maximum absolute value of the elements in 'audioAmplitudes'.
qint64 amplitudeMax = 0;
};
/**
Blocks until the AudioSummary has been computed.
REQUIRES: startComputeEnvelope() has been called.
*/
const AudioSummary& audioSummary();
/**
Actually computes the envelope data, synchronously.
*/
AudioSummary loadAndNormalizeEnvelope() const;
std::shared_ptr<Mlt::Producer> m_producer;
std::unique_ptr<AudioInfo> m_info;
QFutureWatcher<AudioSummary> m_watcher;
QFuture<AudioSummary> m_audioSummary;
const int m_offset;
const int m_clipId;
const int m_startpos;
const int m_length;
int m_envelopeSize;
qint64 m_envelopeMax;
qint64 m_envelopeMean;
qint64 m_envelopeStdDev;
bool m_envelopeStdDevCalculated;
bool m_envelopeIsNormalized;
private slots:
void slotProcessEnveloppe();
signals:
void envelopeReady(AudioEnvelope *envelope);
......
......@@ -1539,8 +1539,8 @@ void TimelineController::splitVideo(int clipId)
void TimelineController::setAudioRef(int clipId)
{
m_audioRef = clipId;
AudioEnvelope *envelope = new AudioEnvelope(getClipBinId(clipId), clipId);
m_audioCorrelator.reset(new AudioCorrelation(envelope));
std::unique_ptr<AudioEnvelope> envelope(new AudioEnvelope(getClipBinId(clipId), clipId));
m_audioCorrelator.reset(new AudioCorrelation(std::move(envelope)));
connect(m_audioCorrelator.get(), &AudioCorrelation::gotAudioAlignData, [&] (int cid, int shift) {
int pos = m_model->getClipPosition(m_audioRef) + shift + m_model->getClipIn(m_audioRef);
bool result = m_model->requestClipMove(cid, m_model->getClipTrackId(cid), pos, true, true);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment