Commit d21a500c authored by Jean-Baptiste Mardelle's avatar Jean-Baptiste Mardelle Committed by Vincent Pinon

Make audio align work asynchronously, fix timeline corruption when trying to move clip before 0

Conflicts:
	src/customtrackview.cpp
	src/customtrackview.h
parent 3fcd1802
......@@ -87,7 +87,7 @@ add_subdirectory(renderer)
add_subdirectory(src)
add_subdirectory(thumbnailer)
add_subdirectory(titles)
add_subdirectory(testingArea)
#add_subdirectory(testingArea)
macro_display_feature_log()
......
......@@ -1090,12 +1090,8 @@ void CustomTrackView::mousePressEvent(QMouseEvent * event)
m_selectionGroup->setProperty("locked_tracks", lockedTracks);
}
m_selectionMutex.unlock();
if (m_dragItem) {
ClipItem *clip = qgraphicsitem_cast<ClipItem *>(m_dragItem);
updateClipTypeActions(dragGroup == NULL ? clip : NULL);
if (m_dragItem)
m_pasteEffectsAction->setEnabled(m_copiedItems.count() == 1);
}
else updateClipTypeActions(NULL);
}
else {
QGraphicsView::mousePressEvent(event);
......@@ -6723,6 +6719,8 @@ void CustomTrackView::setAudioAlignReference()
}
if (m_audioCorrelator != NULL) {
delete m_audioCorrelator;
m_audioCorrelator = NULL;
m_audioAlignmentReference = NULL;
}
if (selection.at(0)->type() == AVWidget) {
ClipItem *clip = static_cast<ClipItem*>(selection.at(0));
......@@ -6731,15 +6729,9 @@ void CustomTrackView::setAudioAlignReference()
AudioEnvelope *envelope = new AudioEnvelope(clip->baseClip()->fileURL().path(), clip->getProducer(clip->track()));
m_audioCorrelator = new AudioCorrelation(envelope);
#ifdef DEBUG
envelope->drawEnvelope().save("kdenlive-audio-reference-envelope.png");
envelope->dumpInfo();
#endif
emit displayMessage(i18n("Audio align reference set."), InformationMessage);
connect(m_audioCorrelator, SIGNAL(gotAudioAlignData(int,int,int)), this, SLOT(slotAlignClip(int,int,int)));
connect(m_audioCorrelator, SIGNAL(displayMessage(QString,MessageType)), this, SIGNAL(displayMessage(QString,MessageType)));
emit displayMessage(i18n("Processing audio, please wait."), ProcessingJobMessage);
}
return;
}
......@@ -6763,7 +6755,6 @@ void CustomTrackView::alignAudio()
return;
}
int counter = 0;
QList<QGraphicsItem *> selection = scene()->selectedItems();
foreach (QGraphicsItem *item, selection) {
if (item->type() == AVWidget) {
......@@ -6774,82 +6765,59 @@ void CustomTrackView::alignAudio()
}
if (clip->clipType() == AV || clip->clipType() == Audio) {
AudioEnvelope *envelope = new AudioEnvelope(clip->baseClip()->fileURL().path(), clip->getProducer(clip->track()), clip->info().cropStart.frames(m_document->fps()), clip->info().cropDuration.frames(m_document->fps()));
// FFT only for larger vectors. We could use it all time, but for small vectors
// the (anyway not noticeable) overhead is smaller with a nested for loop correlation.
int index = m_audioCorrelator->addChild(envelope, envelope->envelopeSize() > 200);
int shift = m_audioCorrelator->getShift(index);
counter++;
#ifdef DEBUG
m_audioCorrelator->info(index)->toImage().save("kdenlive-audio-align-cross-correlation.png");
envelope->drawEnvelope().save("kdenlive-audio-align-envelope.png");
envelope->dumpInfo();
int targetPos = m_audioAlignmentReference->startPos().frames(m_document->fps()) + shift;
qDebug() << "Reference starts at " << m_audioAlignmentReference->startPos().frames(m_document->fps());
qDebug() << "We will start at " << targetPos;
qDebug() << "to shift by " << shift;
qDebug() << "(eventually)";
qDebug() << "(maybe)";
#endif
QUndoCommand *moveCommand = new QUndoCommand();
GenTime add(shift, m_document->fps());
ItemInfo start = clip->info();
ItemInfo end = start;
end.startPos = m_audioAlignmentReference->startPos() + add - m_audioAlignmentReference->cropStart();
end.endPos = end.startPos + start.cropDuration;
if ( end.startPos.seconds() < 0 ) {
// Clip would start before 0, so crop it first
GenTime cropBy = -end.startPos;
#ifdef DEBUG
qDebug() << "Need to crop clip. " << start;
qDebug() << "end.startPos: " << end.startPos.toString() << ", cropBy: " << cropBy.toString();
#endif
ItemInfo resized = start;
resized.startPos += cropBy;
resizeClip(start, resized);
new ResizeClipCommand(this, start, resized, false, false, moveCommand);
start = clip->info();
end.startPos += cropBy;
#ifdef DEBUG
qDebug() << "Clip cropped. " << start;
qDebug() << "Moving to: " << end;
#endif
}
ItemInfo info = clip->info();
AudioEnvelope *envelope = new AudioEnvelope(clip->baseClip()->fileURL().path(), clip->getProducer(clip->track()), info.cropStart.frames(m_document->fps()), info.cropDuration.frames(m_document->fps()), clip->track(), info.startPos.frames(m_document->fps()));
m_audioCorrelator->addChild(envelope);
}
}
}
emit displayMessage(i18n("Processing audio, please wait."), ProcessingJobMessage);
}
if (itemCollision(clip, end)) {
delete moveCommand;
emit displayMessage(i18n("Unable to move clip due to collision."), ErrorMessage);
return;
}
void CustomTrackView::slotAlignClip(int track, int pos, int shift)
{
QUndoCommand *moveCommand = new QUndoCommand();
ClipItem *clip = getClipItemAt(pos, track);
if (!clip) {
emit displayMessage(i18n("Cannot find clip to align."), ErrorMessage);
//emit displayMessage(i18n("Auto-aligned %1 clips.", counter), InformationMessage);
return;
}
GenTime add(shift, m_document->fps());
ItemInfo start = clip->info();
ItemInfo end = start;
end.startPos = m_audioAlignmentReference->startPos() + add - m_audioAlignmentReference->cropStart();
end.endPos = end.startPos + start.cropDuration;
if ( end.startPos < GenTime() ) {
// Clip would start before 0, so crop it first
GenTime cropBy = -end.startPos;
if (cropBy > start.cropDuration) {
delete moveCommand;
emit displayMessage(i18n("Unable to move clip out of timeline."), ErrorMessage);
return;
}
ItemInfo resized = start;
resized.startPos += cropBy;
moveCommand->setText(i18n("Auto-align clip"));
new MoveClipCommand(this, start, end, true, moveCommand);
updateTrackDuration(clip->track(), moveCommand);
m_commandStack->push(moveCommand);
resizeClip(start, resized);
new ResizeClipCommand(this, start, resized, false, false, moveCommand);
}
}
start = clip->info();
end.startPos += cropBy;
}
if (counter == 0) {
emit displayMessage(i18n("No audio clips selected."), ErrorMessage);
} else {
emit displayMessage(i18n("Auto-aligned %1 clips.", counter), InformationMessage);
if (itemCollision(clip, end)) {
delete moveCommand;
emit displayMessage(i18n("Unable to move clip due to collision."), ErrorMessage);
return;
}
emit displayMessage(i18n("Clip aligned."), OperationCompletedMessage);
moveCommand->setText(i18n("Auto-align clip"));
new MoveClipCommand(this, start, end, true, moveCommand);
updateTrackDuration(clip->track(), moveCommand);
m_commandStack->push(moveCommand);
}
void CustomTrackView::doSplitAudio(const GenTime &pos, int track, EffectsList effects, bool split)
......
......@@ -213,6 +213,9 @@ public:
/** @brief Returns frame number of current mouse position. */
int getMousePos() const;
/** @brief Get effect parameters ready for MLT*/
static void adjustEffectParameters(EffectsParameterList &parameters, QDomNodeList params, MltVideoProfile profile, const QString &prefix = QString());
public slots:
/** @brief Send seek request to MLT. */
void seekCursorPos(int pos);
......@@ -298,12 +301,11 @@ public slots:
void slotAddEffect(ClipItem *clip, const QDomElement &effect);
void slotImportClipKeyframes(GraphicsRectItem type);
/** @brief Get effect parameters ready for MLT*/
static void adjustEffectParameters(EffectsParameterList &parameters, QDomNodeList params, MltVideoProfile profile, const QString &prefix = QString());
/** @brief Move playhead to mouse curser position if defined key is pressed */
/** @brief Move playhead to mouse curser position if defined key is pressed */
void slotAlignPlayheadToMousePos();
void slotInfoProcessingFinished();
void slotAlignClip(int, int, int);
protected:
virtual void drawBackground(QPainter * painter, const QRectF & rect);
......
......@@ -113,6 +113,7 @@ enum TransitionType {
enum MessageType {
DefaultMessage,
ProcessingJobMessage,
OperationCompletedMessage,
InformationMessage,
ErrorMessage,
......
......@@ -11,6 +11,7 @@ the Free Software Foundation, either version 3 of the License, or
#include "audioCorrelation.h"
#include "fftCorrelation.h"
#include <KLocale>
#include <QDebug>
#include <QTime>
#include <cmath>
......@@ -21,6 +22,7 @@ AudioCorrelation::AudioCorrelation(AudioEnvelope *mainTrackEnvelope) :
m_mainTrackEnvelope(mainTrackEnvelope)
{
m_mainTrackEnvelope->normalizeEnvelope();
connect(m_mainTrackEnvelope, SIGNAL(envelopeReady(AudioEnvelope *)), this, SLOT(slotAnnounceEnvelope()));
}
AudioCorrelation::~AudioCorrelation()
......@@ -36,10 +38,19 @@ AudioCorrelation::~AudioCorrelation()
qDebug() << "Envelope deleted.";
}
int AudioCorrelation::addChild(AudioEnvelope *envelope, bool useFFT)
void AudioCorrelation::slotAnnounceEnvelope()
{
emit displayMessage(i18n("Audio analysis finished"), OperationCompletedMessage);
}
void AudioCorrelation::addChild(AudioEnvelope *envelope)
{
envelope->normalizeEnvelope();
connect(envelope, SIGNAL(envelopeReady(AudioEnvelope *)), this, SLOT(slotProcessChild(AudioEnvelope *)));
}
void AudioCorrelation::slotProcessChild(AudioEnvelope *envelope)
{
const int sizeMain = m_mainTrackEnvelope->envelopeSize();
const int sizeSub = envelope->envelopeSize();
......@@ -50,7 +61,7 @@ int AudioCorrelation::addChild(AudioEnvelope *envelope, bool useFFT)
const int64_t *envSub = envelope->envelope();
int64_t max = 0;
if (useFFT) {
if (sizeSub > 200) {
FFTCorrelation::correlate(envMain, sizeMain,
envSub, sizeSub,
correlation);
......@@ -67,8 +78,9 @@ int AudioCorrelation::addChild(AudioEnvelope *envelope, bool useFFT)
m_correlations.append(info);
Q_ASSERT(m_correlations.size() == m_children.size());
return m_children.indexOf(envelope);
int index = m_children.indexOf(envelope);
int shift = getShift(index);
emit gotAudioAlignData(envelope->track(), envelope->startPos(), shift);
}
int AudioCorrelation::getShift(int childIndex) const
......@@ -154,3 +166,5 @@ void AudioCorrelation::correlate(const int64_t *envMain, int sizeMain,
*out_max = max;
}
}
#include "audioCorrelation.moc"
......@@ -13,6 +13,7 @@
#include "audioCorrelationInfo.h"
#include "audioEnvelope.h"
#include "definitions.h"
#include <QList>
......@@ -23,8 +24,9 @@
It uses one main track (used in the initializer); further tracks will be
aligned relative to this main track.
*/
class AudioCorrelation
class AudioCorrelation : public QObject
{
Q_OBJECT
public:
/// AudioCorrelation will take ownership of mainTrackEnvelope
AudioCorrelation(AudioEnvelope *mainTrackEnvelope);
......@@ -32,9 +34,8 @@ public:
/**
This object will take ownership of the passed envelope.
\return The child's index
*/
int addChild(AudioEnvelope *envelope, bool useFFT = false);
void addChild(AudioEnvelope *envelope);
const AudioCorrelationInfo *info(int childIndex) const;
int getShift(int childIndex) const;
......@@ -52,6 +53,14 @@ private:
QList<AudioEnvelope*> m_children;
QList<AudioCorrelationInfo*> m_correlations;
private slots:
void slotProcessChild(AudioEnvelope *envelope);
void slotAnnounceEnvelope();
signals:
void gotAudioAlignData(int, int, int);
void displayMessage(const QString &, MessageType);
};
#endif // AUDIOCORRELATION_H
......@@ -13,13 +13,16 @@
#include "audioStreamInfo.h"
#include <QDebug>
#include <QImage>
#include <QtConcurrentRun>
#include <QTime>
#include <cmath>
AudioEnvelope::AudioEnvelope(const QString &url, Mlt::Producer *producer, int offset, int length) :
AudioEnvelope::AudioEnvelope(const QString &url, Mlt::Producer *producer, int offset, int length, int track, int startPos) :
m_envelope(NULL),
m_offset(offset),
m_length(length),
m_track(track),
m_startpos(startPos),
m_envelopeSize(producer->get_length()),
m_envelopeMax(0),
m_envelopeMean(0),
......@@ -32,6 +35,7 @@ AudioEnvelope::AudioEnvelope(const QString &url, Mlt::Producer *producer, int of
if (path == QLatin1String("<playlist>") || path == QLatin1String("<tractor>") || path ==QLatin1String( "<producer>"))
path = url;
m_producer = new Mlt::Producer(*(producer->profile()), path.toUtf8().constData());
connect(&m_watcher, SIGNAL(finished()), this, SLOT(slotProcessEnveloppe()));
if (!m_producer || !m_producer->is_valid()) {
qDebug()<<"// Cannot create envelope for producer: "<<path;
}
......@@ -151,12 +155,26 @@ int64_t AudioEnvelope::loadStdDev()
return m_envelopeStdDev;
}
int AudioEnvelope::track() const
{
return m_track;
}
int AudioEnvelope::startPos() const
{
return m_startpos;
}
void AudioEnvelope::normalizeEnvelope(bool clampTo0)
{
if (m_envelope == NULL) {
loadEnvelope();
if (m_envelope == NULL && !m_future.isRunning()) {
m_future = QtConcurrent::run(this, &AudioEnvelope::loadEnvelope);
m_watcher.setFuture(m_future);
}
}
void AudioEnvelope::slotProcessEnveloppe()
{
if (!m_envelopeIsNormalized) {
m_envelopeMax = 0;
......@@ -165,9 +183,9 @@ void AudioEnvelope::normalizeEnvelope(bool clampTo0)
m_envelope[i] -= m_envelopeMean;
if (clampTo0) {
/*if (clampTo0) {
if (m_envelope[i] < 0) { m_envelope[i] = 0; }
}
}*/
if (m_envelope[i] > m_envelopeMax) {
m_envelopeMax = m_envelope[i];
......@@ -179,6 +197,7 @@ void AudioEnvelope::normalizeEnvelope(bool clampTo0)
m_envelopeIsNormalized = true;
}
emit envelopeReady(this);
}
......@@ -218,3 +237,6 @@ void AudioEnvelope::dumpInfo() const
}
}
}
#include "audioEnvelope.moc"
......@@ -14,6 +14,9 @@
#include "audioInfo.h"
#include <mlt++/Mlt.h>
#include <QFutureWatcher>
#include <QObject>
class QImage;
/**
......@@ -23,11 +26,13 @@ class QImage;
See also: http://bemasc.net/wordpress/2011/07/26/an-auto-aligner-for-pitivi/
*/
class AudioEnvelope
class AudioEnvelope : public QObject
{
Q_OBJECT
public:
explicit AudioEnvelope(const QString &url, Mlt::Producer *producer, int offset = 0, int length = 0);
~AudioEnvelope();
explicit AudioEnvelope(const QString &url, Mlt::Producer *producer, int offset = 0, int length = 0, int track = 0, int startPos = 0);
virtual ~AudioEnvelope();
/// Returns the envelope, calculates it if necessary.
int64_t const* envelope();
......@@ -41,14 +46,21 @@ public:
QImage drawEnvelope();
void dumpInfo() const;
int track() const;
int startPos() const;
private:
int64_t *m_envelope;
Mlt::Producer *m_producer;
AudioInfo *m_info;
QFutureWatcher<void> m_watcher;
QFuture<void> m_future;
int m_offset;
int m_length;
int m_track;
int m_startpos;
int m_envelopeSize;
int64_t m_envelopeMax;
......@@ -57,6 +69,12 @@ private:
bool m_envelopeStdDevCalculated;
bool m_envelopeIsNormalized;
private slots:
void slotProcessEnveloppe();
signals:
void envelopeReady(AudioEnvelope *envelope);
};
#endif // AUDIOENVELOPE_H
......@@ -78,7 +78,7 @@ void StatusBarMessageLabel::setMessage(const QString& text,
m_queueSemaphore.acquire();
if (!m_messageQueue.contains(item)) {
if (item.type == ErrorMessage || item.type == MltError) {
if (item.type == ErrorMessage || item.type == MltError || item.type == ProcessingJobMessage) {
qDebug() << item.text;
// Put the new errror message at first place and immediately show it
......@@ -112,10 +112,21 @@ bool StatusBarMessageLabel::slotMessageTimeout()
bool newMessage = false;
// Get the next message from the queue, unless the current one needs to be confirmed
if (!m_messageQueue.isEmpty()) {
if (m_currentMessage.type == ProcessingJobMessage) {
// Check if we have a job completed message to cancel this one
StatusBarMessageItem item;
while (!m_messageQueue.isEmpty()) {
item = m_messageQueue.at(0);
m_messageQueue.removeFirst();
if (item.type == OperationCompletedMessage || item.type == ErrorMessage || item.type == MltError) {
m_currentMessage = item;
newMessage = true;
break;
}
}
}
else if (!m_messageQueue.isEmpty()) {
if (!m_currentMessage.needsConfirmation()) {
m_currentMessage = m_messageQueue.at(0);
m_messageQueue.removeFirst();
newMessage = true;
......@@ -147,6 +158,10 @@ bool StatusBarMessageLabel::slotMessageTimeout()
const char* iconName = 0;
switch (m_currentMessage.type) {
case ProcessingJobMessage:
iconName = "chronometer";
m_closeButton->hide();
break;
case OperationCompletedMessage:
iconName = "dialog-ok";
m_closeButton->hide();
......
......@@ -53,7 +53,7 @@ struct StatusBarMessageItem {
/// \return true if the error still needs to be confirmed
bool needsConfirmation() const
{
return type == MltError && !confirmed;
return (type == MltError && !confirmed);
}
StatusBarMessageItem(const QString& text = QString(), MessageType type = DefaultMessage, int timeoutMS = 0) :
......
......@@ -139,7 +139,8 @@ int main(int argc, char *argv[])
// Calculate the correlation and hereby the audio shift
AudioCorrelation corr(envelopeMain);
int index = corr.addChild(envelopeSub, useFFT);
int index = 0;
corr.addChild(envelopeSub/*, useFFT*/);
int shift = corr.getShift(index);
std::cout << " Should be shifted by " << shift << " frames: " << fileSub << std::endl
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment