Skip to content

GitLab

Commit a0db1142 authored by Jean-Baptiste Mardelle's avatar Jean-Baptiste Mardelle
Browse files

Rewrite audio thumbnails to only use FFmpeg's data and optimize memory usage on creation. 

Related to #102
parent 3d5c3ceb
Pipeline #28962 passed with stage
in 30 minutes and 40 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 100 additions and 125 deletions
src/bin/bin.cpp
View file @ a0db1142
......@@ -2517,13 +2517,13 @@ void Bin::slotEditClipCommand(const QString &id, const QMap<QString, QString> &o
m_doc->commandStack()->push(command);
}
void Bin::reloadClip(const QString &id, bool reloadAudio)
void Bin::reloadClip(const QString &id)
{
std::shared_ptr<ProjectClip> clip = m_itemModel->getClipByBinID(id);
if (!clip) {
return;
}
clip->reloadProducer(false, false, reloadAudio);
clip->reloadProducer(false, false);
}
void Bin::reloadMonitorIfActive(const QString &id)
......@@ -3947,11 +3947,11 @@ void Bin::reloadAllProducers(bool reloadThumbs)
if (!xml.isNull()) {
clip->setClipStatus(AbstractProjectItem::StatusWaiting);
pCore->jobManager()->slotDiscardClipJobs(clip->clipId());
clip->discardAudioThumb(false);
clip->discardAudioThumb();
// We need to set a temporary id before all outdated producers are replaced;
int jobId = pCore->jobManager()->startJob<LoadJob>({clip->clipId()}, -1, QString(), xml);
if (reloadThumbs) {
ThumbnailCache::get()->invalidateThumbsForClip(clip->clipId(), true);
ThumbnailCache::get()->invalidateThumbsForClip(clip->clipId());
}
pCore->jobManager()->startJob<ThumbJob>({clip->clipId()}, jobId, QString(), -1, true, true);
if (KdenliveSettings::audiothumbnails()) {
......
src/bin/bin.h
View file @ a0db1142
......@@ -226,7 +226,7 @@ public:
const QString getDocumentProperty(const QString &key);
/** @brief Ask MLT to reload this clip's producer */
void reloadClip(const QString &id, bool reloadAudio = true);
void reloadClip(const QString &id);
/** @brief refresh monitor (if clip changed) */
void reloadMonitorIfActive(const QString &id);
......
src/bin/projectclip.cpp
View file @ a0db1142
......@@ -207,6 +207,50 @@ QString ProjectClip::getXmlProperty(const QDomElement &producer, const QString &
void ProjectClip::updateAudioThumbnail()
{
emit audioThumbReady();
if (m_clipType == ClipType::Audio) {
QImage thumb = ThumbnailCache::get()->getThumbnail(m_binId, 0);
if (thumb.isNull()) {
int iconHeight = QFontInfo(qApp->font()).pixelSize() * 3.5;
QImage img(QSize(iconHeight * pCore->getCurrentDar(), iconHeight), QImage::Format_ARGB32);
img.fill(Qt::darkGray);
QMap <int, QString> streams = audioInfo()->streams();
QMap <int, int> channelsList = audioInfo()->streamChannels();
QPainter painter(&img);
QPen pen = painter.pen();
pen.setColor(Qt::white);
painter.setPen(pen);
int streamCount = 0;
qreal indicesPrPixel = qreal(getFramePlaytime()) / img.width();
double increment = qMax(1., 1. / qAbs(indicesPrPixel));
if (streams.count() > 0) {
double streamHeight = iconHeight / streams.count();
QMapIterator<int, QString> st(streams);
while (st.hasNext()) {
st.next();
int channels = channelsList.value(st.key());
double channelHeight = (double) streamHeight / channels;
const QVector <uint8_t> audioLevels = audioFrameCache(st.key());
for (int channel = 0; channel < channels; channel++) {
double y = (streamHeight * streamCount) + (channel * channelHeight) + channelHeight / 2;
for (int i = 0; i <= img.width(); i++) {
double j = i * increment;
int idx = j * indicesPrPixel;
idx += idx % channels;
idx += channel;
if (idx >= audioLevels.length() || idx < 0) break;
double level = audioLevels.at(idx) * channelHeight / 510.; // divide height by 510 (2*255) to get height
painter.drawLine(i, y - level, i, y + level);
}
}
streamCount++;
}
}
thumb = img;
// Cache thumbnail
ThumbnailCache::get()->storeThumbnail(m_binId, 0, thumb, true);
}
setThumbnail(thumb);
}
if (!KdenliveSettings::audiothumbnails()) {
return;
}
......@@ -309,7 +353,7 @@ size_t ProjectClip::frameDuration() const
return (size_t)getFramePlaytime();
}
void ProjectClip::reloadProducer(bool refreshOnly, bool audioStreamChanged, bool reloadAudio)
void ProjectClip::reloadProducer(bool refreshOnly, bool isProxy)
{
// we find if there are some loading job on that clip
int loadjobId = -1;
......@@ -319,7 +363,7 @@ void ProjectClip::reloadProducer(bool refreshOnly, bool audioStreamChanged, bool
// In that case, we only want a new thumbnail.
// We thus set up a thumb job. We must make sure that there is no pending LOADJOB
// Clear cache first
ThumbnailCache::get()->invalidateThumbsForClip(clipId(), false);
ThumbnailCache::get()->invalidateThumbsForClip(clipId());
pCore->jobManager()->discardJobs(clipId(), AbstractClipJob::THUMBJOB);
m_thumbsProducer.reset();
emit pCore->jobManager()->startJob<ThumbJob>({clipId()}, loadjobId, QString(), -1, true, true);
......@@ -352,14 +396,11 @@ void ProjectClip::reloadProducer(bool refreshOnly, bool audioStreamChanged, bool
}
}
ThumbnailCache::get()->invalidateThumbsForClip(clipId(), reloadAudio);
ThumbnailCache::get()->invalidateThumbsForClip(clipId());
int loadJob = pCore->jobManager()->startJob<LoadJob>({clipId()}, loadjobId, QString(), xml);
emit pCore->jobManager()->startJob<ThumbJob>({clipId()}, loadJob, QString(), -1, true, true);
if (audioStreamChanged || hashChanged) {
if (!isProxy && hashChanged) {
discardAudioThumb();
} else {
// refresh bin/monitor mini thumb only
discardAudioThumb(true);
}
if (KdenliveSettings::audiothumbnails()) {
emit pCore->jobManager()->startJob<AudioThumbJob>({clipId()}, loadjobId, QString());
......@@ -1124,7 +1165,7 @@ void ProjectClip::setProperties(const QMap<QString, QString> &properties, bool r
setProducerProperty(QStringLiteral("_overwriteproxy"), 1);
emit pCore->jobManager()->startJob<ProxyJob>({clipId()}, -1, QString());
} else {
reloadProducer(refreshOnly, audioStreamChanged, audioStreamChanged || (!refreshOnly && !properties.contains(QStringLiteral("kdenlive:proxy"))));
reloadProducer(refreshOnly, properties.contains(QStringLiteral("kdenlive:proxy")));
}
if (refreshOnly) {
if (auto ptr = m_model.lock()) {
......@@ -1292,7 +1333,7 @@ int ProjectClip::audioChannels() const
return audioInfo()->channels();
}
void ProjectClip::discardAudioThumb(bool miniThumbOnly)
void ProjectClip::discardAudioThumb()
{
if (!m_audioInfo) {
return;
......@@ -1300,18 +1341,16 @@ void ProjectClip::discardAudioThumb(bool miniThumbOnly)
pCore->jobManager()->discardJobs(clipId(), AbstractClipJob::AUDIOTHUMBJOB);
QString audioThumbPath;
QList <int> streams = m_audioInfo->streams().keys();
if (!miniThumbOnly) {
// Delete audio thumbnail data
for (int &st : streams) {
audioThumbPath = getAudioThumbPath(st);
if (!audioThumbPath.isEmpty()) {
QFile::remove(audioThumbPath);
}
// Delete audio thumbnail data
for (int &st : streams) {
audioThumbPath = getAudioThumbPath(st);
if (!audioThumbPath.isEmpty()) {
QFile::remove(audioThumbPath);
}
}
// Delete mini thumb
// Delete thumbnail
for (int &st : streams) {
audioThumbPath = getAudioThumbPath(st, true);
audioThumbPath = getAudioThumbPath(st);
if (!audioThumbPath.isEmpty()) {
QFile::remove(audioThumbPath);
}
......@@ -1332,9 +1371,9 @@ int ProjectClip::getAudioStreamFfmpegIndex(int mltStream)
return -1;
}
const QString ProjectClip::getAudioThumbPath(int stream, bool miniThumb)
const QString ProjectClip::getAudioThumbPath(int stream)
{
if (audioInfo() == nullptr && !miniThumb) {
if (audioInfo() == nullptr) {
return QString();
}
bool ok = false;
......@@ -1348,10 +1387,6 @@ const QString ProjectClip::getAudioThumbPath(int stream, bool miniThumb)
}
QString audioPath = thumbFolder.absoluteFilePath(clipHash);
audioPath.append(QLatin1Char('_') + QString::number(stream));
if (miniThumb) {
audioPath.append(QStringLiteral(".png"));
return audioPath;
}
int roundedFps = (int)pCore->getCurrentFps();
audioPath.append(QStringLiteral("_%1_audio.png").arg(roundedFps));
return audioPath;
......
src/bin/projectclip.h
View file @ a0db1142
......@@ -81,7 +81,7 @@ protected:
public:
~ProjectClip() override;
void reloadProducer(bool refreshOnly = false, bool audioStreamChanged = false, bool reloadAudio = true);
void reloadProducer(bool refreshOnly = false, bool isProxy = false);
/** @brief Returns a unique hash identifier used to store clip thumbnails. */
// virtual void hash() = 0;
......@@ -185,9 +185,9 @@ public:
/** @brief Returns the list of this clip's subclip's ids. */
QStringList subClipIds() const;
/** @brief Delete cached audio thumb - needs to be recreated */
void discardAudioThumb(bool miniThumbOnly = false);
void discardAudioThumb();
/** @brief Get path for this clip's audio thumbnail */
const QString getAudioThumbPath(int stream, bool miniThumb = false);
const QString getAudioThumbPath(int stream);
/** @brief Returns true if this producer has audio and can be splitted on timeline*/
bool isSplittable() const;
......
src/jobs/audiothumbjob.cpp
View file @ a0db1142
......@@ -120,46 +120,13 @@ bool AudioThumbJob::computeWithFFMPEG()
if (!QFile::exists(filePath)) {
return false;
}
m_ffmpegProcess.reset(new QProcess);
QString thumbPath = m_binClip->getAudioThumbPath(m_audioStream, true);
int audioStreamIndex = m_binClip->getAudioStreamFfmpegIndex(m_audioStream);
if (!QFile::exists(thumbPath)) {
// Generate thumbnail used in monitor overlay
QStringList args = {QStringLiteral("-hide_banner"), QStringLiteral("-y"), QStringLiteral("-i"), QUrl::fromLocalFile(filePath).toLocalFile(), QString("-filter_complex")};
if (m_audioStream >= 0) {
args << QString("[a:%1]showwavespic=s=%2x%3:split_channels=1:scale=cbrt:colors=%4|%5").arg(audioStreamIndex).arg(m_thumbSize.width()).arg(m_thumbSize.height()).arg(KdenliveSettings::thumbColor1().name(), KdenliveSettings::thumbColor2().name());
} else {
// Only 1 audio stream in clip
args << QString("[a]showwavespic=s=%2x%3:split_channels=1:scale=cbrt:colors=%4|%5").arg(m_thumbSize.width()).arg(m_thumbSize.height()).arg(KdenliveSettings::thumbColor1().name(), KdenliveSettings::thumbColor2().name());
}
args << QStringLiteral("-frames:v") << QStringLiteral("1");
args << thumbPath;
qDebug()<<"=== FFARGS: "<<args;
connect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress, Qt::UniqueConnection);
connect(this, &AudioThumbJob::jobCanceled, [&] () {
if (m_ffmpegProcess) {
disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
m_ffmpegProcess->kill();
}
m_done = true;
m_successful = false;
});
m_ffmpegProcess->start(KdenliveSettings::ffmpegpath(), args);
m_ffmpegProcess->waitForFinished(-1);
disconnect(m_ffmpegProcess.get(), &QProcess::readyReadStandardOutput, this, &AudioThumbJob::updateFfmpegProgress);
if (m_ffmpegProcess->exitStatus() != QProcess::CrashExit) {
if (m_dataInCache || !KdenliveSettings::audiothumbnails()) {
m_done = true;
}
}
} else if (!KdenliveSettings::audiothumbnails()) {
m_done = true;
}
if (!KdenliveSettings::audiothumbnails()) {
// We only wanted the thumb generation
return m_done;
}
if (!QFile::exists(m_cachePath) && !m_dataInCache && !m_done) {
int audioStreamIndex = m_binClip->getAudioStreamFfmpegIndex(m_audioStream);
if (!QFile::exists(m_cachePath) && !m_dataInCache) {
// Generate timeline audio thumbnail data
m_audioLevels.clear();
std::vector<std::unique_ptr<QTemporaryFile>> channelFiles;
......@@ -184,9 +151,9 @@ bool AudioThumbJob::computeWithFFMPEG()
args << QStringLiteral("-filter_complex");
if (m_channels == 1) {
if (m_audioStream >= 0) {
args << QStringLiteral("[a:%1]aformat=channel_layouts=mono,%2=100").arg(audioStreamIndex).arg(isFFmpeg ? "aresample=async" : "sample_rates");
args << QStringLiteral("[a:%1]aformat=channel_layouts=mono,%2=100").arg(audioStreamIndex).arg(isFFmpeg ? "aresample=1500:async" : "sample_rates");
} else {
args << QStringLiteral("[a]aformat=channel_layouts=mono,%1=100").arg(isFFmpeg ? "aresample=async" : "sample_rates");
args << QStringLiteral("[a]aformat=channel_layouts=mono,%1=100").arg(isFFmpeg ? "aresample=1500:async" : "sample_rates");
}
/*args << QStringLiteral("-map") << QStringLiteral("0:a%1").arg(m_audioStream > 0 ? ":" + QString::number(audioStreamIndex) : QString())*/
args << QStringLiteral("-c:a") << QStringLiteral("pcm_s16le") << QStringLiteral("-frames:v")
......@@ -195,7 +162,7 @@ bool AudioThumbJob::computeWithFFMPEG()
} else {
QString aformat = QStringLiteral("[a%1]%2=100,channelsplit=channel_layout=%3")
.arg(audioStreamIndex >= 0 ? ":" + QString::number(audioStreamIndex) : QString(),
isFFmpeg ? "aresample=async" : "aformat=sample_rates",
isFFmpeg ? "aresample=1500:async" : "aformat=sample_rates",
m_channels > 2 ? "5.1" : "stereo");
for (int i = 0; i < m_channels; ++i) {
aformat.append(QStringLiteral("[0:%1]").arg(i));
......@@ -279,15 +246,16 @@ bool AudioThumbJob::computeWithFFMPEG()
}
m_done = true;
return true;
} else {
QString err = m_ffmpegProcess->readAllStandardError();
m_ffmpegProcess.reset();
// m_errorMessage += err;
// m_errorMessage.append(i18n("Failed to create FFmpeg audio thumbnails, we now try to use MLT"));
qWarning() << "Failed to create FFmpeg audio thumbs:\n" << err << "\n---------------------";
}
} else {
m_done = true;
}
QString err = m_ffmpegProcess->readAllStandardError();
m_ffmpegProcess.reset();
// m_errorMessage += err;
// m_errorMessage.append(i18n("Failed to create FFmpeg audio thumbnails, we now try to use MLT"));
qWarning() << "Failed to create FFmpeg audio thumbs:\n" << err << "\n---------------------";
return m_done;
}
......@@ -333,20 +301,6 @@ bool AudioThumbJob::startJob()
return false;
}
m_lengthInFrames = m_prod->get_length(); // Multiply this if we want more than 1 sample per frame
int thumbResolution = 3000;
// Increase audio thumb resolution for longer clips to get a better resolution
if (m_lengthInFrames > 10000) {
// More than 10 minutes at 25fps
if (m_lengthInFrames > 90000) {
// More than 1 hour at 25fps
thumbResolution = 10000;
} else {
thumbResolution = 6000;
}
}
m_thumbSize = QSize(thumbResolution, 1000 / pCore->getCurrentDar());
m_frequency = m_binClip->audioInfo()->samplingRate();
m_frequency = m_frequency <= 0 ? 48000 : m_frequency;
......@@ -354,12 +308,16 @@ bool AudioThumbJob::startJob()
m_channels = m_channels <= 0 ? 2 : m_channels;
QMap <int, QString> streams = m_binClip->audioInfo()->streams();
QMap <int, int> audioChannels = m_binClip->audioInfo()->streamChannels();
QMapIterator<int, QString> st(streams);
m_done = true;
ClipType::ProducerType type = m_binClip->clipType();
while (st.hasNext()) {
st.next();
int stream = st.key();
if (audioChannels.contains(stream)) {
m_channels = audioChannels.value(stream);
}
// Generate one thumb per stream
m_audioStream = stream;
m_cachePath = m_binClip->getAudioThumbPath(stream);
......@@ -427,26 +385,13 @@ bool AudioThumbJob::commitResult(Fun &undo, Fun &redo)
if (!m_successful) {
return false;
}
QImage oldImage;
QImage result;
if (m_binClip->clipType() == ClipType::Audio) {
oldImage = m_binClip->thumbnail(200, 200 / pCore->getCurrentDar()).toImage();
result = ThumbnailCache::get()->getAudioThumbnail(m_clipId).scaled(200, 200 / pCore->getCurrentDar());
}
// note that the image is moved into lambda, it won't be available from this class anymore
auto operation = [clip = m_binClip, image = std::move(result)]() {
auto operation = [clip = m_binClip]() {
clip->updateAudioThumbnail();
if (!image.isNull() && clip->clipType() == ClipType::Audio) {
clip->setThumbnail(image);
}
return true;
};
auto reverse = [clip = m_binClip, image = std::move(oldImage)]() {
auto reverse = [clip = m_binClip]() {
clip->updateAudioThumbnail();
if (!image.isNull() && clip->clipType() == ClipType::Audio) {
clip->setThumbnail(image);
}
return true;
};
bool ok = operation();
......
src/jobs/audiothumbjob.hpp
View file @ a0db1142
......@@ -64,12 +64,9 @@ protected:
private:
std::shared_ptr<ProjectClip> m_binClip;
std::shared_ptr<Mlt::Producer> m_prod;
QString m_miniThumbPath;
QString m_cachePath;
QSize m_thumbSize;
bool m_dataInCache;
bool m_thumbInCache;
bool m_done{false}, m_successful{false};
int m_channels, m_frequency, m_lengthInFrames, m_audioStream;
QVector <uint8_t>m_audioLevels;
......
src/jobs/proxyclipjob.cpp
View file @ a0db1142
......@@ -377,14 +377,14 @@ bool ProxyJob::commitResult(Fun &undo, Fun &redo)
binClip->setProducerProperty(QStringLiteral("_overwriteproxy"), QString());
const QString dest = binClip->getProducerProperty(QStringLiteral("kdenlive:proxy"));
binClip->setProducerProperty(QStringLiteral("resource"), dest);
pCore->bin()->reloadClip(clipId, false);
pCore->bin()->reloadClip(clipId);
return true;
};
auto reverse = [clipId = m_clipId]() {
auto binClip = pCore->projectItemModel()->getClipByBinID(clipId);
const QString dest = binClip->getProducerProperty(QStringLiteral("kdenlive:originalurl"));
binClip->setProducerProperty(QStringLiteral("resource"), dest);
pCore->bin()->reloadClip(clipId, false);
pCore->bin()->reloadClip(clipId);
return true;
};
bool ok = operation();
......
src/lib/audio/audioStreamInfo.cpp
View file @ a0db1142
......@@ -116,7 +116,12 @@ QMap <int, QString> AudioStreamInfo::streams() const
return m_audioStreams;
}
QList <int> AudioStreamInfo::streamChannels() const
QMap <int, int> AudioStreamInfo::streamChannels() const
{
return m_audioChannels;
}
QList <int> AudioStreamInfo::activeStreamChannels() const
{
if (m_activeStreams.size() == 1 && m_activeStreams.contains(INT_MAX)) {
return m_audioChannels.values();
......
src/lib/audio/audioStreamInfo.h
View file @ a0db1142
......@@ -31,7 +31,9 @@ public:
/** @brief returns a list of audio stream index > stream description */
QMap <int, QString> streams() const;
/** @brief returns a list of audio stream index > channels per stream */
QList <int> streamChannels() const;
QMap <int, int> streamChannels() const;
/** @brief returns a list of audio channels per active stream */
QList <int> activeStreamChannels() const;
/** @brief returns a list of enabled audio stream indexes > stream description */
QMap <int, QString> activeStreams() const;
int bitrate() const;
......
src/mltcontroller/clipcontroller.cpp
View file @ a0db1142
......@@ -1049,7 +1049,7 @@ QList <int> ClipController::activeStreamChannels() const
if (!audioInfo()) {
return QList <int>();
}
return audioInfo()->streamChannels();
return audioInfo()->activeStreamChannels();
}
QMap <int, QString> ClipController::activeStreams() const
......
src/utils/thumbnailcache.cpp
View file @ a0db1142
......@@ -223,7 +223,7 @@ void ThumbnailCache::saveCachedThumbs(QStringList keys)
}
}
void ThumbnailCache::invalidateThumbsForClip(const QString &binId, bool reloadAudio)
void ThumbnailCache::invalidateThumbsForClip(const QString &binId)
{
QMutexLocker locker(&m_mutex);
if (m_storedVolatile.find(binId) != m_storedVolatile.end()) {
......@@ -243,16 +243,7 @@ void ThumbnailCache::invalidateThumbsForClip(const QString &binId, bool reloadAu
if (ok && m_storedOnDisk.find(binId) != m_storedOnDisk.end()) {
// Remove persistent cache
for (int pos : m_storedOnDisk.at(binId)) {
if (pos < 0) {
if (reloadAudio) {
auto key = getAudioKey(binId, &ok);
if (ok) {
for (const QString &p : qAsConst(key)) {
QFile::remove(audioThumbFolder.absoluteFilePath(p));
}
}
}
} else {
if (pos >= 0) {
auto key = getKey(binId, pos, &ok);
if (ok) {
QFile::remove(thumbFolder.absoluteFilePath(key));
......
src/utils/thumbnailcache.hpp
View file @ a0db1142
......@@ -71,7 +71,7 @@ public:
void storeThumbnail(const QString &binId, int pos, const QImage &img, bool persistent = false);
/* @brief Removes all the thumbnails for a given clip */
void invalidateThumbsForClip(const QString &binId, bool reloadAudio);
void invalidateThumbsForClip(const QString &binId);
/* @brief Save all cached thumbs to disk */
void saveCachedThumbs(QStringList keys);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment