Commit f9865633 authored by Jean-Baptiste Mardelle's avatar Jean-Baptiste Mardelle
Browse files

copy audio thumbnails from shotcut.

We lose some precision but it is faster and cleaner
parent f7c6b422
......@@ -2074,3 +2074,17 @@ void Bin::emitMessage(const QString &text, MessageType type)
emit displayMessage(text, type);
}
void Bin::slotCreateAudioThumb(const QString &id)
{
ProjectClip *clip = m_rootFolder->clip(id);
if (!clip) return;
clip->createAudioThumbs();
}
void Bin::slotAbortAudioThumb(const QString &id)
{
ProjectClip *clip = m_rootFolder->clip(id);
if (!clip) return;
clip->abortAudioThumbs();
}
......@@ -484,6 +484,10 @@ public slots:
void slotLoadClipMarkers(const QString &id);
void slotDuplicateClip();
void slotDeleteEffect(const QString &id, QDomElement effect);
/** @brief Request audio thumbnail for clip with id */
void slotCreateAudioThumb(const QString &id);
/** @brief Abort audio thumbnail for clip with id */
void slotAbortAudioThumb(const QString &id);
protected:
void contextMenuEvent(QContextMenuEvent *event);
......
......@@ -46,7 +46,6 @@ ProjectClip::ProjectClip(const QString &id, QIcon thumb, ClipController *control
AbstractProjectItem(AbstractProjectItem::ClipItem, id, parent)
, m_controller(controller)
, audioFrameCache()
, m_audioThumbCreated(false)
, m_gpuProducer(NULL)
, m_abortAudioThumb(false)
{
......@@ -58,7 +57,7 @@ ProjectClip::ProjectClip(const QString &id, QIcon thumb, ClipController *control
setParent(parent);
bin()->loadSubClips(id, m_controller->getSubClips());
if (KdenliveSettings::audiothumbnails()) {
QtConcurrent::run(this, &ProjectClip::slotCreateAudioThumbs);
m_audioThumbsThread = QtConcurrent::run(this, &ProjectClip::slotCreateAudioThumbs);
}
}
......@@ -66,7 +65,6 @@ ProjectClip::ProjectClip(const QDomElement& description, QIcon thumb, ProjectFol
AbstractProjectItem(AbstractProjectItem::ClipItem, description, parent)
, m_controller(NULL)
, audioFrameCache()
, m_audioThumbCreated(false)
, m_gpuProducer(NULL)
, m_abortAudioThumb(false)
{
......@@ -89,6 +87,8 @@ ProjectClip::ProjectClip(const QDomElement& description, QIcon thumb, ProjectFol
ProjectClip::~ProjectClip()
{
// controller is deleted in bincontroller
m_abortAudioThumb = true;
m_audioThumbsThread.waitForFinished();
}
QString ProjectClip::getToolTip() const
......@@ -109,11 +109,11 @@ QString ProjectClip::getXmlProperty(const QDomElement &producer, const QString &
return value;
}
void ProjectClip::updateAudioThumbnail(const audioByteArray& data)
void ProjectClip::updateAudioThumbnail(QVariantList* audioLevels)
{
////qDebug() << "CLIPBASE RECIEDVED AUDIO DATA*********************************************";
audioFrameCache = data;
m_audioThumbCreated = true;
audioFrameCache = audioLevels;
m_controller->audioThumbCreated = true;
emit gotAudioData();
}
......@@ -125,7 +125,7 @@ QList < CommentedTime > ProjectClip::commentedSnapMarkers() const
bool ProjectClip::audioThumbCreated() const
{
return m_audioThumbCreated;
return (m_controller && m_controller->audioThumbCreated);
}
ClipType ProjectClip::clipType() const
......@@ -273,7 +273,21 @@ void ProjectClip::setProducer(ClipController *controller, bool replaceProducer)
m_clipStatus = StatusReady;
bin()->emitItemUpdated(this);
getFileHash();
if (KdenliveSettings::audiothumbnails()) QtConcurrent::run(this, &ProjectClip::slotCreateAudioThumbs);
createAudioThumbs();
}
void ProjectClip::createAudioThumbs()
{
if (KdenliveSettings::audiothumbnails() && !m_audioThumbsThread.isRunning()) {
m_audioThumbsThread = QtConcurrent::run(this, &ProjectClip::slotCreateAudioThumbs);
}
}
void ProjectClip::abortAudioThumbs()
{
if (m_audioThumbsThread.isRunning()) {
m_abortAudioThumb = true;
}
}
Mlt::Producer *ProjectClip::producer()
......@@ -660,6 +674,12 @@ void ProjectClip::slotExtractImage(QList <int> frames)
}
}
int ProjectClip::audioChannels() const
{
if (!m_controller || !m_controller->audioInfo()) return 0;
return m_controller->audioInfo()->channels();
}
void ProjectClip::slotCreateAudioThumbs()
{
Mlt::Producer *prod = producer();
......@@ -667,7 +687,7 @@ void ProjectClip::slotCreateAudioThumbs()
if (audioInfo == NULL) return;
QString clipHash = hash();
if (clipHash.isEmpty()) return;
QString audioPath = bin()->projectFolder().path() + "/thumbs/" + clipHash + ".thumb";
QString audioPath = bin()->projectFolder().path() + "/thumbs/" + clipHash + "_audio.png";
double lengthInFrames = prod->get_playtime();
int frequency = audioInfo->samplingRate();
if (frequency <= 0) frequency = 48000;
......@@ -676,109 +696,95 @@ void ProjectClip::slotCreateAudioThumbs()
int arrayWidth = 20;
double frame = 0.0;
int maxVolume = 0;
audioByteArray storeIn;
QFile f(audioPath);
if (QFileInfo(audioPath).size() > 0 && f.open(QIODevice::ReadOnly)) {
bool reading = true;
const QByteArray channelarray = f.readAll();
f.close();
if (channelarray.size() != arrayWidth*(frame + lengthInFrames) * channels) {
//qDebug() << "--- BROKEN THUMB FOR: " << url.fileName() << " ---------------------- ";
f.remove();
reading = false;
}
//qDebug() << "reading audio thumbs from file";
if (reading) {
int h1 = arrayWidth * channels;
int h2 = (int) frame * h1;
for (int z = (int) frame; z < (int)(frame + lengthInFrames) && !m_abortAudioThumb; ++z) {
int h3 = 0;
for (int c = 0; c < channels; ++c) {
QByteArray audioArray(arrayWidth, '\x00');
for (int i = 0; i < arrayWidth; ++i) {
audioArray[i] = channelarray.at(h2 + h3 + i);
if (audioArray.at(i) > maxVolume) maxVolume = audioArray.at(i);
}
h3 += arrayWidth;
storeIn[z][c] = audioArray;
}
h2 += h1;
}
if (!m_abortAudioThumb) {
setProducerProperty("audio_max", QString::number(maxVolume - 64));
updateAudioThumbnail(storeIn);
}
return;
QVariantList* audioLevels = new QVariantList;
QImage image(audioPath);
if (!image.isNull()) {
// convert cached image
int n = image.width() * image.height();
for (int i = 0; i < n; i++) {
QRgb p = image.pixel(i / 2, i % channels);
*audioLevels << qRed(p);
*audioLevels << qGreen(p);
*audioLevels << qBlue(p);
*audioLevels << qAlpha(p);
}
}
if (!f.open(QIODevice::WriteOnly)) {
//qDebug() << "++++++++ ERROR WRITING TO FILE: " << audioPath;
//qDebug() << "++++++++ DISABLING AUDIO THUMBS";
KdenliveSettings::setAudiothumbnails(false);
if (audioLevels->size() > 0) {
updateAudioThumbnail(audioLevels);
return;
}
QString service = prod->get("mlt_service");
if (service == "avformat-novalidate")
service = "avformat";
else if (service.startsWith("xml"))
service = "xml-nogl";
Mlt::Producer *audioProducer = new Mlt::Producer(*prod->profile(), service.toUtf8().constData(), prod->get("resource"));
if (!audioProducer->is_valid()) {
//qDebug() << "++++++++ INVALID CLIP: " << url.path();
delete audioProducer;
return;
}
audioProducer->set("video_index", "-1");
//if (KdenliveSettings::normaliseaudiothumbs()) {
//Mlt::Filter m_convert(prof, "volume");
//m_convert.set("gain", "normalise");
//producer.attach(m_convert);
//}
Mlt::Filter chans(*prod->profile(), "audiochannels");
Mlt::Filter converter(*prod->profile(), "audioconvert");
Mlt::Filter levels(*prod->profile(), "audiolevel");
audioProducer->attach(chans);
audioProducer->attach(converter);
audioProducer->attach(levels);
audioProducer->set("video_index", "-1");
int last_val = 0;
setJobStatus(AbstractClipJob::THUMBJOB, JobWaiting, 0, i18n("Creating audio thumbnails"));
double framesPerSecond = audioProducer->get_fps();
mlt_audio_format audioFormat = mlt_audio_s16;
for (int z = (int) frame; z < (int)(frame + lengthInFrames) && !m_abortAudioThumb; ++z) {
QStringList keys;
for (int i = 0; i < channels; i++) {
keys << "meta.media.audio_level." + QString::number(i);
}
for (int z = (int) frame; z < (int)(frame + lengthInFrames) && !m_abortAudioThumb; ++z) {
int val = (int)((z - frame) / (frame + lengthInFrames) * 100.0);
if (last_val != val && val > 1) {
setJobStatus(AbstractClipJob::THUMBJOB, JobWorking, val);
last_val = val;
}
audioProducer->seek(z);
Mlt::Frame *mlt_frame = audioProducer->get_frame();
if (mlt_frame && mlt_frame->is_valid()) {
int samples = mlt_sample_calculator(framesPerSecond, frequency, mlt_frame->get_position());
qint16* pcm = static_cast<qint16*>(mlt_frame->get_audio(audioFormat, frequency, channels, samples));
for (int c = 0; c < channels; ++c) {
QByteArray audioArray;
audioArray.resize(arrayWidth);
for (int i = 0; i < audioArray.size(); ++i) {
double pcmval = *(pcm + c + i * samples / audioArray.size());
if (pcmval >= 0) {
pcmval = sqrt(pcmval) / 2.83 + 64;
audioArray[i] = pcmval;
if (pcmval > maxVolume) maxVolume = pcmval;
}
else {
pcmval = -sqrt(-pcmval) / 2.83 + 64;
audioArray[i] = pcmval;
if (-pcmval > maxVolume) maxVolume = -pcmval;
}
}
f.write(audioArray);
storeIn[z][c] = audioArray;
if (mlt_frame && mlt_frame->is_valid() && !mlt_frame->get_int("test_audio")) {
int samples = mlt_sample_calculator(framesPerSecond, frequency, z);
mlt_frame->get_audio(audioFormat, frequency, channels, samples);
for (int channel = 0; channel < channels; ++channel) {
double level = 256 * qMin(mlt_frame->get_double(keys.at(channel).toUtf8().constData()) * 0.9, 1.0);
*audioLevels << level;
}
} else {
f.write(QByteArray(arrayWidth, '\x00'));
} else if (!audioLevels->isEmpty()) {
for (int channel = 0; channel < channels; channel++)
*audioLevels << audioLevels->last();
}
delete mlt_frame;
}
f.close();
if (!m_abortAudioThumb && audioLevels->size() > 0) {
// Put into an image for caching.
int count = audioLevels->size();
QImage image((count + 3) / 4, channels, QImage::Format_ARGB32);
int n = image.width() * image.height();
for (int i = 0; i < n; i ++) {
QRgb p;
if ((4*i + 3) < count) {
p = qRgba(audioLevels->at(4*i).toInt(), audioLevels->at(4*i+1).toInt(), audioLevels->at(4*i+2).toInt(), audioLevels->at(4*i+3).toInt());
} else {
int last = audioLevels->last().toInt();
int r = (4*i+0) < count? audioLevels->at(4*i+0).toInt() : last;
int g = (4*i+1) < count? audioLevels->at(4*i+1).toInt() : last;
int b = (4*i+2) < count? audioLevels->at(4*i+2).toInt() : last;
int a = last;
p = qRgba(r, g, b, a);
}
image.setPixel(i / 2, i % channels, p);
}
image.save(audioPath);
}
delete audioProducer;
setJobStatus(AbstractClipJob::THUMBJOB, JobDone, 0, i18n("Audio thumbnails done"));
if (m_abortAudioThumb) {
f.remove();
} else {
updateAudioThumbnail(storeIn);
setProducerProperty("audio_max", QString::number(maxVolume - 64));
if (!m_abortAudioThumb) {
updateAudioThumbnail(audioLevels);
}
m_abortAudioThumb = false;
}
......@@ -29,6 +29,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
#include <QUrl>
#include <QMutex>
#include <QFuture>
class ProjectFolder;
class QDomElement;
......@@ -173,7 +174,7 @@ public:
/** Cache for every audio Frame with 10 Bytes */
/** format is frame -> channel ->bytes */
QMap<int, QMap<int, QByteArray> > audioFrameCache;
QVariantList *audioFrameCache;
bool audioThumbCreated() const;
void updateParentInfo(const QString &folderid, const QString &foldername);
......@@ -191,9 +192,15 @@ public:
/** @brief Add an effect to bin clip. */
void addEffect(const ProfileInfo pInfo, QDomElement &effect);
void removeEffect(const ProfileInfo pInfo, int ix);
/** @brief Create audio thumbnail for this clip. */
void createAudioThumbs();
/** @brief Abort audio thumbnail for this clip. */
void abortAudioThumbs();
/** @brief Returns the number of audio channels. */
int audioChannels() const;
public slots:
void updateAudioThumbnail(const audioByteArray& data);
void updateAudioThumbnail(QVariantList* audioLevels);
void slotExtractImage(QList <int> frames);
void slotCreateAudioThumbs();
......@@ -208,10 +215,11 @@ private:
Mlt::Producer *m_gpuProducer;
/** @brief Generate and store file hash if not available. */
const QString getFileHash() const;
bool m_audioThumbCreated;
/** @brief Store clip url temporarily while the clip controller has not been created. */
QUrl m_temporaryUrl;
bool m_abortAudioThumb;
/** @brief Indicates whether audio thumbnail creation is running. */
QFuture<void> m_audioThumbsThread;
signals:
void gotAudioData();
......
......@@ -53,6 +53,8 @@ void Core::init()
m_binController = new BinController();
connect(m_binWidget, SIGNAL(storeFolder(QString,QString,QString,QString)), m_binController, SLOT(slotStoreFolder(QString,QString,QString,QString)));
connect(m_binController, SIGNAL(loadFolders(QMap<QString,QString>)), m_binWidget, SLOT(slotLoadFolders(QMap<QString,QString>)));
connect(m_binController, SIGNAL(requestAudioThumb(QString)), m_binWidget, SLOT(slotCreateAudioThumb(QString)));
connect(m_binController, SIGNAL(abortAudioThumb(QString)), m_binWidget, SLOT(slotAbortAudioThumb(QString)));
connect(m_binController, SIGNAL(loadThumb(QString,QImage,bool)), m_binWidget, SLOT(slotThumbnailReady(QString,QImage,bool)));
m_monitorManager = new MonitorManager(this);
emit coreIsReady();
......
......@@ -354,11 +354,6 @@ void KThumb::getThumbs(QUrl url, int startframe, int endframe, int width, int he
}
*/
void KThumb::slotCreateAudioThumbs()
{
m_clipManager->askForAudioThumb(m_id);
}
void KThumb::queryIntraThumbs(const QSet <int> &missingFrames)
{
m_intraMutex.lock();
......
......@@ -68,7 +68,6 @@ public:
public slots:
void updateClipUrl(const QUrl &url, const QString &hash);
void slotCreateAudioThumbs();
public:
static QPixmap getImage(const QUrl &url, int width, int height);
......
......@@ -176,12 +176,7 @@
<entry name="displayallchannels" type="Bool">
<label>Display all channels in audio thumbnails.</label>
<default>true</default>
</entry>
<entry name="normaliseaudiothumbs" type="Bool">
<label>Normalise audio before creating thumbnails.</label>
<default>true</default>
<default>false</default>
</entry>
<entry name="autoscroll" type="Bool">
......
......@@ -1749,8 +1749,6 @@ void MainWindow::updateConfiguration()
pCore->projectManager()->currentTimeline()->refresh();
pCore->projectManager()->currentTimeline()->projectView()->checkAutoScroll();
pCore->projectManager()->currentTimeline()->checkTrackHeight();
if (pCore->projectManager()->current())
pCore->projectManager()->current()->clipManager()->checkAudioThumbs();
}
m_buttonAudioThumbs->setChecked(KdenliveSettings::audiothumbnails());
m_buttonVideoThumbs->setChecked(KdenliveSettings::videothumbnails());
......@@ -1780,12 +1778,10 @@ void MainWindow::slotSwitchVideoThumbs()
void MainWindow::slotSwitchAudioThumbs()
{
KdenliveSettings::setAudiothumbnails(!KdenliveSettings::audiothumbnails());
pCore->binController()->checkAudioThumbs();
if (pCore->projectManager()->currentTimeline()) {
pCore->projectManager()->currentTimeline()->refresh();
pCore->projectManager()->currentTimeline()->projectView()->checkAutoScroll();
if (pCore->projectManager()->current()) {
pCore->projectManager()->current()->clipManager()->checkAudioThumbs();
}
}
m_buttonAudioThumbs->setChecked(KdenliveSettings::audiothumbnails());
}
......
......@@ -84,7 +84,6 @@ void BinController::destroyBin()
delete m_binPlaylist;
m_binPlaylist = NULL;
}
// Controllers are deleted from the Bin's ProjectClip
qDeleteAll(m_clipList.values());
m_clipList.clear();
}
......@@ -439,3 +438,22 @@ void BinController::checkThumbnails(const QString thumbFolder)
}
}
}
void BinController::checkAudioThumbs()
{
QMapIterator<QString, ClipController *> i(m_clipList);
while (i.hasNext()) {
i.next();
ClipController *ctrl = i.value();
if (!ctrl->audioThumbCreated) {
if (KdenliveSettings::audiothumbnails()) {
// We want audio thumbnails
emit requestAudioThumb(ctrl->clipId());
} else {
// Abort all pending thumb creation
emit abortAudioThumb(ctrl->clipId());
}
}
}
}
......@@ -143,6 +143,9 @@ public:
/** @brief Load thumbnails for all producers */
void checkThumbnails(const QString thumbFolder);
/** @brief Request audio thumbnails for all producers */
void checkAudioThumbs();
public slots:
/** @brief Stored a Bin Folder id / name to MLT's bin playlist. Using an empry folderName deletes the property */
void slotStoreFolder(const QString &folderId, const QString &parentId, const QString &oldParentId, const QString &folderName);
......@@ -174,6 +177,8 @@ signals:
void loadThumb(QString,QImage,bool);
void createThumb(const QDomElement&,const QString&,int);
void reloadTrackProducers(const QString &id);
void requestAudioThumb(const QString&);
void abortAudioThumb(const QString&);
};
#endif
......@@ -40,6 +40,7 @@ ClipController::ClipController(BinController *bincontroller, Mlt::Producer& prod
, m_properties(new Mlt::Properties(producer.get_properties()))
, selectedEffectIndex(1)
, m_audioInfo(NULL)
, audioThumbCreated(false)
{
m_masterProducer = &producer;
m_effectList = EffectsList(true);
......@@ -68,6 +69,7 @@ ClipController::ClipController(BinController *bincontroller) : QObject()
, m_properties(NULL)
, selectedEffectIndex(1)
, m_audioInfo(NULL)
, audioThumbCreated(false)
{
m_masterProducer = NULL;
m_effectList = EffectsList(true);
......@@ -657,7 +659,3 @@ bool ClipController::hasEffects() const
return !m_effectList.isEmpty();
}
......@@ -167,9 +167,12 @@ public:
/** @brief Enable/disable an effect. */
void changeEffectState(const QList <int> indexes, bool disable);
void updateEffect(const ProfileInfo pInfo, const QDomElement &old, const QDomElement &e, int ix);
/** @brief Returns true if the bin clip has effects */
bool hasEffects() const;
/** @brief Returns info about clip audio */
AudioStreamInfo *audioInfo() const;
/** @brief Returns true if audio thumbnails for this clip are cached */
bool audioThumbCreated;
private:
Mlt::Producer *m_masterProducer;
......
......@@ -149,10 +149,6 @@ void ClipManager::stopThumbs(const QString &id)
if (!m_thumbsThread.isRunning() && !m_requestedThumbs.isEmpty()) {
m_thumbsThread = QtConcurrent::run(this, &ClipManager::slotGetThumbs);
}
if (!m_audioThumbsThread.isRunning() && !m_audioThumbsQueue.isEmpty()) {
m_audioThumbsThread = QtConcurrent::run(this, &ClipManager::slotGetAudioThumbs);
}
}
void ClipManager::slotGetThumbs()
......@@ -208,181 +204,6 @@ void ClipManager::slotGetThumbs()
emit displayMessage(QString(), -1);
}
void ClipManager::checkAudioThumbs()
{
if (!KdenliveSettings::audiothumbnails()) {
if (m_audioThumbsThread.isRunning()) {
m_abortAudioThumb = true;
m_thumbsMutex.lock();
m_audioThumbsQueue.clear();
m_thumbsMutex.unlock();
m_audioThumbsThread.waitForFinished();
m_abortAudioThumb = false;
}
return;
}
m_thumbsMutex.lock();
//TODO
/*
for (int i = 0; i < m_clipList.count(); ++i) {
DocClipBase *clip = m_clipList.at(i);
if (clip->hasAudioThumb() && !clip->audioThumbCreated())
m_audioThumbsQueue.append(m_clipList.at(i)->getId());
}
*/
m_thumbsMutex.unlock();
if (!m_audioThumbsThread.isRunning() && !m_audioThumbsQueue.isEmpty()) {
m_audioThumbsThread = QtConcurrent::run(this, &ClipManager::slotGetAudioThumbs);
}
}
void ClipManager::askForAudioThumb(const QString &id)
{
/*DocClipBase *clip = getClipById(id);
if (clip && KdenliveSettings::audiothumbnails() && (clip->hasAudioThumb())) {
m_thumbsMutex.lock();
if (!m_audioThumbsQueue.contains(id)) m_audioThumbsQueue.append(id);
m_thumbsMutex.unlock();
if (!m_audioThumbsThread.isRunning()) m_audioThumbsThread = QtConcurrent::run(this, &ClipManager::slotGetAudioThumbs);
}*/
}
void ClipManager::slotGetAudioThumbs()
{
/*
Mlt::Profile prof((char*) KdenliveSettings::current_profile().toUtf8().constData());
mlt_audio_format audioFormat = mlt_audio_s16;
while (!m_abortAudioThumb && !m_audioThumbsQueue.isEmpty()) {
m_thumbsMutex.lock();
m_processingAudioThumbId = m_audioThumbsQueue.takeFirst();
m_thumbsMutex.unlock();
DocClipBase *clip = getClipById(m_processingAudioThumbId);
if (!clip || clip->audioThumbCreated()) continue;
QUrl url = clip->fileURL();
QString hash = clip->getClipHash();
if (hash.isEmpty()) continue;
QString audioPath = projectFolder() + "/thumbs/" + hash + ".thumb";
double lengthInFrames = clip->duration().frames(m_doc->fps());
int frequency = 0;
int channels = 0;
QString data = clip->getProperty("frequency");
if (!data.isEmpty()) frequency = data.toInt();
if (frequency <= 0) frequency = 48000;
data = clip->getProperty("channels");
if (!data.isEmpty()) channels = data.toInt();
if (channels <= 0) channels = 2;
int arrayWidth = 20;
double frame = 0.0;
int maxVolume = 0;
audioByteArray storeIn;
QFile f(audioPath);
if (QFileInfo(audioPath).size() > 0 && f.open(QIODevice::ReadOnly)) {
const QByteArray channelarray = f.readAll();
f.close();
if (channelarray.size() != arrayWidth*(frame + lengthInFrames) * channels) {
//qDebug() << "--- BROKEN THUMB FOR: " << url.fileName() << " ---------------------- ";
f.remove();
continue;
}
//qDebug() << "reading audio thumbs from file";
int h1 = arrayWidth * channels;
int h2 = (int) frame * h1;
for (int z = (int) frame; z < (int)(frame + lengthInFrames) && !m_abortAudioThumb; ++z) {
int h3 = 0;
for (int c = 0; c < channels; ++c) {