Members of the KDE Community are recommended to subscribe to the kde-community mailing list at https://mail.kde.org/mailman/listinfo/kde-community to allow them to participate in important discussions and receive other important announcements

Commit bf4df116 authored by Michael Pyne's avatar Michael Pyne

startup: Async-ify cached item loading on startup, add timers.

In an attempt to get rid of processEvents() (related to several existing
crash bugs) I am trying to port the startup code towards more
async-friendly schemes.

There's no threading but we at least get back to the event loop much
more frequently while loading files.

Additionally I have added debug output with instrumentation to show how
long it takes to advance through each step of the startup (I think this
might be the first time anyone has understood JuK startup sequence in
years).

This leaves some essentially dead code with Cache (which no longer
acts as a container), which I will try to cleanup in later commits.
parent 20cf50ef
......@@ -40,7 +40,8 @@
using namespace ActionCollection;
static const int playlistCacheVersion = 3;
const int Cache::playlistListCacheVersion = 3;
const int Cache::playlistItemsCacheVersion = 2;
enum PlaylistType
{
......@@ -58,53 +59,9 @@ enum PlaylistType
Cache *Cache::instance()
{
static Cache cache;
// load() indirectly calls instance() so we have to protect against recursion.
static bool loaded = false;
if(!loaded) {
loaded = true;
cache.load();
}
return &cache;
}
void Cache::save()
{
QString dirName = KGlobal::dirs()->saveLocation("appdata");
QString cacheFileName = dirName + "cache";
KSaveFile f(cacheFileName);
if(!f.open(QIODevice::WriteOnly)) {
kError() << "Error saving cache:" << f.errorString();
return;
}
QByteArray data;
QDataStream s(&data, QIODevice::WriteOnly);
s.setVersion(QDataStream::Qt_4_3);
for(Iterator it = begin(); it != end(); ++it) {
s << (*it).absFilePath();
s << *it;
}
QDataStream fs(&f);
qint32 checksum = qChecksum(data.data(), data.size());
fs << qint32(m_currentVersion)
<< checksum
<< data;
f.close();
if(!f.finalize())
kError() << "Error saving cache:" << f.errorString();
}
void Cache::loadPlaylists(PlaylistCollection *collection) // static
{
QString playlistsFile = KGlobal::dirs()->saveLocation("appdata") + "playlists";
......@@ -318,7 +275,7 @@ void Cache::savePlaylists(const PlaylistList &playlists)
}
QDataStream fs(&f);
fs << qint32(playlistCacheVersion);
fs << qint32(playlistListCacheVersion);
fs << qChecksum(data.data(), data.size());
fs << data;
......@@ -342,25 +299,20 @@ Cache::Cache() : FileHandleHash()
}
void Cache::load()
bool Cache::prepareToLoadCachedItems()
{
QString cacheFileName = KGlobal::dirs()->saveLocation("appdata") + "cache";
QFile f(cacheFileName);
m_loadFile.setFileName(cacheFileName);
if(!m_loadFile.open(QIODevice::ReadOnly))
return false;
if(!f.open(QIODevice::ReadOnly))
return;
m_loadDataStream.setDevice(&m_loadFile);
CacheDataStream s(&f);
int dataStreamVersion = CacheDataStream::Qt_3_3;
qint32 version;
s >> version;
QBuffer buffer;
QByteArray data;
// Do the version specific stuff.
m_loadDataStream >> version;
switch(version) {
case 2:
......@@ -370,46 +322,68 @@ void Cache::load()
// to setCacheVersion
case 1: {
s.setCacheVersion(1);
s.setVersion(dataStreamVersion);
m_loadDataStream.setCacheVersion(1);
m_loadDataStream.setVersion(dataStreamVersion);
qint32 checksum;
s >> checksum
>> data;
m_loadDataStream >> checksum
>> m_loadFileBuffer.buffer();
buffer.setBuffer(&data);
buffer.open(QIODevice::ReadOnly);
s.setDevice(&buffer);
m_loadFileBuffer.open(QIODevice::ReadOnly);
m_loadDataStream.setDevice(&m_loadFileBuffer);
if(s.status() != CacheDataStream::Ok ||
checksum != qChecksum(data.data(), data.size()))
qint32 checksumExpected = qChecksum(
m_loadFileBuffer.data(), m_loadFileBuffer.size());
if(m_loadDataStream.status() != CacheDataStream::Ok ||
checksum != checksumExpected)
{
kError() << "Music cache checksum expected to get" << checksumExpected <<
"actually was" << checksum;
KMessageBox::sorry(0, i18n("The music data cache has been corrupted. JuK "
"needs to rescan it now. This may take some time."));
return;
return false;
}
break;
}
default: {
s.device()->reset();
s.setCacheVersion(0);
m_loadDataStream.device()->reset();
m_loadDataStream.setCacheVersion(0);
// This cache is so old that this is just a wild guess here that 3.3
// is compatible.
s.setVersion(CacheDataStream::Qt_3_3);
m_loadDataStream.setVersion(CacheDataStream::Qt_3_3);
break;
}
}
// Read the cached tags.
return true;
}
FileHandle Cache::loadNextCachedItem()
{
if(!m_loadFile.isOpen() || !m_loadDataStream.device()) {
kWarning() << "Already completed reading cache file.";
return FileHandle::null();
}
if(m_loadDataStream.status() == QDataStream::ReadCorruptData) {
kError() << "Attempted to read file handle from corrupt cache file.";
return FileHandle::null();
}
while(!s.atEnd()) {
if(!m_loadDataStream.atEnd()) {
QString fileName;
s >> fileName;
m_loadDataStream >> fileName;
fileName.squeeze();
FileHandle f(fileName, s);
return FileHandle(fileName, m_loadDataStream);
}
else {
m_loadDataStream.setDevice(0);
m_loadFile.close();
return FileHandle::null();
}
}
......
......@@ -16,7 +16,9 @@
#ifndef CACHE_H
#define CACHE_H
#include <QDataStream>
#include <QtCore/QDataStream>
#include <QtCore/QFile>
#include <QtCore/QBuffer>
#include "stringhash.h"
......@@ -28,27 +30,6 @@ class QList;
typedef QList<Playlist *> PlaylistList;
class Cache : public FileHandleHash
{
public:
static Cache *instance();
void save();
static void loadPlaylists(PlaylistCollection *collection);
static void savePlaylists(const PlaylistList &playlists);
static bool cacheFileExists();
protected:
Cache();
void load();
private:
// 1: Original cache version
// 2: KDE 4.0.1+, explicitly sets QDataStream encoding.
static const int m_currentVersion = 2;
};
/**
* A simple QDataStream subclass that has an extra field to indicate the cache
* version.
......@@ -58,8 +39,7 @@ class CacheDataStream : public QDataStream
{
public:
CacheDataStream(QIODevice *d) : QDataStream(d), m_cacheVersion(0) {}
virtual ~CacheDataStream() {}
CacheDataStream() : m_cacheVersion(0) { }
int cacheVersion() const { return m_cacheVersion; }
void setCacheVersion(int v) { m_cacheVersion = v; }
......@@ -68,6 +48,43 @@ private:
int m_cacheVersion;
};
class Cache : public FileHandleHash
{
public:
static Cache *instance();
static void loadPlaylists(PlaylistCollection *collection);
static void savePlaylists(const PlaylistList &playlists);
static bool cacheFileExists();
bool prepareToLoadCachedItems();
FileHandle loadNextCachedItem();
/**
* QDataStream version for serialized list of playlists
* 1, 2: Who knows?
* 3: Current.
*/
static const int playlistListCacheVersion;
/**
* QDataStream version for serialized list of playlist items in a playlist
* 1: Original cache version
* 2: KDE 4.0.1+, explicitly sets QDataStream encoding.
*/
static const int playlistItemsCacheVersion;
protected:
Cache();
private:
QFile m_loadFile;
QBuffer m_loadFileBuffer;
CacheDataStream m_loadDataStream;
};
#endif
// vim: set et sw=4 tw=0 sta:
......@@ -21,14 +21,20 @@
#include <kmenu.h>
#include <kconfig.h>
#include <kconfiggroup.h>
#include <kglobal.h>
#include <kactioncollection.h>
#include <ksavefile.h>
#include <kstandarddirs.h>
#include <ktoolbarpopupaction.h>
#include <kdirwatch.h>
#include <QStringBuilder>
#include <QList>
#include <QDragMoveEvent>
#include <QDropEvent>
#include <QApplication>
#include <QTimer>
#include <QTime>
#include <QClipboard>
#include <QFileInfo>
......@@ -53,22 +59,59 @@ CollectionList *CollectionList::instance()
return m_list;
}
void CollectionList::loadCachedItems()
static QTime stopwatch;
void CollectionList::startLoadingCachedItems()
{
if(!m_list)
return;
FileHandleHash::ConstIterator end = Cache::instance()->constEnd();
for(FileHandleHash::ConstIterator it = Cache::instance()->constBegin(); it != end; ++it) {
kDebug() << "Starting to load cached items";
stopwatch.start();
if(!Cache::instance()->prepareToLoadCachedItems()) {
kError() << "Unable to setup to load cache... perhaps it doesn't exist?";
completedLoadingCachedItems();
return;
}
kDebug() << "Kicked off first batch";
QTimer::singleShot(0, this, SLOT(loadNextBatchCachedItems()));
}
void CollectionList::loadNextBatchCachedItems()
{
Cache *cache = Cache::instance();
bool done = false;
for(int i = 0; i < 20; ++i) {
FileHandle cachedItem(cache->loadNextCachedItem());
if(cachedItem.isNull()) {
done = true;
break;
}
// This may have already been created via a loaded playlist.
if(!m_itemsDict.contains(it.key())) {
CollectionListItem *newItem = new CollectionListItem(this, *it);
if(!m_itemsDict.contains(cachedItem.absFilePath())) {
CollectionListItem *newItem = new CollectionListItem(this, cachedItem);
setupItem(newItem);
}
}
SplashScreen::update();
if(!done) {
QTimer::singleShot(0, this, SLOT(loadNextBatchCachedItems()));
}
else {
completedLoadingCachedItems();
}
}
void CollectionList::completedLoadingCachedItems()
{
// The CollectionList is created with sorting disabled for speed. Re-enable
// it here, and perform the sort.
KConfigGroup config(KGlobal::config(), "Playlists");
......@@ -83,6 +126,11 @@ void CollectionList::loadCachedItems()
m_list->sort();
SplashScreen::finishedLoading();
kDebug() << "Finished loading cached items, took" << stopwatch.elapsed() << "ms";
kDebug() << m_itemsDict.size() << "items are in the CollectionList";
emit cachedItemsLoaded();
}
void CollectionList::initialize(PlaylistCollection *collection)
......@@ -129,7 +177,6 @@ CollectionListItem *CollectionList::createItem(const FileHandle &file, Q3ListVie
void CollectionList::clearItems(const PlaylistItemList &items)
{
foreach(PlaylistItem *item, items) {
Cache::instance()->remove(item->file());
delete item;
}
......@@ -190,6 +237,44 @@ void CollectionList::slotDeleteItem(const KFileItem &item)
delete lookup(item.url().path());
}
void CollectionList::saveItemsToCache() const
{
kDebug() << "Saving collection list to cache";
QString cacheFileName =
KGlobal::dirs()->saveLocation("appdata") % QLatin1String("cache");
KSaveFile f(cacheFileName);
if(!f.open(QIODevice::WriteOnly)) {
kError() << "Error saving cache:" << f.errorString();
return;
}
QByteArray data;
QDataStream s(&data, QIODevice::WriteOnly);
s.setVersion(QDataStream::Qt_4_3);
QHash<QString, CollectionListItem *>::const_iterator it;
for(it = m_itemsDict.begin(); it != m_itemsDict.end(); ++it) {
s << it.key();
s << (*it)->file();
}
QDataStream fs(&f);
qint32 checksum = qChecksum(data.data(), data.size());
fs << qint32(Cache::playlistItemsCacheVersion)
<< checksum
<< data;
f.close();
if(!f.finalize())
kError() << "Error saving cache:" << f.errorString();
}
////////////////////////////////////////////////////////////////////////////////
// public slots
////////////////////////////////////////////////////////////////////////////////
......@@ -215,17 +300,21 @@ void CollectionList::clear()
void CollectionList::slotCheckCache()
{
loadCachedItems();
PlaylistItemList invalidItems;
kDebug() << "Starting to check cached items for consistency";
stopwatch.start();
int i = 0;
foreach(CollectionListItem *item, m_itemsDict) {
if(!item->checkCurrent())
invalidItems.append(item);
processEvents();
if(++i == (m_itemsDict.size() / 2))
kDebug() << "Checkpoint";
}
clearItems(invalidItems);
kDebug() << "Finished consistency check, took" << stopwatch.elapsed() << "ms";
}
void CollectionList::slotRemoveItem(const QString &file)
......
......@@ -123,6 +123,8 @@ public:
virtual bool canReload() const { return true; }
void saveItemsToCache() const;
public slots:
virtual void paste();
virtual void clear();
......@@ -172,12 +174,28 @@ signals:
void signalNewTag(const QString &, unsigned);
void signalRemovedTag(const QString &, unsigned);
// Emitted once cached items are loaded, which allows for folder scanning
// and invalid track detection to proceed.
void cachedItemsLoaded();
public slots:
/**
* Loads the CollectionListItems from the Cache. Should be called after program
* initialization.
*/
void loadCachedItems();
void startLoadingCachedItems();
/**
* Loads a few items at a time. Intended to be single-shotted into the event
* loop so that loading the music doesn't freeze the GUI.
*/
void loadNextBatchCachedItems();
/**
* Teardown from cache loading (e.g. splash screen, sorting, etc.). Should
* always be called if startLoadingCachedItems is called.
*/
void completedLoadingCachedItems();
private:
/**
......
......@@ -129,7 +129,6 @@ FileHandle::FileHandle(const QString &path, CacheDataStream &s)
d->fileInfo = QFileInfo(path);
d->absFilePath = path;
read(s);
Cache::instance()->insert(*this);
}
FileHandle::~FileHandle()
......@@ -279,22 +278,12 @@ void FileHandle::setup(const QFileInfo &info, const QString &path)
QString fileName = path.isEmpty() ? info.absoluteFilePath() : path;
FileHandle cached = Cache::instance()->value(resolveSymLinks(fileName));
if(cached != null()) {
d = cached.d;
d->ref();
}
else {
d = new FileHandlePrivate;
d->fileInfo = info;
d->absFilePath = resolveSymLinks(fileName);
d->modificationTime = info.lastModified();
if(info.exists())
Cache::instance()->insert(*this);
else
kWarning() << "File" << path << "no longer exists!";
}
d = new FileHandlePrivate;
d->fileInfo = info;
d->absFilePath = resolveSymLinks(fileName);
d->modificationTime = info.lastModified();
if(!info.exists())
kWarning() << "File" << path << "no longer exists!";
}
////////////////////////////////////////////////////////////////////////////////
......
......@@ -39,6 +39,7 @@
#include <QCoreApplication>
#include <QKeyEvent>
#include <QDir>
#include <QTime>
#include <QTimer>
#include <QDesktopWidget>
......@@ -132,7 +133,7 @@ JuK::JuK(QWidget *parent) :
// slotCheckCache loads the cached entries first to populate the collection list
QTimer::singleShot(0, this, SLOT(slotClearOldCovers()));
QTimer::singleShot(0, CollectionList::instance(), SLOT(slotCheckCache()));
QTimer::singleShot(0, CollectionList::instance(), SLOT(startLoadingCachedItems()));
QTimer::singleShot(0, this, SLOT(slotProcessArgs()));
}
......@@ -173,6 +174,10 @@ void JuK::setupLayout()
{
new TagTransactionManager(this);
kDebug() << "Creating GUI";
QTime stopwatch;
stopwatch.start();
m_splitter = new PlaylistSplitter(m_player, this);
setCentralWidget(m_splitter);
......@@ -183,6 +188,8 @@ void JuK::setupLayout()
m_player->setStatusLabel(m_statusLabel);
m_splitter->setFocus();
kDebug() << "GUI created in" << stopwatch.elapsed() << "ms";
}
void JuK::setupActions()
......@@ -338,11 +345,14 @@ void JuK::setupActions()
void JuK::slotSetupSystemTray()
{
if(m_toggleSystemTrayAction && m_toggleSystemTrayAction->isChecked()) {
kDebug() << "Setting up systray";
QTime stopwatch; stopwatch.start();
m_systemTray = new SystemTray(m_player, this);
m_systemTray->setObjectName( QLatin1String("systemTray" ));
m_toggleDockOnCloseAction->setEnabled(true);
m_togglePopupsAction->setEnabled(true);
kDebug() << "Finished setting up systray, took" << stopwatch.elapsed() << "ms";
}
else {
m_systemTray = 0;
......@@ -513,7 +523,6 @@ bool JuK::queryExit()
// Save configuration data.
m_startDocked = !isVisible();
saveConfig();
Cache::instance()->save();
return true;
}
......
......@@ -1862,7 +1862,6 @@ void Playlist::addFile(const QString &file, FileHandleList &files, bool importPl
if(hasItem(file) && !m_allowDuplicates)
return;
processEvents();
addFileHelper(files, after);
// Our biggest thing that we're fighting during startup is too many stats
......@@ -1945,8 +1944,6 @@ void Playlist::addFileHelper(FileHandleList &files, PlaylistItem **after, bool i
if(focus)
setFocus();
processEvents();
}
}
......
......@@ -37,6 +37,7 @@
#include <QDropEvent>
#include <QMouseEvent>
#include <QFileInfo>
#include <QTime>
#include <QApplication>
#include <QClipboard>
......@@ -151,10 +152,8 @@ PlaylistBox::PlaylistBox(PlayerManager *player, QWidget *parent, QStackedWidget
this, SLOT(slotAddItem(QString,uint)));
connect(CollectionList::instance(), SIGNAL(signalRemovedTag(QString,uint)),
this, SLOT(slotRemoveItem(QString,uint)));
QTimer::singleShot(0, this, SLOT(slotLoadCachedPlaylists()));
QTimer::singleShot(0, object(), SLOT(slotScanFolders()));
enableDirWatch(true);
connect(CollectionList::instance(), SIGNAL(cachedItemsLoaded()),
this, SLOT(slotLoadCachedPlaylists()));
m_savePlaylistTimer = 0;
......@@ -220,6 +219,18 @@ void PlaylistBox::duplicate()
p->createItems(item->playlist()->items());
}
void PlaylistBox::scanFolders()
{
kDebug() << "Starting folder scan";
QTime stopwatch; stopwatch.start();
PlaylistCollection::scanFolders();
kDebug() << "Folder scan complete, took" << stopwatch.elapsed() << "ms";
kDebug() << "Startup complete!";
emit startupComplete();
}
////////////////////////////////////////////////////////////////////////////////
// PlaylistBox public slots
////////////////////////////////////////////////////////////////////////////////
......@@ -733,18 +744,27 @@ void PlaylistBox::setupUpcomingPlaylist()
void PlaylistBox::slotLoadCachedPlaylists()
{
kDebug() << "Loading cached playlists.";
QTime stopwatch;
stopwatch.start();
Cache::loadPlaylists(this);
kDebug() << "Cached playlists loaded, took" << stopwatch.elapsed() << "ms";
// Auto-save playlists after they change.
m_savePlaylistTimer = new QTimer(this);
m_savePlaylistTimer->setInterval(3000); // 3 seconds with no change? -> commit
m_savePlaylistTimer->setSingleShot(true);
connect(m_savePlaylistTimer, SIGNAL(timeout()), SLOT(slotSavePlaylists()));
emit startupComplete();
clearSelection();
setSelected(m_playlistDict[CollectionList::instance()], true);
QTimer::singleShot(0, CollectionList::instance(), SLOT(slotCheckCache()));
QTimer::singleShot(0, object(), SLOT(slotScanFolders()));
}
////////////////////////////////////////////////////////////////////////////////
// PlaylistBox::Item protected methods
////////////////////////////////////////////////////////////////////////////////
......
......@@ -55,6 +55,10 @@ public:
virtual void duplicate();
virtual void remove();
// Called after files loaded to pickup any new files that might be present
// in managed directories.
virtual void scanFolders();
/**
* For view modes that have dynamic playlists, this freezes them from
* removing playlists.
......
......@@ -114,6 +114,7 @@ PlaylistCollection::PlaylistCollection(PlayerManager *player, QStackedWidget *pl
PlaylistCollection::~PlaylistCollection()
{
saveConfig();
CollectionList::instance()->saveItemsToCache();
delete m_actionHandler;
Playlist::setShuttingDown();
}
......@@ -512,6 +513,8 @@ void PlaylistCollection::scanFolders()
if(CollectionList::instance()->count() == 0)
addFolder();
enableDirWatch(true);
}
void PlaylistCollection::createPlaylist()
......
......@@ -107,7 +107,8 @@ public:
virtual PlaylistItemList selectedItems();
void scanFolders();
// virtual to allow our QWidget subclass to emit a signal after we're done
virtual void scanFolders();