Commit 2b6bc295 authored by Martin Tobias Holmedahl Sandsmark's avatar Martin Tobias Holmedahl Sandsmark
Browse files

Implement multi-threading for disk scanning

For each subfolder we see if there's a thread free, if not we scan
immediately in the same thread (as before), otherwise we run it in a new
thread and wait for it.
parent 2b70311b
Pipeline #90890 passed with stage
in 35 seconds
......@@ -16,6 +16,8 @@
#include <QElapsedTimer>
#include <QGuiApplication> //postEvent()
#include <QFile>
#include <QThreadPool>
#include <QSemaphore>
#include <dirent.h>
#include <sys/stat.h>
......@@ -140,6 +142,8 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
#define KDE_lstat kdewin32_stat64
#endif
QVector<QPair<QByteArray, QByteArray>> subDirectories;
struct stat statbuf;
dirent *ent;
while ((ent = readdir(dir)))
......@@ -150,8 +154,9 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
return cwd;
}
if (qstrcmp(ent->d_name, ".") == 0 || qstrcmp(ent->d_name, "..") == 0)
if (qstrcmp(ent->d_name, ".") == 0 || qstrcmp(ent->d_name, "..") == 0) {
continue;
}
// QStringBuilder is used here. It assumes ent->d_name is char[NAME_MAX + 1],
// and thus copies only first NAME_MAX + 1 chars.
......@@ -191,6 +196,7 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
//check to see if we've scanned this section already
QMutexLocker lock(&m_treeMutex);
for (Folder *folder : *m_trees)
{
if (new_path == folder->name8Bit())
......@@ -202,10 +208,11 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
cwd->append(folder, new_dirname.constData());
}
}
lock.unlock();
if (!d) //then scan
if ((d = scan(new_path, new_dirname))) //then scan was successful
cwd->append(d);
if (!d) { //then scan
subDirectories.append({new_path, new_dirname});
}
}
++m_parent->m_files;
......@@ -213,6 +220,27 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
closedir(dir);
// Scan all subdirectories, either in separate threads or immediately,
// depending on how many free threads there are in the threadpool.
// Yes, it isn't optimal, but it's better than nothing and pretty simple.
QVector<Folder*> returnedCwds(subDirectories.count());
QSemaphore semaphore;
for (int i=0; i<subDirectories.count(); i++) {
std::function<void()> scanSubdir = [this, i, &subDirectories, &semaphore, &returnedCwds]() {
returnedCwds[i] = scan(subDirectories[i].first, subDirectories[i].second);
semaphore.release(1);
};
if (!QThreadPool::globalInstance()->tryStart(scanSubdir)) {
scanSubdir();
}
}
semaphore.acquire(subDirectories.count());
for (Folder *d : returnedCwds) {
if (d) { //then scan was successful
cwd->append(d);
}
}
std::sort(cwd->files.begin(), cwd->files.end(), [](File *a, File*b) { return a->size() > b->size(); });
return cwd;
......
......@@ -10,6 +10,7 @@
#include <QByteArray>
#include <QThread>
#include <QMutex>
class Folder;
......@@ -31,6 +32,7 @@ Q_SIGNALS:
private:
QString m_path;
QMutex m_treeMutex;
QList<Folder*> *m_trees;
ScanManager *m_parent;
......
......@@ -12,6 +12,8 @@
#include <QMutex>
#include <QList>
#include <atomic>
class Folder;
namespace Filelight
......@@ -54,7 +56,7 @@ Q_SIGNALS:
void branchCacheHit(Folder* tree);
private:
bool m_abort;
std::atomic_bool m_abort;
QAtomicInt m_files;
QAtomicInteger<size_t> m_totalSize;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment