Commit 238c83a3 authored by Christoph Cullmann's avatar Christoph Cullmann 🐮
Browse files

try multi-threaded directory traversal without signals

parent ee9762a4
......@@ -8,7 +8,12 @@
#include <QDebug>
#include <QDir>
#include <QElapsedTimer>
#include <QFileInfoList>
#include <QtConcurrent>
#include <unordered_set>
#include <vector>
FolderFilesList::FolderFilesList(QObject *parent)
: QThread(parent)
......@@ -27,8 +32,57 @@ void FolderFilesList::run()
{
m_files.clear();
QFileInfo folderInfo(m_folder);
checkNextItem(folderInfo);
/**
* iterative algorithm, in each round, we put in X directories to traverse
* we will get as output X times: new directories + found files
*/
std::vector<DirectoryWithResults> directoriesWithResults{DirectoryWithResults{m_folder, QStringList(), QStringList()}};
std::unordered_set<QString> directoryGuard{m_folder};
QElapsedTimer time;
time.start();
while (!directoriesWithResults.empty()) {
/**
* all 100 ms => inform about progress
*/
if (time.elapsed() > 100) {
time.restart();
Q_EMIT searching(directoriesWithResults[0].directory);
}
/**
* map the stuff blocking, we are in a background thread, easiest way
* this will just span ideal thread count many things while this thread more or less sleeps
* we call the checkNextItem member function, that one must be careful to not do evil things ;=)
*/
QtConcurrent::blockingMap(directoriesWithResults, [this](DirectoryWithResults &item) {
checkNextItem(item);
});
/**
* collect the results to create new worklist for next round
*/
std::vector<DirectoryWithResults> nextRound;
for (const auto &result : directoriesWithResults) {
/**
* one new item for the next round for each new directory
*/
for (const auto &newDirectory : result.newDirectories) {
if (directoryGuard.insert(newDirectory).second) {
nextRound.push_back(DirectoryWithResults{newDirectory, QStringList(), QStringList()});
}
}
/**
* just append found files
*/
m_files << result.newFiles;
}
/**
* let's get next round going
*/
directoriesWithResults = nextRound;
}
if (m_cancelSearch) {
m_files.clear();
......@@ -61,14 +115,11 @@ void FolderFilesList::generateList(const QString &folder, bool recursive, bool h
}
QStringList tmpExcludes = excludes.split(QLatin1Char(','));
m_excludeList.clear();
m_excludes.clear();
for (int i = 0; i < tmpExcludes.size(); i++) {
QRegExp rx(tmpExcludes[i].trimmed());
rx.setPatternSyntax(QRegExp::Wildcard);
m_excludeList << rx;
m_excludes << QRegularExpression(QRegularExpression::wildcardToRegularExpression(tmpExcludes[i].trimmed()));
}
m_time.restart();
start();
}
......@@ -88,55 +139,55 @@ void FolderFilesList::cancelSearch()
m_cancelSearch = true;
}
void FolderFilesList::checkNextItem(const QFileInfo &item)
void FolderFilesList::checkNextItem(DirectoryWithResults &handleOnFolder) const
{
/**
* IMPORTANT: this member function is called by MULTIPLE THREADS
* => it is const, it shall only modify handleOnFolder
*/
if (m_cancelSearch) {
return;
}
if (m_time.elapsed() > 100) {
m_time.restart();
Q_EMIT searching(item.absoluteFilePath());
QDir currentDir(handleOnFolder.directory);
if (!currentDir.isReadable()) {
// qDebug() << currentDir.absolutePath() << "Not readable";
return;
}
QDir::Filters filter = QDir::Files | QDir::NoDotAndDotDot | QDir::Readable;
if (m_hidden) {
filter |= QDir::Hidden;
}
if (m_recursive) {
filter |= QDir::AllDirs;
}
if (!m_symlinks) {
filter |= QDir::NoSymLinks;
}
if (item.isFile()) {
m_files << item.canonicalFilePath();
} else {
QDir currentDir(item.absoluteFilePath());
if (!currentDir.isReadable()) {
// qDebug() << currentDir.absolutePath() << "Not readable";
return;
}
QDir::Filters filter = QDir::Files | QDir::NoDotAndDotDot | QDir::Readable;
if (m_hidden) {
filter |= QDir::Hidden;
}
if (m_recursive) {
filter |= QDir::AllDirs;
const QFileInfoList entries = currentDir.entryInfoList(m_types, filter, QDir::Name | QDir::LocaleAware);
for (const auto &entry : entries) {
const QString absFilePath = entry.absoluteFilePath();
bool skip{false};
for (const auto &regex : m_excludes) {
QRegularExpressionMatch match = regex.match(absFilePath);
if (match.hasMatch()) {
skip = true;
break;
}
}
if (!m_symlinks) {
filter |= QDir::NoSymLinks;
if (skip) {
continue;
}
// sort the items to have an deterministic order!
const QFileInfoList currentItems = currentDir.entryInfoList(m_types, filter, QDir::Name | QDir::LocaleAware);
if (entry.isDir()) {
handleOnFolder.newDirectories.append(absFilePath);
}
bool skip;
for (const auto &currentItem : currentItems) {
skip = false;
for (const auto &regex : qAsConst(m_excludeList)) {
QString matchString = currentItem.filePath();
if (currentItem.filePath().startsWith(m_folder)) {
matchString = currentItem.filePath().mid(m_folder.size());
}
if (regex.exactMatch(matchString)) {
skip = true;
break;
}
}
if (!skip) {
checkNextItem(currentItem);
}
if (entry.isFile()) {
handleOnFolder.newFiles.append(absFilePath);
}
}
}
......@@ -7,9 +7,7 @@
#ifndef FolderFilesList_h
#define FolderFilesList_h
#include <QElapsedTimer>
#include <QFileInfo>
#include <QRegExp>
#include <QRegularExpression>
#include <QStringList>
#include <QThread>
#include <QVector>
......@@ -38,7 +36,13 @@ Q_SIGNALS:
void fileListReady();
private:
void checkNextItem(const QFileInfo &item);
struct DirectoryWithResults {
QString directory;
QStringList newDirectories;
QStringList newFiles;
};
void checkNextItem(DirectoryWithResults &handleOnFolder) const;
private:
QString m_folder;
......@@ -49,8 +53,7 @@ private:
bool m_hidden = false;
bool m_symlinks = false;
QStringList m_types;
QVector<QRegExp> m_excludeList;
QElapsedTimer m_time;
QVector<QRegularExpression> m_excludes;
};
#endif
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment