try multi-threaded directory traversal without signals

addons/search/FolderFilesList.cpp

@@ -8,7 +8,12 @@
 
 #include <QDebug>
 #include <QDir>
+#include <QElapsedTimer>
 #include <QFileInfoList>
+#include <QtConcurrent>
+
+#include <unordered_set>
+#include <vector>
 
 FolderFilesList::FolderFilesList(QObject *parent)
     : QThread(parent)
@@ -27,8 +32,57 @@ void FolderFilesList::run()
 {
     m_files.clear();
 
-    QFileInfo folderInfo(m_folder);
-    checkNextItem(folderInfo);
+    /**
+     * iterative algorithm, in each round, we put in X directories to traverse
+     * we will get as output X times: new directories + found files
+     */
+    std::vector<DirectoryWithResults> directoriesWithResults{DirectoryWithResults{m_folder, QStringList(), QStringList()}};
+    std::unordered_set<QString> directoryGuard{m_folder};
+    QElapsedTimer time;
+    time.start();
+    while (!directoriesWithResults.empty()) {
+        /**
+         * all 100 ms => inform about progress
+         */
+        if (time.elapsed() > 100) {
+            time.restart();
+            Q_EMIT searching(directoriesWithResults[0].directory);
+        }
+
+        /**
+         * map the stuff blocking, we are in a background thread, easiest way
+         * this will just span ideal thread count many things while this thread more or less sleeps
+         * we call the checkNextItem member function, that one must be careful to not do evil things ;=)
+         */
+        QtConcurrent::blockingMap(directoriesWithResults, [this](DirectoryWithResults &item) {
+            checkNextItem(item);
+        });
+
+        /**
+         * collect the results to create new worklist for next round
+         */
+        std::vector<DirectoryWithResults> nextRound;
+        for (const auto &result : directoriesWithResults) {
+            /**
+             * one new item for the next round for each new directory
+             */
+            for (const auto &newDirectory : result.newDirectories) {
+                if (directoryGuard.insert(newDirectory).second) {
+                    nextRound.push_back(DirectoryWithResults{newDirectory, QStringList(), QStringList()});
+                }
+            }
+
+            /**
+             * just append found files
+             */
+            m_files << result.newFiles;
+        }
+
+        /**
+         * let's get next round going
+         */
+        directoriesWithResults = nextRound;
+    }
 
     if (m_cancelSearch) {
         m_files.clear();
@@ -61,14 +115,11 @@ void FolderFilesList::generateList(const QString &folder, bool recursive, bool h
     }
 
     QStringList tmpExcludes = excludes.split(QLatin1Char(','));
-    m_excludeList.clear();
+    m_excludes.clear();
     for (int i = 0; i < tmpExcludes.size(); i++) {
-        QRegExp rx(tmpExcludes[i].trimmed());
-        rx.setPatternSyntax(QRegExp::Wildcard);
-        m_excludeList << rx;
+        m_excludes << QRegularExpression(QRegularExpression::wildcardToRegularExpression(tmpExcludes[i].trimmed()));
     }
 
-    m_time.restart();
     start();
 }
 
@@ -88,55 +139,55 @@ void FolderFilesList::cancelSearch()
     m_cancelSearch = true;
 }
 
-void FolderFilesList::checkNextItem(const QFileInfo &item)
+void FolderFilesList::checkNextItem(DirectoryWithResults &handleOnFolder) const
 {
+    /**
+     * IMPORTANT: this member function is called by MULTIPLE THREADS
+     * => it is const, it shall only modify handleOnFolder
+     */
+
     if (m_cancelSearch) {
         return;
     }
-    if (m_time.elapsed() > 100) {
-        m_time.restart();
-        Q_EMIT searching(item.absoluteFilePath());
+
+    QDir currentDir(handleOnFolder.directory);
+    if (!currentDir.isReadable()) {
+        // qDebug() << currentDir.absolutePath() << "Not readable";
+        return;
+    }
+
+    QDir::Filters filter = QDir::Files | QDir::NoDotAndDotDot | QDir::Readable;
+    if (m_hidden) {
+        filter |= QDir::Hidden;
+    }
+    if (m_recursive) {
+        filter |= QDir::AllDirs;
+    }
+    if (!m_symlinks) {
+        filter |= QDir::NoSymLinks;
     }
-    if (item.isFile()) {
-        m_files << item.canonicalFilePath();
-    } else {
-        QDir currentDir(item.absoluteFilePath());
-
-        if (!currentDir.isReadable()) {
-            // qDebug() << currentDir.absolutePath() << "Not readable";
-            return;
-        }
 
-        QDir::Filters filter = QDir::Files | QDir::NoDotAndDotDot | QDir::Readable;
-        if (m_hidden) {
-            filter |= QDir::Hidden;
-        }
-        if (m_recursive) {
-            filter |= QDir::AllDirs;
+    const QFileInfoList entries = currentDir.entryInfoList(m_types, filter, QDir::Name | QDir::LocaleAware);
+    for (const auto &entry : entries) {
+        const QString absFilePath = entry.absoluteFilePath();
+        bool skip{false};
+        for (const auto &regex : m_excludes) {
+            QRegularExpressionMatch match = regex.match(absFilePath);
+            if (match.hasMatch()) {
+                skip = true;
+                break;
+            }
         }
-        if (!m_symlinks) {
-            filter |= QDir::NoSymLinks;
+        if (skip) {
+            continue;
         }
 
-        // sort the items to have an deterministic order!
-        const QFileInfoList currentItems = currentDir.entryInfoList(m_types, filter, QDir::Name | QDir::LocaleAware);
+        if (entry.isDir()) {
+            handleOnFolder.newDirectories.append(absFilePath);
+        }
 
-        bool skip;
-        for (const auto &currentItem : currentItems) {
-            skip = false;
-            for (const auto &regex : qAsConst(m_excludeList)) {
-                QString matchString = currentItem.filePath();
-                if (currentItem.filePath().startsWith(m_folder)) {
-                    matchString = currentItem.filePath().mid(m_folder.size());
-                }
-                if (regex.exactMatch(matchString)) {
-                    skip = true;
-                    break;
-                }
-            }
-            if (!skip) {
-                checkNextItem(currentItem);
-            }
+        if (entry.isFile()) {
+            handleOnFolder.newFiles.append(absFilePath);
         }
     }
 }

addons/search/FolderFilesList.h

@@ -7,9 +7,7 @@
 #ifndef FolderFilesList_h
 #define FolderFilesList_h
 
-#include <QElapsedTimer>
-#include <QFileInfo>
-#include <QRegExp>
+#include <QRegularExpression>
 #include <QStringList>
 #include <QThread>
 #include <QVector>
@@ -38,7 +36,13 @@ Q_SIGNALS:
     void fileListReady();
 
 private:
-    void checkNextItem(const QFileInfo &item);
+    struct DirectoryWithResults {
+        QString directory;
+        QStringList newDirectories;
+        QStringList newFiles;
+    };
+
+    void checkNextItem(DirectoryWithResults &handleOnFolder) const;
 
 private:
     QString m_folder;
@@ -49,8 +53,7 @@ private:
     bool m_hidden = false;
     bool m_symlinks = false;
     QStringList m_types;
-    QVector<QRegExp> m_excludeList;
-    QElapsedTimer m_time;
+    QVector<QRegularExpression> m_excludes;
 };
 
 #endif