From 2b6bc2957f39cfbc55ebce442b303909882e0ee2 Mon Sep 17 00:00:00 2001
From: "Martin T. H. Sandsmark" <martin.sandsmark@kde.org>
Date: Sun, 24 Oct 2021 11:26:33 +0200
Subject: [PATCH 1/2] Implement multi-threading for disk scanning

For each subfolder we see if there's a thread free, if not we scan
immediately in the same thread (as before), otherwise we run it in a new
thread and wait for it.
---
 src/localLister.cpp | 36 ++++++++++++++++++++++++++++++++----
 src/localLister.h   |  2 ++
 src/scan.h          |  4 +++-
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/localLister.cpp b/src/localLister.cpp
index fcc5b89..efe0fae 100644
--- a/src/localLister.cpp
+++ b/src/localLister.cpp
@@ -16,6 +16,8 @@
 #include <QElapsedTimer>
 #include <QGuiApplication> //postEvent()
 #include <QFile>
+#include <QThreadPool>
+#include <QSemaphore>
 
 #include <dirent.h>
 #include <sys/stat.h>
@@ -140,6 +142,8 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
 #define KDE_lstat kdewin32_stat64
 #endif
 
+    QVector<QPair<QByteArray, QByteArray>> subDirectories;
+
     struct stat statbuf;
     dirent *ent;
     while ((ent = readdir(dir)))
@@ -150,8 +154,9 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
             return cwd;
         }
 
-        if (qstrcmp(ent->d_name, ".") == 0 || qstrcmp(ent->d_name, "..") == 0)
+        if (qstrcmp(ent->d_name, ".") == 0 || qstrcmp(ent->d_name, "..") == 0) {
             continue;
+        }
 
         // QStringBuilder is used here. It assumes ent->d_name is char[NAME_MAX + 1],
         // and thus copies only first NAME_MAX + 1 chars.
@@ -191,6 +196,7 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
 
             //check to see if we've scanned this section already
 
+            QMutexLocker lock(&m_treeMutex);
             for (Folder *folder : *m_trees)
             {
                 if (new_path == folder->name8Bit())
@@ -202,10 +208,11 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
                     cwd->append(folder, new_dirname.constData());
                 }
             }
+            lock.unlock();
 
-            if (!d) //then scan
-                if ((d = scan(new_path, new_dirname))) //then scan was successful
-                    cwd->append(d);
+            if (!d) { //then scan
+                subDirectories.append({new_path, new_dirname});
+            }
         }
 
         ++m_parent->m_files;
@@ -213,6 +220,27 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
 
     closedir(dir);
 
+    // Scan all subdirectories, either in separate threads or immediately,
+    // depending on how many free threads there are in the threadpool.
+    // Yes, it isn't optimal, but it's better than nothing and pretty simple.
+    QVector<Folder*> returnedCwds(subDirectories.count());
+    QSemaphore semaphore;
+    for (int i=0; i<subDirectories.count(); i++) {
+        std::function<void()> scanSubdir = [this, i, &subDirectories, &semaphore, &returnedCwds]() {
+            returnedCwds[i] = scan(subDirectories[i].first, subDirectories[i].second);
+            semaphore.release(1);
+        };
+        if (!QThreadPool::globalInstance()->tryStart(scanSubdir)) {
+            scanSubdir();
+        }
+    }
+    semaphore.acquire(subDirectories.count());
+    for (Folder *d : returnedCwds) {
+        if (d) { //then scan was successful
+            cwd->append(d);
+        }
+    }
+
     std::sort(cwd->files.begin(), cwd->files.end(), [](File *a, File*b) { return a->size() > b->size(); });
 
     return cwd;
diff --git a/src/localLister.h b/src/localLister.h
index 3db976b..f97906b 100644
--- a/src/localLister.h
+++ b/src/localLister.h
@@ -10,6 +10,7 @@
 
 #include <QByteArray>
 #include <QThread>
+#include <QMutex>
 
 class Folder;
 
@@ -31,6 +32,7 @@ Q_SIGNALS:
 
 private:
     QString m_path;
+    QMutex m_treeMutex;
     QList<Folder*> *m_trees;
     ScanManager *m_parent;
 
diff --git a/src/scan.h b/src/scan.h
index a4fd892..899b77d 100644
--- a/src/scan.h
+++ b/src/scan.h
@@ -12,6 +12,8 @@
 #include <QMutex>
 #include <QList>
 
+#include <atomic>
+
 class Folder;
 
 namespace Filelight
@@ -54,7 +56,7 @@ Q_SIGNALS:
     void branchCacheHit(Folder* tree);
 
 private:
-    bool m_abort;
+    std::atomic_bool m_abort;
     QAtomicInt m_files;
     QAtomicInteger<size_t> m_totalSize;
 
-- 
GitLab


From c8b1fa5fe3dbed696839c25e3a74098f422fb8cc Mon Sep 17 00:00:00 2001
From: "Martin T. H. Sandsmark" <martin.sandsmark@kde.org>
Date: Mon, 25 Oct 2021 12:21:56 +0200
Subject: [PATCH 2/2] make Harald happy

---
 src/localLister.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/localLister.cpp b/src/localLister.cpp
index efe0fae..de41ad3 100644
--- a/src/localLister.cpp
+++ b/src/localLister.cpp
@@ -235,7 +235,7 @@ LocalLister::scan(const QByteArray &path, const QByteArray &dirname)
         }
     }
     semaphore.acquire(subDirectories.count());
-    for (Folder *d : returnedCwds) {
+    for (Folder *d : std::as_const(returnedCwds)) {
         if (d) { //then scan was successful
             cwd->append(d);
         }
-- 
GitLab