Commit a6a457db authored by Kåre Särs's avatar Kåre Särs Committed by Christoph Cullmann
Browse files

S&R: Make the disk search multi-threaded

Move The QThread to QRunnable and QThreadPool.
The move was at the end not the most beneficial.

Note the removal of msleep() ;)
parent b198ad11
......@@ -43,9 +43,8 @@ void FolderFilesList::run()
if (m_cancelSearch) {
m_files.clear();
} else {
Q_EMIT fileListReady();
}
Q_EMIT fileListReady();
}
void FolderFilesList::generateList(const QString &folder, bool recursive, bool hidden, bool symlinks, const QString &types, const QString &excludes)
......@@ -107,7 +106,7 @@ void FolderFilesList::checkNextItem(const QFileInfo &item)
}
if (m_time.elapsed() > 100) {
m_time.restart();
emit searching(item.absoluteFilePath());
Q_EMIT searching(item.absoluteFilePath());
}
if (item.isFile()) {
m_files << item.canonicalFilePath();
......
......@@ -37,26 +37,26 @@ bool KateSearchCommand::exec(KTextEditor::View * /*view*/, const QString &cmd, Q
QString searchText = args.join(QLatin1Char(' '));
if (command == QLatin1String("grep") || command == QLatin1String("newGrep")) {
emit setSearchPlace(MatchModel::Folder);
emit setCurrentFolder();
Q_EMIT setSearchPlace(MatchModel::Folder);
Q_EMIT setCurrentFolder();
if (command == QLatin1String("newGrep"))
emit newTab();
Q_EMIT newTab();
}
else if (command == QLatin1String("search") || command == QLatin1String("newSearch")) {
emit setSearchPlace(MatchModel::OpenFiles);
Q_EMIT setSearchPlace(MatchModel::OpenFiles);
if (command == QLatin1String("newSearch"))
emit newTab();
Q_EMIT newTab();
}
else if (command == QLatin1String("pgrep") || command == QLatin1String("newPGrep")) {
emit setSearchPlace(MatchModel::Project);
Q_EMIT setSearchPlace(MatchModel::Project);
if (command == QLatin1String("newPGrep"))
emit newTab();
Q_EMIT newTab();
}
emit setSearchString(searchText);
emit startSearch();
Q_EMIT setSearchString(searchText);
Q_EMIT startSearch();
return true;
}
......
......@@ -351,7 +351,7 @@ void MatchModel::doReplaceNextMatch()
if (m_cancelReplace || m_replaceFile >= m_matchFiles.size()) {
m_replaceFile = -1;
emit replaceDone();
Q_EMIT replaceDone();
return;
}
......
......@@ -22,34 +22,25 @@
#include <QTextStream>
#include <QUrl>
SearchDiskFiles::SearchDiskFiles(QObject *parent)
: QThread(parent)
SearchDiskFiles::SearchDiskFiles(const QStringList &files, const QRegularExpression &regexp, const bool includeBinaryFiles)
: QObject(nullptr)
, m_files(files)
, m_regExp(regexp)
, m_includeBinaryFiles(includeBinaryFiles)
{
m_includeBinaryFiles = includeBinaryFiles;
m_cancelSearch = false;
m_files = files;
m_regExp = regexp;
// ensure we have a proper thread name during e.g. perf profiling
setObjectName(QStringLiteral("SearchDiskFiles"));
}
SearchDiskFiles::~SearchDiskFiles()
{
m_cancelSearch = true;
wait();
}
void SearchDiskFiles::startSearch(const QStringList &files, const QRegularExpression &regexp, const bool includeBinaryFiles)
{
if (files.empty()) {
emit searchDone();
return;
}
m_includeBinaryFiles = includeBinaryFiles;
m_cancelSearch = false;
m_terminateSearch = false;
m_files = files;
m_regExp = regexp;
m_matchCount = 0;
m_statusTime.restart();
start();
}
void SearchDiskFiles::run()
{
......@@ -59,11 +50,6 @@ void SearchDiskFiles::run()
break;
}
if (m_statusTime.elapsed() > 100) {
m_statusTime.restart();
emit searching(fileName);
}
// open file early, this allows mime-type detection & search to use same io device
QFile file(fileName);
if (!file.open(QFile::ReadOnly)) {
......@@ -84,11 +70,6 @@ void SearchDiskFiles::run()
searchSingleLineRegExp(file);
}
}
if (!m_terminateSearch) {
emit searchDone();
}
m_cancelSearch = true;
}
void SearchDiskFiles::cancelSearch()
......@@ -96,18 +77,6 @@ void SearchDiskFiles::cancelSearch()
m_cancelSearch = true;
}
void SearchDiskFiles::terminateSearch()
{
m_cancelSearch = true;
m_terminateSearch = true;
wait();
}
bool SearchDiskFiles::searching()
{
return !m_cancelSearch;
}
void SearchDiskFiles::searchSingleLineRegExp(QFile &file)
{
QTextStream stream(&file);
......@@ -134,18 +103,13 @@ void SearchDiskFiles::searchSingleLineRegExp(QFile &file)
match = m_regExp.match(line, column + match.capturedLength());
column = match.capturedStart();
m_matchCount++;
// NOTE: This sleep is here so that the main thread will get a chance to
// handle any stop button clicks if there are a lot of matches
if (m_matchCount % 50)
msleep(1);
}
i++;
}
// emit all matches batched
// Q_EMIT all matches batched
const QUrl fileUrl = QUrl::fromUserInput(file.fileName());
emit matchesFound(fileUrl, matches);
Q_EMIT matchesFound(fileUrl, matches);
}
void SearchDiskFiles::searchMultiLineRegExp(QFile &file)
......@@ -206,14 +170,9 @@ void SearchDiskFiles::searchMultiLineRegExp(QFile &file)
match = tmpRegExp.match(fullDoc, column + match.capturedLength());
column = match.capturedStart();
m_matchCount++;
// NOTE: This sleep is here so that the main thread will get a chance to
// handle any stop button clicks if there are a lot of matches
if (m_matchCount % 50)
msleep(1);
}
// emit all matches batched
// Q_EMIT all matches batched
const QUrl fileUrl = QUrl::fromUserInput(file.fileName());
emit matchesFound(fileUrl, matches);
Q_EMIT matchesFound(fileUrl, matches);
}
......@@ -18,50 +18,43 @@
#ifndef SearchDiskFiles_h
#define SearchDiskFiles_h
#include <QElapsedTimer>
#include <QFileInfo>
#include <QMutex>
#include <QRegularExpression>
#include <QObject>
#include <QStringList>
#include <QThread>
#include <QVector>
#include <QRegularExpression>
#include <QRunnable>
#include "MatchModel.h"
class SearchDiskFiles : public QThread
class QString;
class QUrl;
class QFile;
class SearchDiskFiles : public QObject, public QRunnable
{
Q_OBJECT
public:
SearchDiskFiles(QObject *parent = nullptr);
SearchDiskFiles(const QStringList &iles, const QRegularExpression &regexp, const bool includeBinaryFiles);
~SearchDiskFiles() override;
void startSearch(const QStringList &iles, const QRegularExpression &regexp, const bool includeBinaryFiles);
void run() override;
void terminateSearch();
bool searching();
private:
void searchSingleLineRegExp(QFile &file);
void searchMultiLineRegExp(QFile &file);
public Q_SLOTS:
void cancelSearch();
Q_SIGNALS:
void matchesFound(const QUrl &url, const QVector<KateSearchMatch> &searchMatches);
void searchDone();
void searching(const QString &file);
private:
QRegularExpression m_regExp;
void searchSingleLineRegExp(QFile &file);
void searchMultiLineRegExp(QFile &file);
private:
QStringList m_files;
bool m_cancelSearch = true;
bool m_terminateSearch = false;
int m_matchCount = 0;
QElapsedTimer m_statusTime;
QRegularExpression m_regExp;
bool m_includeBinaryFiles = false;
bool m_cancelSearch = true;
};
#endif
......@@ -195,7 +195,7 @@ Results::Results(QWidget *parent)
matchModel.setMatchColors(fg.name(QColor::HexArgb), search.name(QColor::HexArgb), replace.name(QColor::HexArgb));
treeView->setPalette(pal);
emit colorsChanged();
Q_EMIT colorsChanged();
};
auto e = KTextEditor::Editor::instance();
......@@ -235,7 +235,7 @@ bool ContainerWidget::focusNextPrevChild(bool next)
{
QWidget *fw = focusWidget();
bool found = false;
emit nextFocus(fw, &found, next);
Q_EMIT nextFocus(fw, &found, next);
if (found) {
return true;
......@@ -443,6 +443,10 @@ KatePluginSearchView::KatePluginSearchView(KTextEditor::Plugin *plugin, KTextEdi
connect(&m_searchOpenFiles, &SearchOpenFiles::matchesFound, this, &KatePluginSearchView::matchesFound);
connect(&m_searchOpenFiles, &SearchOpenFiles::searchDone, this, &KatePluginSearchView::searchDone);
m_diskSearchDoneTimer.setSingleShot(true);
m_diskSearchDoneTimer.setInterval(10);
connect(&m_diskSearchDoneTimer, &QTimer::timeout, this, &KatePluginSearchView::searchDone);
connect(&m_folderFilesList, &FolderFilesList::fileListReady, this, &KatePluginSearchView::folderFileListChanged);
connect(&m_folderFilesList, &FolderFilesList::searching, this, [this](const QString &path) {
Results *res = qobject_cast<Results *>(m_ui.resultTabWidget->currentWidget());
......@@ -451,9 +455,6 @@ KatePluginSearchView::KatePluginSearchView(KTextEditor::Plugin *plugin, KTextEdi
}
});
connect(&m_searchDiskFiles, &SearchDiskFiles::matchesFound, this, &KatePluginSearchView::matchesFound);
connect(&m_searchDiskFiles, &SearchDiskFiles::searchDone, this, &KatePluginSearchView::searchDone);
connect(m_kateApp, &KTextEditor::Application::documentWillBeDeleted, this, &KatePluginSearchView::clearDocMarksAndRanges);
connect(m_kateApp, &KTextEditor::Application::documentWillBeDeleted, &m_searchOpenFiles, &SearchOpenFiles::cancelSearch);
connect(m_kateApp, &KTextEditor::Application::documentWillBeDeleted, this, [this]() {
......@@ -712,18 +713,18 @@ QStringList KatePluginSearchView::filterFiles(const QStringList &files) const
void KatePluginSearchView::folderFileListChanged()
{
m_searchDiskFilesDone = false;
m_searchOpenFilesDone = false;
if (!m_curResults) {
qWarning() << "This is a bug";
m_searchDiskFilesDone = true;
m_searchOpenFilesDone = true;
searchDone();
return;
}
QStringList fileList = m_folderFilesList.fileList();
if (fileList.isEmpty()) {
searchDone();
return;
}
QList<KTextEditor::Document *> openList;
for (int i = 0; i < m_kateApp->documents().size(); i++) {
int index = fileList.indexOf(m_kateApp->documents()[i]->url().toLocalFile());
......@@ -738,13 +739,54 @@ void KatePluginSearchView::folderFileListChanged()
// The DiskFile might finish immediately
if (!openList.empty()) {
m_searchOpenFiles.startSearch(openList, m_curResults->regExp);
} else {
m_searchOpenFilesDone = true;
}
m_searchDiskFiles.startSearch(fileList, m_curResults->regExp, m_ui.binaryCheckBox->isChecked());
startDiskFileSearch(fileList, m_curResults->regExp, m_ui.binaryCheckBox->isChecked());
}
void KatePluginSearchView::startDiskFileSearch(const QStringList &fileList, const QRegularExpression &reg, bool includeBinaryFiles)
{
if (fileList.isEmpty()) {
searchDone();
return;
}
// Note: Experimented with different thread counts. 1 to QThread::idealThreadCount()
// The optimum was two threads.... My theory is that the disk is the main bottleneck.
int chunkSize = (fileList.size() / 2) + 1;
chunkSize = qMax(chunkSize, 1);
int nextChunk = 0;
while (nextChunk < fileList.size()) {
QStringList chunckList = fileList.mid(nextChunk, chunkSize);
SearchDiskFiles *runner = new SearchDiskFiles(chunckList, reg, includeBinaryFiles);
connect(runner, &SearchDiskFiles::matchesFound, this, &KatePluginSearchView::matchesFound);
connect(this, &KatePluginSearchView::cancelSearch, runner, &SearchDiskFiles::cancelSearch);
connect(runner, &SearchDiskFiles::destroyed, this, [this]() {
if (m_searchDiskFilePool.activeThreadCount() == 0) {
if (!m_diskSearchDoneTimer.isActive()) {
m_diskSearchDoneTimer.start();
}
}
});
m_searchDiskFilePool.start(runner);
nextChunk += chunkSize;
}
}
void KatePluginSearchView::cancelDiskFileSearch()
{
Q_EMIT cancelSearch();
m_searchDiskFilePool.clear();
m_searchDiskFilePool.waitForDone();
}
bool KatePluginSearchView::searchingDiskFiles()
{
return m_searchDiskFilePool.activeThreadCount() > 0;
}
void KatePluginSearchView::searchPlaceChanged()
{
int searchPlace = m_ui.searchPlaceCombo->currentIndex();
......@@ -779,7 +821,7 @@ void KatePluginSearchView::searchPlaceChanged()
void KatePluginSearchView::matchesFound(const QUrl &url, const QVector<KateSearchMatch> &searchMatches)
{
if (!m_curResults || (sender() == &m_searchDiskFiles && m_blockDiskMatchFound)) {
if (!m_curResults) {
return;
}
......@@ -791,14 +833,11 @@ void KatePluginSearchView::stopClicked()
{
m_folderFilesList.cancelSearch();
m_searchOpenFiles.cancelSearch();
m_searchDiskFiles.cancelSearch();
cancelDiskFileSearch();
Results *res = qobject_cast<Results *>(m_ui.resultTabWidget->currentWidget());
if (res) {
res->matchModel.cancelReplace();
}
m_searchDiskFilesDone = true;
m_searchOpenFilesDone = true;
searchDone(); // Just in case the folder list was being populated...
}
/**
......@@ -847,19 +886,15 @@ void KatePluginSearchView::updateViewColors()
}
}
//static QElapsedTimer s_timer;
void KatePluginSearchView::startSearch()
{
//s_timer.start();
// Forcefully stop any ongoing search or replace
m_blockDiskMatchFound = true; // Do not allow leftover machFound:s from a previous search to be added
m_folderFilesList.terminateSearch();
m_searchOpenFiles.terminateSearch();
m_searchDiskFiles.terminateSearch();
// Re-enable the handling of disk-file-matches after one event loop
// For some reason blocking of signals or disconnect/connect does not prevent the slot from being called,
// so we use m_blockDiskMatchFound to skip any old matchFound signals during the first event loop.
// New matches from disk-files should not come before the first event loop has executed.
QTimer::singleShot(0, this, [this]() { m_blockDiskMatchFound = false; });
// FIXME check if this above is needed
cancelDiskFileSearch();
Results *res = qobject_cast<Results *>(m_ui.resultTabWidget->currentWidget());
if (res) {
......@@ -942,8 +977,6 @@ void KatePluginSearchView::startSearch()
m_ui.resultTabWidget->setTabText(m_ui.resultTabWidget->currentIndex(), m_ui.searchCombo->currentText());
m_toolView->setCursor(Qt::WaitCursor);
m_searchDiskFilesDone = false;
m_searchOpenFilesDone = false;
const bool inCurrentProject = m_ui.searchPlaceCombo->currentIndex() == MatchModel::Project;
const bool inAllOpenProjects = m_ui.searchPlaceCombo->currentIndex() == MatchModel::AllProjects;
......@@ -954,7 +987,6 @@ void KatePluginSearchView::startSearch()
m_curResults->treeView->expand(m_curResults->matchModel.index(0,0));
if (m_ui.searchPlaceCombo->currentIndex() == MatchModel::CurrentFile) {
m_searchDiskFilesDone = true;
m_resultBaseDir.clear();
QList<KTextEditor::Document *> documents;
KTextEditor::View *activeView = m_mainWindow->activeView();
......@@ -963,7 +995,6 @@ void KatePluginSearchView::startSearch()
}
m_searchOpenFiles.startSearch(documents, reg);
} else if (m_ui.searchPlaceCombo->currentIndex() == MatchModel::OpenFiles) {
m_searchDiskFilesDone = true;
m_resultBaseDir.clear();
const QList<KTextEditor::Document *> documents = m_kateApp->documents();
m_searchOpenFiles.startSearch(documents, reg);
......@@ -1023,11 +1054,10 @@ void KatePluginSearchView::startSearch()
// The DiskFile might finish immediately
if (!openList.empty()) {
m_searchOpenFiles.startSearch(openList, m_curResults->regExp);
} else {
m_searchOpenFilesDone = true;
}
m_searchDiskFiles.startSearch(files, reg, m_ui.binaryCheckBox->isChecked());
} else {
startDiskFileSearch(files, m_curResults->regExp, m_ui.binaryCheckBox->isChecked());
}
else {
qDebug() << "Case not handled:" << m_ui.searchPlaceCombo->currentIndex();
Q_ASSERT_X(false, "KatePluginSearchView::startSearch", "case not handled");
}
......@@ -1035,7 +1065,7 @@ void KatePluginSearchView::startSearch()
void KatePluginSearchView::startSearchWhileTyping()
{
if (!m_searchDiskFilesDone || !m_searchOpenFilesDone) {
if (searchingDiskFiles() || m_searchOpenFiles.searching()) {
return;
}
updateViewColors();
......@@ -1130,14 +1160,7 @@ void KatePluginSearchView::searchDone()
{
m_changeTimer.stop(); // avoid "while you type" search directly after
if (sender() == &m_searchDiskFiles) {
m_searchDiskFilesDone = true;
}
if (sender() == &m_searchOpenFiles) {
m_searchOpenFilesDone = true;
}
if (!m_searchDiskFilesDone || !m_searchOpenFilesDone) {
if (searchingDiskFiles() || m_searchOpenFiles.searching()) {
return;
}
......@@ -1164,19 +1187,14 @@ void KatePluginSearchView::searchDone()
m_ui.replaceButton->setDisabled(m_curResults->matches < 1);
m_ui.nextButton->setDisabled(m_curResults->matches < 1);
// FIXME m_curResults->tree->sortItems(0, Qt::AscendingOrder);
m_curResults->treeView->resizeColumnToContents(0);
if (m_curResults->treeView->columnWidth(0) < m_curResults->treeView->width() - 30) {
m_curResults->treeView->setColumnWidth(0, m_curResults->treeView->width() - 30);
}
// Set search to done. This sorts the model and collapses all items in the view
m_curResults->matchModel.setSearchState(MatchModel::SearchDone);
// expand the "header item " to display all files and all results if configured
expandResults();
m_curResults->treeView->resizeColumnToContents(0);
indicateMatch(m_curResults->matches > 0);
m_curResults = nullptr;
......@@ -1188,6 +1206,8 @@ void KatePluginSearchView::searchDone()
m_searchJustOpened = false;
updateMatchMarks();
//qDebug() << "done:" << s_timer.elapsed();
}
void KatePluginSearchView::searchWhileTypingDone()
......@@ -1829,7 +1849,7 @@ void KatePluginSearchView::tabCloseRequested(int index)
Results *tmp = qobject_cast<Results *>(m_ui.resultTabWidget->widget(index));
if (m_curResults == tmp) {
m_searchOpenFiles.cancelSearch();
m_searchDiskFiles.cancelSearch();
cancelDiskFileSearch();
m_folderFilesList.cancelSearch();
}
if (m_ui.resultTabWidget->count() > 1) {
......
......@@ -27,7 +27,8 @@
#include <ktexteditor/sessionconfiginterface.h>
#include <QTimer>
#include <QTreeWidget>
#include <QTreeView>
#include <QThreadPool>
#include <KXMLGUIClient>
......@@ -174,11 +175,18 @@ private Q_SLOTS:
void copySearchToClipboard(CopyResultType type);
void customResMenuRequested(const QPoint &pos);
Q_SIGNALS:
void cancelSearch();
protected:
bool eventFilter(QObject *obj, QEvent *ev) override;
private:
QStringList filterFiles(const QStringList &files) const;
QStringList filterFiles(const QStringList &fileList) const;
void startDiskFileSearch(const QStringList &fileList, const QRegularExpression &reg, bool includeBinaryFiles);
void cancelDiskFileSearch();
bool searchingDiskFiles();
void updateViewColors();
void onResize(const QSize &size);
......@@ -188,17 +196,15 @@ private:
KTextEditor::Application *m_kateApp;
SearchOpenFiles m_searchOpenFiles;
FolderFilesList m_folderFilesList;
SearchDiskFiles m_searchDiskFiles;
QThreadPool m_searchDiskFilePool;
QTimer m_diskSearchDoneTimer;
QAction *m_matchCase = nullptr;
QAction *m_useRegExp = nullptr;
Results *m_curResults = nullptr;
bool m_searchJustOpened = false;
int m_projectSearchPlaceIndex = 0;
bool m_searchDiskFilesDone = true;
bool m_searchOpenFilesDone = true;
bool m_isSearchAsYouType = false;
bool m_isVerticalLayout = false;
bool m_blockDiskMatchFound = false;
QString m_resultBaseDir;
QVector<KTextEditor::MovingRange *> m_matchRanges;
QTimer m_changeTimer;
......
......@@ -78,7 +78,7 @@ void SearchOpenFiles::doSearchNextFile(int startLine)
if (m_nextFileIndex == m_docList.size()) {
m_nextFileIndex = -1;
m_cancelSearch = true;
emit searchDone();
Q_EMIT searchDone();
} else {
m_nextLine = 0;
}
......@@ -92,7 +92,7 @@ int SearchOpenFiles::searchOpenFile(KTextEditor::Document *doc, const QRegularEx
{
if (m_statusTime.elapsed() > 100) {
m_statusTime.restart();
emit searching(doc->url().toString());
Q_EMIT searching(doc->url().toString());
}
if (regExp.pattern().contains(QLatin1String("\\n"))) {
......@@ -135,8 +135,8 @@ int SearchOpenFiles::searchSingleLineRegExp(KTextEditor::Document *doc, const QR
}
}
// emit all matches batched
emit matchesFound(doc->url(), matches);
// Q_EMIT all matches batched
Q_EMIT matchesFound(doc->url(), matches);
return resultLine;
}
......@@ -219,8 +219,8 @@ int SearchOpenFiles::searchMultiLineRegExp(KTextEditor::Document *doc, const QRe
}
}
// emit all matches batched
emit matchesFound(doc->url(), matches);
// Q_EMIT all matches batched
Q_EMIT matchesFound(doc->url(), matches);
return resultLine;
}