kateprojectworker.cpp 16.1 KB
Newer Older
Christoph Cullmann's avatar
Christoph Cullmann committed
1
2
/*  This file is part of the Kate project.
 *
3
 *  SPDX-FileCopyrightText: 2012 Christoph Cullmann <cullmann@kde.org>
Christoph Cullmann's avatar
Christoph Cullmann committed
4
 *
5
 *  SPDX-License-Identifier: LGPL-2.0-or-later
Christoph Cullmann's avatar
Christoph Cullmann committed
6
7
 */

8
#include "kateprojectworker.h"
Christoph Cullmann's avatar
Christoph Cullmann committed
9

Christoph Cullmann's avatar
Christoph Cullmann committed
10
11
12
13
14
#include <QDir>
#include <QDirIterator>
#include <QFile>
#include <QFileInfo>
#include <QProcess>
15
#include <QRegularExpression>
16
#include <QSet>
17
#include <QSettings>
18
#include <QThread>
Alexander Lohnau's avatar
Alexander Lohnau committed
19
#include <QTime>
Christoph Cullmann's avatar
Christoph Cullmann committed
20

21
22
#include <algorithm>

23
KateProjectWorker::KateProjectWorker(const QString &baseDir, const QString &indexDir, const QVariantMap &projectMap, bool force)
24
    : m_baseDir(baseDir)
25
    , m_indexDir(indexDir)
26
    , m_projectMap(projectMap)
27
    , m_force(force)
Christoph Cullmann's avatar
Christoph Cullmann committed
28
{
29
    Q_ASSERT(!m_baseDir.isEmpty());
Christoph Cullmann's avatar
Christoph Cullmann committed
30
31
}

32
void KateProjectWorker::run()
Christoph Cullmann's avatar
Christoph Cullmann committed
33
{
34
35
36
37
38
    /**
     * Create dummy top level parent item and empty map inside shared pointers
     * then load the project recursively
     */
    KateProjectSharedQStandardItem topLevel(new QStandardItem());
39
    KateProjectSharedQHashStringItem file2Item(new QHash<QString, KateProjectItem *>());
40
    loadProject(topLevel.data(), m_projectMap, file2Item.data());
41
42
43
44

    /**
     * create some local backup of some data we need for further processing!
     */
45
46
    const QStringList files = file2Item->keys();
    Q_EMIT loadDone(topLevel, file2Item);
47

48
    // trigger index loading, will internally handle enable/disabled
49
    loadIndex(files, m_force);
Christoph Cullmann's avatar
Christoph Cullmann committed
50
51
}

52
void KateProjectWorker::loadProject(QStandardItem *parent, const QVariantMap &project, QHash<QString, KateProjectItem *> *file2Item)
Christoph Cullmann's avatar
Christoph Cullmann committed
53
54
{
    /**
55
     * recurse to sub-projects FIRST
Christoph Cullmann's avatar
Christoph Cullmann committed
56
     */
57
    QVariantList subGroups = project[QStringLiteral("projects")].toList();
58
    for (const QVariant &subGroupVariant : subGroups) {
59
60
61
62
        /**
         * convert to map and get name, else skip
         */
        QVariantMap subProject = subGroupVariant.toMap();
63
64
        const QString keyName = QStringLiteral("name");
        if (subProject[keyName].toString().isEmpty()) {
65
66
67
68
69
70
            continue;
        }

        /**
         * recurse
         */
71
        QStandardItem *subProjectItem = new KateProjectItem(KateProjectItem::Project, subProject[keyName].toString());
72
73
74
        loadProject(subProjectItem, subProject, file2Item);
        parent->appendRow(subProjectItem);
    }
Christoph Cullmann's avatar
Christoph Cullmann committed
75
76

    /**
77
     * load all specified files
Christoph Cullmann's avatar
Christoph Cullmann committed
78
     */
79
80
    const QString keyFiles = QStringLiteral("files");
    QVariantList files = project[keyFiles].toList();
Michal Humpula's avatar
Michal Humpula committed
81
82
83
    for (const QVariant &fileVariant : files) {
        loadFilesEntry(parent, fileVariant.toMap(), file2Item);
    }
Christoph Cullmann's avatar
Christoph Cullmann committed
84
85
86
87
88
89
90
91
}

/**
 * small helper to construct directory parent items
 * @param dir2Item map for path => item
 * @param path current path we need item for
 * @return correct parent item for given path, will reuse existing ones
 */
92
static QStandardItem *directoryParent(QHash<QString, QStandardItem *> &dir2Item, QString path)
Christoph Cullmann's avatar
Christoph Cullmann committed
93
{
94
95
96
    /**
     * throw away simple /
     */
97
    if (path == QLatin1String("/")) {
98
99
        path = QString();
    }
Christoph Cullmann's avatar
Christoph Cullmann committed
100
101

    /**
102
     * quick check: dir already seen?
Christoph Cullmann's avatar
Christoph Cullmann committed
103
     */
104
105
106
    const auto existingIt = dir2Item.find(path);
    if (existingIt != dir2Item.end()) {
        return existingIt.value();
107
    }
Christoph Cullmann's avatar
Christoph Cullmann committed
108
109

    /**
110
     * else: construct recursively
Christoph Cullmann's avatar
Christoph Cullmann committed
111
     */
112
    const int slashIndex = path.lastIndexOf(QLatin1Char('/'));
Christoph Cullmann's avatar
Christoph Cullmann committed
113
114

    /**
115
116
     * no slash?
     * simple, no recursion, append new item toplevel
Christoph Cullmann's avatar
Christoph Cullmann committed
117
     */
118
    if (slashIndex < 0) {
119
120
121
122
        const auto item = new KateProjectItem(KateProjectItem::Directory, path);
        dir2Item[path] = item;
        dir2Item[QString()]->appendRow(item);
        return item;
Christoph Cullmann's avatar
Christoph Cullmann committed
123
124
    }

125
    /**
126
     * else, split and recurse
127
     */
128
129
    const QString leftPart = path.left(slashIndex);
    const QString rightPart = path.right(path.size() - (slashIndex + 1));
130
131

    /**
132
     * special handling if / with nothing on one side are found
133
     */
134
135
136
    if (leftPart.isEmpty() || rightPart.isEmpty()) {
        return directoryParent(dir2Item, leftPart.isEmpty() ? rightPart : leftPart);
    }
137
138

    /**
139
     * else: recurse on left side
140
     */
141
142
143
144
    const auto item = new KateProjectItem(KateProjectItem::Directory, rightPart);
    dir2Item[path] = item;
    directoryParent(dir2Item, leftPart)->appendRow(item);
    return item;
145
146
}

147
void KateProjectWorker::loadFilesEntry(QStandardItem *parent, const QVariantMap &filesEntry, QHash<QString, KateProjectItem *> *file2Item)
148
149
150
151
{
    QDir dir(m_baseDir);
    if (!dir.cd(filesEntry[QStringLiteral("directory")].toString())) {
        return;
152
153
    }

154
155
156
    /**
     * get list of files for this directory, might query the VCS
     */
157
    QStringList files = findFiles(dir, filesEntry);
Christoph Cullmann's avatar
Christoph Cullmann committed
158

159
160
161
162
163
164
165
166
167
168
169
    /**
     * sort out non-files
     * even for git, that just reports non-directories, we need to filter out e.g. sym-links to directories
     */
    files.erase(std::remove_if(files.begin(),
                               files.end(),
                               [](const QString &item) {
                                   return !QFileInfo(item).isFile();
                               }),
                files.end());

170
171
172
173
    if (files.isEmpty()) {
        return;
    }

174
    files.sort(Qt::CaseInsensitive);
Christoph Cullmann's avatar
Christoph Cullmann committed
175
176

    /**
177
     * construct paths first in tree and items in a map
Christoph Cullmann's avatar
Christoph Cullmann committed
178
     */
179
    QHash<QString, QStandardItem *> dir2Item;
180
    dir2Item[QString()] = parent;
181
    QVector<QPair<QStandardItem *, QStandardItem *>> item2ParentPath;
182
    for (const QString &filePath : files) {
183
        /**
184
185
         * cheap file name computation
         * we do this A LOT, QFileInfo is very expensive just for this operation
186
         */
187
188
189
        const int slashIndex = filePath.lastIndexOf(QLatin1Char('/'));
        const QString fileName = (slashIndex < 0) ? filePath : filePath.mid(slashIndex + 1);
        const QString filePathName = (slashIndex < 0) ? QString() : filePath.left(slashIndex);
190
191

        /**
192
193
         * construct the item with right directory prefix
         * already hang in directories in tree
194
         */
195
        KateProjectItem *fileItem = new KateProjectItem(KateProjectItem::File, fileName);
196
        fileItem->setData(filePath, Qt::ToolTipRole);
Valentin Rouet's avatar
Valentin Rouet committed
197
198

        // get the directory's relative path to the base directory
199
        QString dirRelPath = dir.relativeFilePath(filePathName);
Valentin Rouet's avatar
Valentin Rouet committed
200
        // if the relative path is ".", clean it up
201
        if (dirRelPath == QLatin1Char('.')) {
Valentin Rouet's avatar
Valentin Rouet committed
202
203
204
205
            dirRelPath = QString();
        }

        item2ParentPath.append(QPair<QStandardItem *, QStandardItem *>(fileItem, directoryParent(dir2Item, dirRelPath)));
206
207
        fileItem->setData(filePath, Qt::UserRole);
        (*file2Item)[filePath] = fileItem;
208
    }
Christoph Cullmann's avatar
Christoph Cullmann committed
209

210
    /**
211
     * plug in the file items to the tree
212
     */
213
214
    for (const auto &item : qAsConst(item2ParentPath)) {
        item.second->appendRow(item.first);
215
216
    }
}
217

218
QStringList KateProjectWorker::findFiles(const QDir &dir, const QVariantMap &filesEntry)
219
{
220
221
222
    /**
     * shall we collect files recursively or not?
     */
223
    const bool recursive = !filesEntry.contains(QLatin1String("recursive")) || filesEntry[QStringLiteral("recursive")].toBool();
224

225
226
227
228
    /**
     * try the different version control systems first
     */

229
230
    if (filesEntry[QStringLiteral("git")].toBool()) {
        return filesFromGit(dir, recursive);
231
    }
232

233
    if (filesEntry[QStringLiteral("svn")].toBool()) {
234
        return filesFromSubversion(dir, recursive);
235
    }
236

237
    if (filesEntry[QStringLiteral("hg")].toBool()) {
238
        return filesFromMercurial(dir, recursive);
239
    }
240
241

    if (filesEntry[QStringLiteral("darcs")].toBool()) {
242
        return filesFromDarcs(dir, recursive);
243
244
245
246
247
248
249
    }

    /**
     * if we arrive here, we have some manual specification of files, no VCS
     */

    /**
250
     * try explicit list of stuff
251
     */
252
    QStringList userGivenFilesList = filesEntry[QStringLiteral("list")].toStringList();
253
254
255
256
257
258
259
260
261
262
263
264
265
    if (!userGivenFilesList.empty()) {
        /**
         * users might have specified duplicates, this can't happen for the other ways
         */
        userGivenFilesList.removeDuplicates();
        return userGivenFilesList;
    }

    /**
     * if nothing found for that, try to use filters to scan the directory
     * here we only get files
     */
    return filesFromDirectory(dir, recursive, filesEntry[QStringLiteral("filters")].toStringList());
266
}
Christoph Cullmann's avatar
Christoph Cullmann committed
267

268
QStringList KateProjectWorker::filesFromGit(const QDir &dir, bool recursive)
269
{
270
    /**
Christoph Cullmann's avatar
Christoph Cullmann committed
271
     * query files via ls-files and make them absolute afterwards
272
273
     */
    const QStringList relFiles = gitLsFiles(dir);
274
    QStringList files;
275
    for (const QString &relFile : relFiles) {
276
        if (!recursive && (relFile.indexOf(QLatin1Char('/')) != -1)) {
277
278
            continue;
        }
279

280
281
282
283
284
285
286
        files.append(dir.absolutePath() + QLatin1Char('/') + relFile);
    }
    return files;
}

QStringList KateProjectWorker::gitLsFiles(const QDir &dir)
{
287
288
289
290
291
292
293
    /**
     * git ls-files -z results a bytearray where each entry is \0-terminated.
     * NOTE: Without -z, Umlauts such as "Der Bäcker/Das Brötchen.txt" do not work (#389415)
     *
     * use --recurse-submodules, there since git 2.11 (released 2016)
     * our own submodules handling code leads to file duplicates
     */
294
    QStringList args;
295
    args << QStringLiteral("ls-files") << QStringLiteral("-z") << QStringLiteral("--recurse-submodules") << QStringLiteral(".");
296
297
298

    QProcess git;
    git.setWorkingDirectory(dir.absolutePath());
299
    git.start(QStringLiteral("git"), args);
300
    QStringList files;
301
    if (!git.waitForStarted() || !git.waitForFinished(-1)) {
302
303
        return files;
    }
304

305
    const QList<QByteArray> byteArrayList = git.readAllStandardOutput().split('\0');
306
    for (const QByteArray &byteArray : byteArrayList) {
307
308
309
310
        const QString fileName = QString::fromUtf8(byteArray);
        if (!fileName.isEmpty()) {
            files << fileName;
        }
311
    }
312

313
314
    return files;
}
315

316
QStringList KateProjectWorker::filesFromMercurial(const QDir &dir, bool recursive)
317
318
{
    QStringList files;
319

320
321
322
323
324
    QProcess hg;
    hg.setWorkingDirectory(dir.absolutePath());
    QStringList args;
    args << QStringLiteral("manifest") << QStringLiteral(".");
    hg.start(QStringLiteral("hg"), args);
325
    if (!hg.waitForStarted() || !hg.waitForFinished(-1)) {
326
327
        return files;
    }
328

Laurent Montel's avatar
Laurent Montel committed
329
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
Alexander Lohnau's avatar
Alexander Lohnau committed
330
331
    const QStringList relFiles =
        QString::fromLocal8Bit(hg.readAllStandardOutput()).split(QRegularExpression(QStringLiteral("[\n\r]")), QString::SkipEmptyParts);
Laurent Montel's avatar
Laurent Montel committed
332
333
334
#else
    const QStringList relFiles = QString::fromLocal8Bit(hg.readAllStandardOutput()).split(QRegularExpression(QStringLiteral("[\n\r]")), Qt::SkipEmptyParts);
#endif
335

336
    for (const QString &relFile : relFiles) {
337
        if (!recursive && (relFile.indexOf(QLatin1Char('/')) != -1)) {
338
            continue;
339
        }
340
341
342
343
344
345
346

        files.append(dir.absolutePath() + QLatin1Char('/') + relFile);
    }

    return files;
}

347
QStringList KateProjectWorker::filesFromSubversion(const QDir &dir, bool recursive)
348
349
350
351
352
353
354
355
356
357
358
359
360
{
    QStringList files;

    QProcess svn;
    svn.setWorkingDirectory(dir.absolutePath());
    QStringList args;
    args << QStringLiteral("status") << QStringLiteral("--verbose") << QStringLiteral(".");
    if (recursive) {
        args << QStringLiteral("--depth=infinity");
    } else {
        args << QStringLiteral("--depth=files");
    }
    svn.start(QStringLiteral("svn"), args);
361
    if (!svn.waitForStarted() || !svn.waitForFinished(-1)) {
362
        return files;
Christoph Cullmann's avatar
Christoph Cullmann committed
363
    }
364

Christoph Cullmann's avatar
Christoph Cullmann committed
365
    /**
366
     * get output and split up into lines
Christoph Cullmann's avatar
Christoph Cullmann committed
367
     */
Laurent Montel's avatar
Laurent Montel committed
368
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
369
    const QStringList lines = QString::fromLocal8Bit(svn.readAllStandardOutput()).split(QRegularExpression(QStringLiteral("[\n\r]")), QString::SkipEmptyParts);
Laurent Montel's avatar
Laurent Montel committed
370
371
372
#else
    const QStringList lines = QString::fromLocal8Bit(svn.readAllStandardOutput()).split(QRegularExpression(QStringLiteral("[\n\r]")), Qt::SkipEmptyParts);
#endif
Christoph Cullmann's avatar
Christoph Cullmann committed
373
374

    /**
375
     * remove start of line that is no filename, sort out unknown and ignore
376
     */
377
378
379
    bool first = true;
    int prefixLength = -1;

380
    for (const QString &line : lines) {
381
        /**
382
         * get length of stuff to cut
383
         */
384
385
386
387
        if (first) {
            /**
             * try to find ., else fail
             */
388
            prefixLength = line.lastIndexOf(QLatin1Char('.'));
389
390
391
            if (prefixLength < 0) {
                break;
            }
392

393
394
395
396
            /**
             * skip first
             */
            first = false;
397
398
399
400
            continue;
        }

        /**
401
402
         * get file, if not unknown or ignored
         * prepend directory path
403
         */
404
405
406
407
408
409
410
411
        if ((line.size() > prefixLength) && line[0] != QLatin1Char('?') && line[0] != QLatin1Char('I')) {
            files.append(dir.absolutePath() + QLatin1Char('/') + line.right(line.size() - prefixLength));
        }
    }

    return files;
}

412
QStringList KateProjectWorker::filesFromDarcs(const QDir &dir, bool recursive)
413
414
415
416
417
418
419
420
421
422
423
424
425
426
{
    QStringList files;

    const QString cmd = QStringLiteral("darcs");
    QString root;

    {
        QProcess darcs;
        darcs.setWorkingDirectory(dir.absolutePath());
        QStringList args;
        args << QStringLiteral("list") << QStringLiteral("repo");

        darcs.start(cmd, args);

427
        if (!darcs.waitForStarted() || !darcs.waitForFinished(-1)) {
428
            return files;
429
        }
430
431
432
433
434

        auto str = QString::fromLocal8Bit(darcs.readAllStandardOutput());
        QRegularExpression exp(QStringLiteral("Root: ([^\\n\\r]*)"));
        auto match = exp.match(str);

435
        if (!match.hasMatch()) {
436
            return files;
437
        }
438
439
440
441
442
443
444
445
446

        root = match.captured(1);
    }

    QStringList relFiles;
    {
        QProcess darcs;
        QStringList args;
        darcs.setWorkingDirectory(dir.absolutePath());
447
        args << QStringLiteral("list") << QStringLiteral("files") << QStringLiteral("--no-directories") << QStringLiteral("--pending");
448
449
450

        darcs.start(cmd, args);

451
        if (!darcs.waitForStarted() || !darcs.waitForFinished(-1)) {
452
            return files;
453
        }
454

Laurent Montel's avatar
Laurent Montel committed
455
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
456
        relFiles = QString::fromLocal8Bit(darcs.readAllStandardOutput()).split(QRegularExpression(QStringLiteral("[\n\r]")), QString::SkipEmptyParts);
Laurent Montel's avatar
Laurent Montel committed
457
458
459
#else
        relFiles = QString::fromLocal8Bit(darcs.readAllStandardOutput()).split(QRegularExpression(QStringLiteral("[\n\r]")), Qt::SkipEmptyParts);
#endif
460
461
    }

462
    for (const QString &relFile : relFiles) {
463
        const QString path = dir.relativeFilePath(root + QLatin1String("/") + relFile);
464

465
        if ((!recursive && (relFile.indexOf(QLatin1Char('/')) != -1)) || (recursive && (relFile.indexOf(QLatin1String("..")) == 0))) {
466
            continue;
467
        }
468
469
470
471
472
473
474

        files.append(dir.absoluteFilePath(path));
    }

    return files;
}

475
QStringList KateProjectWorker::filesFromDirectory(const QDir &_dir, bool recursive, const QStringList &filters)
476
{
477
478
479
    /**
     * setup our filters, we only want files!
     */
480
481
482
483
    QDir dir(_dir);
    dir.setFilter(QDir::Files);
    if (!filters.isEmpty()) {
        dir.setNameFilters(filters);
484
485
486
    }

    /**
487
     * construct flags for iterator
488
     */
489
490
491
    QDirIterator::IteratorFlags flags = QDirIterator::NoIteratorFlags;
    if (recursive) {
        flags = flags | QDirIterator::Subdirectories;
492
    }
493
494
495
496

    /**
     * create iterator and collect all files
     */
497
    QStringList files;
498
499
500
501
502
503
    QDirIterator dirIterator(dir, flags);
    while (dirIterator.hasNext()) {
        dirIterator.next();
        files.append(dirIterator.filePath());
    }
    return files;
Christoph Cullmann's avatar
Christoph Cullmann committed
504
505
}

506
void KateProjectWorker::loadIndex(const QStringList &files, bool force)
507
{
508
509
    const QString keyCtags = QStringLiteral("ctags");
    const QVariantMap ctagsMap = m_projectMap[keyCtags].toMap();
510
511
512
    /**
     * load index, if enabled
     * before this was default on, which is dangerous for large repositories, e.g. out-of-memory or out-of-disk
513
     * if specified in project map; use that setting, otherwise fall back to global setting
514
     */
515
    bool indexEnabled = !m_indexDir.isEmpty();
516
    auto indexValue = ctagsMap[QStringLiteral("enable")];
517
518
519
520
    if (!indexValue.isNull()) {
        indexEnabled = indexValue.toBool();
    }
    if (!indexEnabled) {
521
522
523
524
        emit loadIndexDone(KateProjectSharedProjectIndex());
        return;
    }

525
526
527
528
    /**
     * create new index, this will do the loading in the constructor
     * wrap it into shared pointer for transfer to main thread
     */
529
    KateProjectSharedProjectIndex index(new KateProjectIndex(m_baseDir, m_indexDir, files, ctagsMap, force));
530

531
    emit loadIndexDone(index);
532
}