storagejanitor.cpp 31.3 KB
Newer Older
Volker Krause's avatar
Volker Krause committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/*
    Copyright (c) 2011 Volker Krause <vkrause@kde.org>

    This library is free software; you can redistribute it and/or modify it
    under the terms of the GNU Library General Public License as published by
    the Free Software Foundation; either version 2 of the License, or (at your
    option) any later version.

    This library is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
    License for more details.

    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to the
    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.
*/

#include "storagejanitor.h"
21

22
23
#include "storage/queryhelper.h"
#include "storage/transaction.h"
Volker Krause's avatar
Volker Krause committed
24
#include "storage/datastore.h"
25
#include "storage/selectquerybuilder.h"
26
#include "storage/parthelper.h"
27
#include "storage/dbconfig.h"
28
#include "storage/collectionstatistics.h"
29
30
#include "search/searchrequest.h"
#include "search/searchmanager.h"
31
#include "resourcemanager.h"
32
#include "entities.h"
33
#include "dbusconnectionpool.h"
34
#include "agentmanagerinterface.h"
35
#include "akonadiserver_debug.h"
36

37
#include <private/dbus_p.h>
38
#include <private/imapset_p.h>
39
#include <private/protocol_p.h>
40
#include <private/standarddirs_p.h>
41
#include <private/externalpartstorage_p.h>
42

Volker Krause's avatar
Volker Krause committed
43
#include <QStringBuilder>
Daniel Vrátil's avatar
Daniel Vrátil committed
44
45
46
47
48
#include <QDBusConnection>
#include <QSqlQuery>
#include <QSqlError>
#include <QDir>
#include <qdiriterator.h>
49
#include <QDateTime>
Volker Krause's avatar
Volker Krause committed
50

51
52
#include <algorithm>

53
using namespace Akonadi;
54
using namespace Akonadi::Server;
Volker Krause's avatar
Volker Krause committed
55

56
StorageJanitor::StorageJanitor(QObject *parent)
57
    : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority, parent)
58
    , m_lostFoundCollectionId(-1)
Volker Krause's avatar
Volker Krause committed
59
60
61
{
}

62
StorageJanitor::~StorageJanitor()
Volker Krause's avatar
Volker Krause committed
63
{
64
    quitThread();
65
66
}

67
void StorageJanitor::init()
68
{
69
70
71
72
73
74
    AkThread::init();

    QDBusConnection conn = DBusConnectionPool::threadConnection();
    conn.registerService(DBus::serviceName(DBus::StorageJanitor));
    conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), this,
                        QDBusConnection::ExportScriptableSlots | QDBusConnection::ExportScriptableSignals);
75
76
}

77
void StorageJanitor::quit()
78
{
79
80
81
82
83
84
85
    QDBusConnection conn = DBusConnectionPool::threadConnection();
    conn.unregisterObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), QDBusConnection::UnregisterTree);
    conn.unregisterService(DBus::serviceName(DBus::StorageJanitor));
    conn.disconnectFromBus(conn.name());

    // Make sure all childrens are deleted within context of this thread
    qDeleteAll(children());
86

87
88
    qDebug() << "chainup()";
    AkThread::quit();
Volker Krause's avatar
Volker Krause committed
89
90
}

91
void StorageJanitor::check() // implementation of `akonadictl fsck`
Volker Krause's avatar
Volker Krause committed
92
{
93
    m_lostFoundCollectionId = -1; // start with a fresh one each time
94

95
96
    inform("Looking for resources in the DB not matching a configured resource...");
    findOrphanedResources();
97

98
99
    inform("Looking for collections not belonging to a valid resource...");
    findOrphanedCollections();
100

101
102
    inform("Checking collection tree consistency...");
    const Collection::List cols = Collection::retrieveAll();
Laurent Montel's avatar
Laurent Montel committed
103
104
105
    std::for_each(cols.begin(), cols.end(), [this](const Collection & col) {
        checkPathToRoot(col);
    });
106

107
108
    inform("Looking for items not belonging to a valid collection...");
    findOrphanedItems();
109

110
111
    inform("Looking for item parts not belonging to a valid item...");
    findOrphanedParts();
112

113
114
    inform("Looking for item flags not belonging to a valid item...");
    findOrphanedPimItemFlags();
115

116
117
    inform("Looking for overlapping external parts...");
    findOverlappingParts();
118

119
120
    inform("Verifying external parts...");
    verifyExternalParts();
121

122
123
    inform("Checking size treshold changes...");
    checkSizeTreshold();
124

125
126
    inform("Looking for dirty objects...");
    findDirtyObjects();
127

128
129
130
    inform("Looking for rid-duplicates not matching the content mime-type of the parent collection");
    findRIDDuplicates();

131
132
133
    inform("Migrating parts to new cache hierarchy...");
    migrateToLevelledCacheHierarchy();

134
    inform("Checking search index consistency...");
Andreas Hartmetz's avatar
Andreas Hartmetz committed
135
    findOrphanSearchIndexEntries();
136

137
138
139
    inform("Flushing collection statistics memory cache...");
    CollectionStatistics::self()->expireCache();

140
141
142
143
144
145
146
147
    /* TODO some ideas for further checks:
     * the collection tree is non-cyclic
     * content type constraints of collections are not violated
     * find unused flags
     * find unused mimetypes
     * check for dead entries in relation tables
     * check if part size matches file size
     */
148

149
    inform("Consistency check done.");
150
151

    Q_EMIT done();
152
153
}

154
155
qint64 StorageJanitor::lostAndFoundCollection()
{
156
157
    if (m_lostFoundCollectionId > 0) {
        return m_lostFoundCollectionId;
Guy Maurel's avatar
Guy Maurel committed
158
    }
159

160
    Transaction transaction(DataStore::self(), QStringLiteral("JANITOR LOST+FOUND"));
161
    Resource lfRes = Resource::retrieveByName(QStringLiteral("akonadi_lost+found_resource"));
162
    if (!lfRes.isValid()) {
163
        lfRes.setName(QStringLiteral("akonadi_lost+found_resource"));
164
        if (!lfRes.insert()) {
165
            qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!";
166
        }
Guy Maurel's avatar
Guy Maurel committed
167
    }
168
169
170
171
172

    Collection lfRoot;
    SelectQueryBuilder<Collection> qb;
    qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id());
    qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant());
173
    if (!qb.exec()) {
174
        qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections";
175
176
        return -1;
    }
177
178
    const Collection::List cols = qb.result();
    if (cols.size() > 1) {
179
        qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?";
180
181
182
    } else if (cols.size() == 1) {
        lfRoot = cols.first();
    } else {
183
        lfRoot.setName(QStringLiteral("lost+found"));
184
        lfRoot.setResourceId(lfRes.id());
185
        lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL"));
186
187
188
        lfRoot.setCachePolicyCacheTimeout(-1);
        lfRoot.setCachePolicyInherit(false);
        if (!lfRoot.insert()) {
189
            qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root.";
190
191
192
193
194
        }
        DataStore::self()->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8());
    }

    Collection lfCol;
195
    lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")));
196
197
198
    lfCol.setResourceId(lfRes.id());
    lfCol.setParentId(lfRoot.id());
    if (!lfCol.insert()) {
199
        qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!";
200
201
202
203
204
205
206
207
208
209
210
    }

    Q_FOREACH (const MimeType &mt, MimeType::retrieveAll()) {
        lfCol.addMimeType(mt);
    }

    DataStore::self()->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8());

    transaction.commit();
    m_lostFoundCollectionId = lfCol.id();
    return m_lostFoundCollectionId;
211
212
}

213
214
void StorageJanitor::findOrphanedResources()
{
215
216
    SelectQueryBuilder<Resource> qbres;
    OrgFreedesktopAkonadiAgentManagerInterface iface(
217
        DBus::serviceName(DBus::Control),
218
        QStringLiteral("/AgentManager"),
219
220
221
        QDBusConnection::sessionBus(),
        this);
    if (!iface.isValid()) {
222
        inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control));
223
        return;
224
    }
225
226
    const QStringList knownResources = iface.agentInstances();
    if (knownResources.isEmpty()) {
227
        inform(QStringLiteral("ERROR: no known resources. This must be a mistake?"));
228
229
        return;
    }
230
    qCDebug(AKONADISERVER_LOG) << "Known resources:" << knownResources;
231
232
    qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources));
    qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1);   // skip akonadi_search_resource
233
234
235
236
    if (!qbres.exec()) {
        inform("Failed to query known resources, skipping test");
        return;
    }
237
    //qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery();
238
    const Resource::List orphanResources = qbres.result();
Laurent Montel's avatar
Laurent Montel committed
239
240
    const int orphanResourcesSize(orphanResources.size());
    if (orphanResourcesSize > 0) {
241
        QStringList resourceNames;
Laurent Montel's avatar
Laurent Montel committed
242
        resourceNames.reserve(orphanResourcesSize);
Laurent Montel's avatar
Laurent Montel committed
243
        for (const Resource &resource : orphanResources) {
244
245
            resourceNames.append(resource.name());
        }
246
        inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResources.size()). arg(resourceNames.join(QLatin1Char(','))));
Laurent Montel's avatar
Laurent Montel committed
247
        for (const QString &resourceName : qAsConst(resourceNames)) {
248
            inform(QStringLiteral("Removing resource %1").arg(resourceName));
249
250
            ResourceManager::self()->removeResourceInstance(resourceName);
        }
251
252
253
    }
}

254
255
void StorageJanitor::findOrphanedCollections()
{
256
257
258
259
    SelectQueryBuilder<Collection> qb;
    qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
    qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant());

260
261
262
263
    if (!qb.exec()) {
        inform("Failed to query orphaned collections, skipping test");
        return;
    }
264
265
266
267
268
    const Collection::List orphans = qb.result();
    if (!orphans.isEmpty()) {
        inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan collections."));
        // TODO: attach to lost+found resource
    }
269
270
}

271
void StorageJanitor::checkPathToRoot(const Collection &col)
272
{
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
    if (col.parentId() == 0) {
        return;
    }
    const Collection parent = col.parent();
    if (!parent.isValid()) {
        inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id())
               + QLatin1Literal(") has no valid parent."));
        // TODO fix that by attaching to a top-level lost+found folder
        return;
    }

    if (col.resourceId() != parent.resourceId()) {
        inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id())
               + QLatin1Literal(") belongs to a different resource than its parent."));
        // can/should we actually fix that?
    }

    checkPathToRoot(parent);
Volker Krause's avatar
Volker Krause committed
291
292
}

293
294
void StorageJanitor::findOrphanedItems()
{
295
296
297
    SelectQueryBuilder<PimItem> qb;
    qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
    qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant());
298
299
300
301
    if (!qb.exec()) {
        inform("Failed to query orphaned items, skipping test");
        return;
    }
302
    const PimItem::List orphans = qb.result();
Laurent Montel's avatar
Laurent Montel committed
303
    if (!orphans.isEmpty()) {
304
305
        inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan items."));
        // Attach to lost+found collection
306
        Transaction transaction(DataStore::self(), QStringLiteral("JANITOR ORPHANS"));
307
308
        QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
        qint64 col = lostAndFoundCollection();
309
310
311
        if (col == -1) {
            return;
        }
312
313
        qb.setColumnValue(PimItem::collectionIdFullColumnName(), col);
        QVector<ImapSet::Id> imapIds;
314
        imapIds.reserve(orphans.count());
Laurent Montel's avatar
Laurent Montel committed
315
        for (const PimItem &item : qAsConst(orphans)) {
316
317
318
319
320
321
322
323
324
325
            imapIds.append(item.id());
        }
        ImapSet set;
        set.add(imapIds);
        QueryHelper::setToQuery(set, PimItem::idFullColumnName(), qb);
        if (qb.exec() && transaction.commit()) {
            inform(QLatin1Literal("Moved orphan items to collection ") + QString::number(col));
        } else {
            inform(QLatin1Literal("Error moving orphan items to collection ") + QString::number(col) + QLatin1Literal(" : ") + qb.query().lastError().text());
        }
Guy Maurel's avatar
Guy Maurel committed
326
    }
327
328
}

329
330
void StorageJanitor::findOrphanedParts()
{
331
332
333
    SelectQueryBuilder<Part> qb;
    qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
    qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
334
335
336
337
    if (!qb.exec()) {
        inform("Failed to query orphaned parts, skipping test");
        return;
    }
338
    const Part::List orphans = qb.result();
Laurent Montel's avatar
Laurent Montel committed
339
    if (!orphans.isEmpty()) {
340
341
342
        inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan parts."));
        // TODO: create lost+found items for those? delete?
    }
343
344
}

345
void StorageJanitor:: findOrphanedPimItemFlags()
346
{
347
348
349
350
351
    QueryBuilder sqb(PimItemFlagRelation::tableName(), QueryBuilder::Select);
    sqb.addColumn(PimItemFlagRelation::leftFullColumnName());
    sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName());
    sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant());
    if (!sqb.exec()) {
352
        inform("Failed to query orphaned item flags, skipping test");
353
354
355
356
357
358
359
        return;
    }
    QVector<ImapSet::Id> imapIds;
    int count = 0;
    while (sqb.query().next()) {
        ++count;
        imapIds.append(sqb.query().value(0).toInt());
360
361
    }

362
363
364
365
366
367
    if (count > 0) {
        ImapSet set;
        set.add(imapIds);
        QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete);
        QueryHelper::setToQuery(set, PimItemFlagRelation::leftFullColumnName(), qb);
        if (!qb.exec()) {
368
            qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text();
369
370
371
372
373
            return;
        }

        inform(QLatin1Literal("Found and deleted ") + QString::number(count) + QLatin1Literal(" orphan pim item flags."));
    }
374
375
}

376
377
void StorageJanitor::findOverlappingParts()
{
378
379
380
    QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
    qb.addColumn(Part::dataColumn());
    qb.addColumn(QLatin1Literal("count(") + Part::idColumn() + QLatin1Literal(") as cnt"));
381
    qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
382
383
384
    qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
    qb.addGroupColumn(Part::dataColumn());
    qb.addValueCondition(QLatin1Literal("count(") + Part::idColumn() + QLatin1Literal(")"), Query::Greater, 1, QueryBuilder::HavingCondition);
385
386
387
388
    if (!qb.exec()) {
        inform("Failed to query overlapping parts, skipping test");
        return;
    }
389
390
391
392
393
394
395
396
397
398
399

    int count = 0;
    while (qb.query().next()) {
        ++count;
        inform(QLatin1Literal("Found overlapping part data: ") + qb.query().value(0).toString());
        // TODO: uh oh, this is bad, how do we recover from that?
    }

    if (count > 0) {
        inform(QLatin1Literal("Found ") + QString::number(count) + QLatin1Literal(" overlapping parts - bad."));
    }
400
401
}

402
403
void StorageJanitor::verifyExternalParts()
{
404
405
406
407
    QSet<QString> existingFiles;
    QSet<QString> usedFiles;

    // list all files
408
    const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data"));
409
    QDirIterator it(dataDir, QDir::Files, QDirIterator::Subdirectories);
410
411
412
413
414
415
416
417
418
419
420
421
    while (it.hasNext()) {
        existingFiles.insert(it.next());
    }
    existingFiles.remove(dataDir + QDir::separator() + QLatin1String("."));
    existingFiles.remove(dataDir + QDir::separator() + QLatin1String(".."));
    inform(QLatin1Literal("Found ") + QString::number(existingFiles.size()) + QLatin1Literal(" external files."));

    // list all parts from the db which claim to have an associated file
    QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
    qb.addColumn(Part::dataColumn());
    qb.addColumn(Part::pimItemIdColumn());
    qb.addColumn(Part::idColumn());
422
    qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
423
    qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant());
424
425
426
427
    if (!qb.exec()) {
        inform("Failed to query existing parts, skipping test");
        return;
    }
428
    while (qb.query().next()) {
429
        QString partPath = ExternalPartStorage::resolveAbsolutePath(qb.query().value(0).toByteArray());
430
431
432
433
434
435
436
437
438
439
440
441
        const Entity::Id pimItemId = qb.query().value(1).value<Entity::Id>();
        const Entity::Id id = qb.query().value(2).value<Entity::Id>();
        if (existingFiles.contains(partPath)) {
            usedFiles.insert(partPath);
        } else {
            inform(QLatin1Literal("Cleaning up missing external file: ") + partPath + QLatin1Literal(" for item: ") + QString::number(pimItemId) + QLatin1Literal(" on part: ") + QString::number(id));

            Part part;
            part.setId(id);
            part.setPimItemId(pimItemId);
            part.setData(QByteArray());
            part.setDatasize(0);
442
            part.setStorage(Part::Internal);
443
444
            part.update();
        }
445
    }
446
447
448
449
450
    inform(QLatin1Literal("Found ") + QString::number(usedFiles.size()) + QLatin1Literal(" external parts."));

    // see what's left and move it to lost+found
    const QSet<QString> unreferencedFiles = existingFiles - usedFiles;
    if (!unreferencedFiles.isEmpty()) {
451
        const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found"));
Laurent Montel's avatar
Laurent Montel committed
452
        for (const QString &file : unreferencedFiles) {
453
454
455
456
            inform(QLatin1Literal("Found unreferenced external file: ") + file);
            const QFileInfo f(file);
            QFile::rename(file, lfDir + QDir::separator() + f.fileName());
        }
457
        inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size()));
458
459
    } else {
        inform("Found no unreferenced external files.");
460
    }
461
462
}

463
464
void StorageJanitor::findDirtyObjects()
{
465
466
467
468
    SelectQueryBuilder<Collection> cqb;
    cqb.setSubQueryMode(Query::Or);
    cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant());
    cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString());
469
470
471
472
    if (!cqb.exec()) {
        inform("Failed to query collections without RID, skipping test");
        return;
    }
473
    const Collection::List ridLessCols = cqb.result();
Laurent Montel's avatar
Laurent Montel committed
474
    for (const Collection &col : ridLessCols) {
475
476
477
478
479
480
481
482
483
        inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id())
               + QLatin1Literal(") has no RID."));
    }
    inform(QLatin1Literal("Found ") + QString::number(ridLessCols.size()) + QLatin1Literal(" collections without RID."));

    SelectQueryBuilder<PimItem> iqb1;
    iqb1.setSubQueryMode(Query::Or);
    iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant());
    iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString());
484
485
486
487
    if (!iqb1.exec()) {
        inform("Failed to query items without RID, skipping test");
        return;
    }
488
    const PimItem::List ridLessItems = iqb1.result();
Laurent Montel's avatar
Laurent Montel committed
489
    for (const PimItem &item : ridLessItems) {
490
491
492
493
494
495
496
497
        inform(QLatin1Literal("Item \"") + QString::number(item.id()) + QLatin1Literal("\" has no RID."));
    }
    inform(QLatin1Literal("Found ") + QString::number(ridLessItems.size()) + QLatin1Literal(" items without RID."));

    SelectQueryBuilder<PimItem> iqb2;
    iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true);
    iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
    iqb2.addSortColumn(PimItem::idFullColumnName());
498
499
500
501
    if (!iqb2.exec()) {
        inform("Failed to query dirty items, skipping test");
        return;
    }
502
    const PimItem::List dirtyItems = iqb2.result();
Laurent Montel's avatar
Laurent Montel committed
503
    for (const PimItem &item : dirtyItems) {
504
505
506
        inform(QLatin1Literal("Item \"") + QString::number(item.id()) + QLatin1Literal("\" has RID and is dirty."));
    }
    inform(QLatin1Literal("Found ") + QString::number(dirtyItems.size()) + QLatin1Literal(" dirty items."));
507
508
}

509
510
511
512
513
514
void StorageJanitor::findRIDDuplicates()
{
    QueryBuilder qb(Collection::tableName(), QueryBuilder::Select);
    qb.addColumn(Collection::idColumn());
    qb.addColumn(Collection::nameColumn());
    qb.exec();
Laurent Montel's avatar
Laurent Montel committed
515

516
517
518
519
    while (qb.query().next()) {
        const Collection::Id id = qb.query().value(0).value<Collection::Id>();
        const QString name = qb.query().value(1).toString();
        inform(QStringLiteral("Checking ") + name);
Laurent Montel's avatar
Laurent Montel committed
520

521
522
523
524
525
526
        QueryBuilder duplicates(PimItem::tableName(), QueryBuilder::Select);
        duplicates.addColumn(PimItem::remoteIdColumn());
        duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt"));
        duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant());
        duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, id);
        duplicates.addGroupColumn(PimItem::remoteIdColumn());
Laurent Montel's avatar
Laurent Montel committed
527
        duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition);
528
        duplicates.exec();
Laurent Montel's avatar
Laurent Montel committed
529

530
531
532
        Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(id);
        const QVector<Akonadi::Server::MimeType> contentMimeTypes = col.mimeTypes();
        QVariantList contentMimeTypesVariantList;
Laurent Montel's avatar
Laurent Montel committed
533
        for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) {
534
535
536
537
538
            contentMimeTypesVariantList << mimeType.id();
        }
        while (duplicates.query().next()) {
            const QString rid = duplicates.query().value(0).toString();
            inform(QStringLiteral("Found duplicates ") + rid);
Laurent Montel's avatar
Laurent Montel committed
539

540
541
542
543
544
545
546
547
548
549
            QueryBuilder items(PimItem::tableName(), QueryBuilder::Delete);
            items.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid);
            items.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList);
            if (!items.exec()) {
                inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text());
            }
        }
    }
}

Volker Krause's avatar
Volker Krause committed
550
551
void StorageJanitor::vacuum()
{
552
553
554
555
    const DbType::Type dbType = DbType::type(DataStore::self()->database());
    if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) {
        inform("vacuuming database, that'll take some time and require a lot of temporary disk space...");
        Q_FOREACH (const QString &table, allDatabaseTables()) {
556
            inform(QStringLiteral("optimizing table %1...").arg(table));
557
558
559
560
561
562
563
564
565
566
567

            QString queryStr;
            if (dbType == DbType::MySQL) {
                queryStr = QLatin1Literal("OPTIMIZE TABLE ") + table;
            } else if (dbType == DbType::PostgreSQL) {
                queryStr = QLatin1Literal("VACUUM FULL ANALYZE ") + table;
            } else {
                continue;
            }
            QSqlQuery q(DataStore::self()->database());
            if (!q.exec(queryStr)) {
568
                qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text();
569
570
571
572
573
            }
        }
        inform("vacuum done");
    } else {
        inform("Vacuum not supported for this database backend.");
Volker Krause's avatar
Volker Krause committed
574
    }
575
576

    Q_EMIT done();
Volker Krause's avatar
Volker Krause committed
577
578
}

579
580
void StorageJanitor::checkSizeTreshold()
{
581
582
583
    {
        QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
        qb.addColumn(Part::idFullColumnName());
584
        qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal);
585
        qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, DbConfig::configuredDatabase()->sizeThreshold());
586
587
588
589
        if (!qb.exec()) {
            inform("Failed to query parts larger than treshold, skipping test");
            return;
        }
590
591

        QSqlQuery query = qb.query();
592
        inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size()));
593
594

        while (query.next()) {
595
            Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD"));
596
            Part part = Part::retrieveById(query.value(0).toLongLong());
597
            const QByteArray name = ExternalPartStorage::nameForPartId(part.id());
598
            const QString partPath = ExternalPartStorage::resolveAbsolutePath(name);
599
600
            QFile f(partPath);
            if (f.exists()) {
601
                qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists";
602
603
604
605
                // That however is not a critical issue, since the part is not external,
                // so we can safely overwrite it
            }
            if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
606
                qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing";
607
608
609
                continue;
            }
            if (f.write(part.data()) != part.datasize()) {
610
                qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name;
611
612
613
614
615
                f.remove();
                continue;
            }

            part.setData(name);
616
            part.setStorage(Part::External);
617
            if (!part.update() || !transaction.commit()) {
618
                qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
619
620
621
622
                f.remove();
                continue;
            }

623
            inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name)));
624
        }
625
626
    }

627
628
629
    {
        QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
        qb.addColumn(Part::idFullColumnName());
630
        qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
631
        qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold());
632
633
634
635
        if (!qb.exec()) {
            inform("Failed to query parts smaller than treshold, skipping test");
            return;
        }
636
637

        QSqlQuery query = qb.query();
638
        inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size()));
639
640

        while (query.next()) {
641
            Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2"));
642
            Part part = Part::retrieveById(query.value(0).toLongLong());
643
            const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data());
644
645
            QFile f(partPath);
            if (!f.exists()) {
646
                qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist";
647
648
649
                continue;
            }
            if (!f.open(QIODevice::ReadOnly)) {
650
                qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading";
651
652
653
                continue;
            }

654
            part.setStorage(Part::Internal);
655
656
            part.setData(f.readAll());
            if (part.data().size() != part.datasize()) {
657
                qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match";
658
659
660
                continue;
            }
            if (!part.update() || !transaction.commit()) {
661
                qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id();
662
663
664
665
666
                continue;
            }

            f.close();
            f.remove();
667
            inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id()));
668
        }
669
670
671
    }
}

672
673
674
675
676
void StorageJanitor::migrateToLevelledCacheHierarchy()
{
    QueryBuilder qb(Part::tableName(), QueryBuilder::Select);
    qb.addColumn(Part::idColumn());
    qb.addColumn(Part::dataColumn());
677
    qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External);
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
    if (!qb.exec()) {
        inform("Failed to query external payload parts, skipping test");
        return;
    }

    QSqlQuery query = qb.query();
    while (query.next()) {
        const qint64 id = query.value(0).toLongLong();
        const QByteArray data = query.value(1).toByteArray();
        const QString fileName = QString::fromUtf8(data);
        bool oldExists = false, newExists = false;
        // Resolve the current path
        const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists);
        // Resolve the new path with legacy fallback disabled, so that it always
        // returns the new levelled-cache path, even when the old one exists
        const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false);
        if (!oldExists) {
695
            qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName;
696
697
698
699
            continue;
        }
        if (currentPath != newPath) {
            if (newExists) {
700
                qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName;
701
702
703
704
705
                continue;
            }

            QFile f(currentPath);
            if (!f.rename(newPath)) {
706
                qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString();
707
708
709
710
711
712
713
                continue;
            }
            inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id));
        }
    }
}

Andreas Hartmetz's avatar
Andreas Hartmetz committed
714
void StorageJanitor::findOrphanSearchIndexEntries()
715
716
717
718
{
    QueryBuilder qb(Collection::tableName(), QueryBuilder::Select);
    qb.addSortColumn(Collection::idColumn(), Query::Ascending);
    qb.addColumn(Collection::idColumn());
719
    qb.addColumn(Collection::isVirtualColumn());
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
    if (!qb.exec()) {
        inform("Failed to query collections, skipping test");
        return;
    }

    QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent),
                         QStringLiteral("/"),
                         QStringLiteral("org.freedesktop.Akonadi.Indexer"),
                         DBusConnectionPool::threadConnection());
    if (!iface.isValid()) {
        inform("Akonadi Indexing Agent is not running, skipping test");
        return;
    }

    QSqlQuery query = qb.query();
    while (query.next()) {
        const qint64 colId = query.value(0).toLongLong();
737
738
739
740
741
742
        // Skip virtual collections, they are not indexed
        if (query.value(1).toBool()) {
            inform(QStringLiteral("Skipping virtual Collection %1").arg(colId));
            continue;
        }

743
744
745
746
747
748
749
750
751
752
        inform(QStringLiteral("Checking Collection %1 search index...").arg(colId));
        SearchRequest req("StorageJanitor");
        req.setStoreResults(true);
        req.setCollections({ colId });
        req.setRemoteSearch(false);
        req.setQuery(QStringLiteral("{ }")); // empty query to match all
        QStringList mts;
        Collection col;
        col.setId(colId);
        const auto colMts = col.mimeTypes();
753
754
755
756
757
        if (colMts.isEmpty()) {
            // No mimetypes means we don't know which search store to look into,
            // skip it.
            continue;
        }
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
        for (const auto &mt : colMts) {
            mts << mt.name();
        }
        req.setMimeTypes(mts);
        req.exec();
        auto searchResults = req.results();

        QueryBuilder iqb(PimItem::tableName(), QueryBuilder::Select);
        iqb.addColumn(PimItem::idColumn());
        iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId);
        if (!iqb.exec()) {
            inform(QStringLiteral("Failed to query items in collection %1").arg(colId));
            continue;
        }

        QSqlQuery itemQuery = iqb.query();
        while (itemQuery.next()) {
            searchResults.remove(itemQuery.value(0).toLongLong());
        }

        if (!searchResults.isEmpty()) {
Andreas Hartmetz's avatar
Andreas Hartmetz committed
779
            inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count()));
780
781
782
783
784
785
            iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId);
        }
    }
}


786
void StorageJanitor::inform(const char *msg)
Volker Krause's avatar
Volker Krause committed
787
{
788
    inform(QLatin1String(msg));
Volker Krause's avatar
Volker Krause committed
789
790
}

791
void StorageJanitor::inform(const QString &msg)
Volker Krause's avatar
Volker Krause committed
792
{
793
    qCDebug(AKONADISERVER_LOG) << msg;
794
    Q_EMIT information(msg);
Volker Krause's avatar
Volker Krause committed
795
}