Commit de7eea95 authored by Daniel Vrátil's avatar Daniel Vrátil 🤖

CollectionStatistics: prefetch all stats on start

When Akonadi is started each application and agent starts requesting
its Collections and statistics. This causes the CollectionStatistics
cache to query the stats for each Collection individually with the
very expensive query. On my system all the statistics queries together
take over 17 seconds.

With this change the CollectionStatistics cache simply pre-fetches
statistics for all Collections in single a query, which is MUCH
faster (less than a second on my system), because the expensive JOINs
only have to be done once.

As a result of this change applications and agents become responsive
much faster after start and CPU and IO load during start is massively
decreased.
parent 94fdc1d4
......@@ -26,12 +26,13 @@
using namespace Akonadi::Server;
Q_DECLARE_METATYPE(Akonadi::Server::Collection)
class IntrospectableCollectionStatistics : public CollectionStatistics
{
public:
IntrospectableCollectionStatistics()
: CollectionStatistics()
IntrospectableCollectionStatistics(bool prefetch)
: CollectionStatistics(prefetch)
, mCalculationsCount(0)
{}
~IntrospectableCollectionStatistics()
......@@ -61,6 +62,8 @@ class CollectionStatisticsTest : public QObject
public:
CollectionStatisticsTest()
{
qRegisterMetaType<Collection>();
try {
FakeAkonadiServer::instance()->setPopulateDb(false);
FakeAkonadiServer::instance()->init();
......@@ -76,6 +79,45 @@ public:
}
private Q_SLOTS:
void testPrefetch_data()
{
DbInitializer initializer;
initializer.createResource("testresource");
auto col1 = initializer.createCollection("col1");
initializer.createItem("item1", col1);
initializer.createItem("item2", col1);
auto col2 = initializer.createCollection("col2");
// empty
auto col3 = initializer.createCollection("col3");
initializer.createItem("item3", col3);
QTest::addColumn<Collection>("collection");
QTest::addColumn<int>("calculationsCount");
QTest::addColumn<qint64>("count");
QTest::addColumn<qint64>("read");
QTest::addColumn<qint64>("size");
QTest::newRow("col1") << col1 << 0 << 2ll << 0ll << 0ll;
QTest::newRow("col2") << col2 << 0 << 0ll << 0ll << 0ll;
QTest::newRow("col3") << col3 << 0 << 1ll << 0ll << 0ll;
}
void testPrefetch()
{
QFETCH(Collection, collection);
QFETCH(int, calculationsCount);
QFETCH(qint64, count);
QFETCH(qint64, read);
QFETCH(qint64, size);
IntrospectableCollectionStatistics cs(true);
auto stats = cs.statistics(collection);
QCOMPARE(cs.calculationsCount(), calculationsCount);
QCOMPARE(stats.count, count);
QCOMPARE(stats.read, read);
QCOMPARE(stats.size, size);
}
void testCalculateStats()
{
DbInitializer initializer;
......@@ -85,7 +127,7 @@ private Q_SLOTS:
initializer.createItem("item2", col);
initializer.createItem("item3", col);
IntrospectableCollectionStatistics cs;
IntrospectableCollectionStatistics cs(false);
auto stats = cs.statistics(col);
QCOMPARE(cs.calculationsCount(), 1);
QCOMPARE(stats.count, 3);
......@@ -102,7 +144,7 @@ private Q_SLOTS:
initializer.createItem("item2", col);
initializer.createItem("item3", col);
IntrospectableCollectionStatistics cs;
IntrospectableCollectionStatistics cs(false);
auto stats = cs.statistics(col);
QCOMPARE(cs.calculationsCount(), 1);
QCOMPARE(stats.count, 3);
......@@ -131,7 +173,7 @@ void testItemAdded()
auto col = initializer.createCollection("col1");
initializer.createItem("item1", col);
IntrospectableCollectionStatistics cs;
IntrospectableCollectionStatistics cs(false);
auto stats = cs.statistics(col);
QCOMPARE(cs.calculationsCount(), 1);
QCOMPARE(stats.count, 1);
......
......@@ -46,6 +46,64 @@ void CollectionStatistics::destroy()
sInstance = nullptr;
}
CollectionStatistics::CollectionStatistics(bool prefetch)
{
if (prefetch) {
QMutexLocker lock(&mCacheLock);
QList<QueryBuilder> builders;
// This single query will give us statistics for all non-empty non-virtual
// Collections at much better speed than individual queries.
auto qb = prepareGenericQuery();
qb.addColumn(PimItem::collectionIdFullColumnName());
qb.addGroupColumn(PimItem::collectionIdFullColumnName());
builders << qb;
// This single query will give us statistics for all non-empty virtual
// Collections
qb = prepareGenericQuery();
qb.addColumn(CollectionPimItemRelation::leftFullColumnName());
qb.addJoin(QueryBuilder::InnerJoin, CollectionPimItemRelation::tableName(),
CollectionPimItemRelation::rightFullColumnName(), PimItem::idFullColumnName());
qb.addGroupColumn(CollectionPimItemRelation::leftFullColumnName());
builders << qb;
for (auto &qb : builders) {
if (!qb.exec()) {
return;
}
auto query = qb.query();
while (query.next()) {
mCache.insert(query.value(3).toLongLong(),
{ query.value(0).toLongLong(),
query.value(1).toLongLong(),
query.value(2).toLongLong()
});
}
}
// Now quickly get all non-virtual enabled Collections and if they are
// not in mCache yet, insert them with empty statistics.
qb = QueryBuilder(Collection::tableName());
qb.addColumn(Collection::idColumn());
qb.addValueCondition(Collection::enabledColumn(), Query::Equals, true);
qb.addValueCondition(Collection::isVirtualColumn(), Query::Equals, false);
if (!qb.exec()) {
return;
}
auto query = qb.query();
while (query.next()) {
const auto colId = query.value(0).toLongLong();
const auto isVirtual = query.value(1).toBool();
if (!isVirtual && !mCache.contains(colId)) {
mCache.insert(colId, { 0, 0, 0 });
}
}
}
}
void CollectionStatistics::itemAdded(const Collection &col, qint64 size, bool seen)
{
if (!col.isValid()) {
......@@ -104,12 +162,12 @@ const CollectionStatistics::Statistics CollectionStatistics::statistics(const Co
return it.value();
}
CollectionStatistics::Statistics CollectionStatistics::calculateCollectionStatistics(const Collection &col)
QueryBuilder CollectionStatistics::prepareGenericQuery()
{
static const QString SeenFlagsTableName = QStringLiteral("SeenFlags");
static const QString IgnoredFlagsTableName = QStringLiteral("IgnoredFlags");
#define FLAGS_COLUMN(table, column) \
#define FLAGS_COLUMN(table, column) \
QStringLiteral("%1.%2").arg(table##TableName, PimItemFlagRelation::column())
// COUNT(DISTINCT PimItemTable.id)
......@@ -153,7 +211,14 @@ CollectionStatistics::Statistics CollectionStatistics::calculateCollectionStatis
ignoredCondition);
}
#undef FLAGS_COLUMN
#undef FLAGS_COLUMN
return qb;
}
CollectionStatistics::Statistics CollectionStatistics::calculateCollectionStatistics(const Collection &col)
{
auto qb = prepareGenericQuery();
if (col.isVirtual()) {
qb.addJoin(QueryBuilder::InnerJoin, CollectionPimItemRelation::tableName(),
......
......@@ -30,6 +30,7 @@ namespace Akonadi
namespace Server
{
class QueryBuilder;
class Collection;
/**
......@@ -67,6 +68,9 @@ public:
void expireCache();
protected:
explicit CollectionStatistics(bool prefetch = true);
QueryBuilder prepareGenericQuery();
virtual Statistics calculateCollectionStatistics(const Collection &col);
QMutex mCacheLock;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment