Commit 788f9eca authored by Ivan Čukić's avatar Ivan Čukić 👁
Browse files

Resources database backups

More and more things depend on KAMD's resources database.
Recent documents, favourites, KRunner, to name a few.

Now, SQLite is fairly stable with the write-ahead-log that
has been enabled in KAMD for ages, but corruptions can
still happen. Sometimes in SQLite, sometimes in the underlying
file system.

This patch introduces a two-step backup system for the
resources database. On each start, the current database
files are copied to `test-backup` directory.

If no error appeared during the previous KAMD run, the
`test-backup` files are promoted to `working-backup`.

This means that in order for a specific version of the
database to be considered as 'working' (promoted to a
working backup), it needs to have been a beginning of
an error-free KAMD session.
parent 9b2d5e4e
......@@ -110,6 +110,11 @@ public:
qCDebug(KAMD_LOG_RESOURCES) << "Closing SQL connection: " << m_connectionName;
}
void close()
{
m_database.close();
}
QSqlDatabase &get()
{
return m_database;
......@@ -207,6 +212,9 @@ Database::Ptr Database::instance(Source source, OpenMode openMode)
qCWarning(KAMD_LOG_RESOURCES) << "KActivities: Database can not be opened in WAL mode. Check the "
"SQLite version (required >3.7.0). And whether your filesystem "
"supports shared memory";
ptr->d->database->close();
return nullptr;
}
......@@ -237,6 +245,11 @@ QSqlQuery Database::createQuery() const
return d->query();
}
void Database::reportError(const QSqlError &error_)
{
Q_EMIT error(error_);
}
QString Database::lastQuery() const
{
#ifdef QT_DEBUG
......@@ -249,7 +262,13 @@ QSqlQuery Database::execQuery(const QString &query, bool ignoreErrors) const
{
Q_UNUSED(ignoreErrors);
#ifdef QT_NO_DEBUG
return d->query(query);
auto result = d->query(query);
if (!ignoreErrors && result.lastError().isValid()) {
Q_EMIT error(result.lastError());
}
return result;
#else
auto result = d->query(query);
......
......@@ -25,10 +25,13 @@
#include <memory>
#include <QSqlQuery>
#include <QRegExp>
#include <QObject>
namespace Common {
class Database {
class Database: public QObject {
Q_OBJECT
public:
typedef std::shared_ptr<Database> Ptr;
......@@ -67,10 +70,15 @@ public:
QSqlDatabase &m_database;
};
void reportError(const QSqlError &error);
#define DATABASE_TRANSACTION(A) \
/* enable this for debugging only: qCDebug(KAMD_LOG_RESOURCES) << "Location:" << __FILE__ << __LINE__; */ \
Common::Database::Locker lock(A)
Q_SIGNALS:
void error(const QSqlError &error) const;
private:
D_PTR;
};
......
......@@ -170,6 +170,18 @@ void initSchema(Database &database)
database.execQueries(ResourcesDatabaseSchema::schema());
// We are asking for trouble. If the database is corrupt,
// some of these should fail.
// WARNING: Sqlite specific!
database.execQueries(QStringList{
".tables",
"SELECT count(*) FROM SchemaInfo",
"SELECT count(*) FROM ResourceEvent",
"SELECT count(*) FROM ResourceScoreCache",
"SELECT count(*) FROM ResourceLink",
"SELECT count(*) FROM ResourceInfo"
});
// We can not allow empty fields for activity and agent, they need to
// be at least magic values. These do not change the structure
// of the database, but the old data.
......
......@@ -47,7 +47,7 @@
#include <common/database/Database.h>
#include <common/database/schema/ResourcesDatabaseSchema.h>
class ResourcesDatabaseMigrator::Private {
class ResourcesDatabaseInitializer::Private {
public:
Common::Database::Ptr database;
......@@ -55,33 +55,175 @@ public:
Common::Database::Ptr resourcesDatabase()
{
static ResourcesDatabaseMigrator instance;
static ResourcesDatabaseInitializer instance;
return instance.d->database;
}
ResourcesDatabaseMigrator::ResourcesDatabaseMigrator()
void ResourcesDatabaseInitializer::initDatabase(bool retryOnFail)
{
const QString databaseDir
//
// There are three situations we want to handle:
// 1. The database can not be opened at all.
// This means that the current database files have
// been corrupted and that we need to replace them
// with the last working backup.
// 2. The database was opened, but an error appeared
// somewhere at runtime.
// 3. The database was successfully opened and no errors
// appeared during runtime.
//
// To achieve this, we will have three locations for
// database files:
//
// 1. `resources` - the current database files
// 2. `resources-test-backup` - at each KAMD start,
// we copy the current database files here.
// If an error appears during execution, the files
// will be removed and the error will be added to
// the log file `resources/errors.log`
// 3. `resources-working-backup` - on each KAMD start,
// if there are files in `resources-test-backup`
// (meaning no error appeared at runtime), they
// will be copied to `resources-working-backup`.
//
// This means that the `working` backup will be a bit
// older, but it will be the last database that produced
// no errors at runtime.
//
const QString databaseDirectoryPath
= QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation)
+ QStringLiteral("/kactivitymanagerd/resources/");
qCDebug(KAMD_LOG_RESOURCES) << "Creating directory: " << databaseDir;
auto created = QDir().mkpath(databaseDir);
const QString databaseTestBackupDirectoryPath
= QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation)
+ QStringLiteral("/kactivitymanagerd/resources/test-backup/");
const QString databaseWorkingBackupDirectoryPath
= QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation)
+ QStringLiteral("/kactivitymanagerd/resources/working-backup/");
const QStringList databaseFiles{"database", "database-wal", "database-shm"};
{
QDir dir;
dir.mkpath(databaseDirectoryPath);
dir.mkpath(databaseTestBackupDirectoryPath);
dir.mkpath(databaseWorkingBackupDirectoryPath);
if (!dir.exists(databaseDirectoryPath) ||
!dir.exists(databaseTestBackupDirectoryPath) ||
!dir.exists(databaseWorkingBackupDirectoryPath)) {
qCWarning(KAMD_LOG_RESOURCES) << "Database directory can not be created!";
return;
}
}
if (!created || !QDir(databaseDir).exists()) {
qCWarning(KAMD_LOG_RESOURCES) << "Database folder can not be created!";
const QDir databaseDirectory(databaseDirectoryPath);
const QDir databaseTestBackupDirectory(databaseTestBackupDirectoryPath);
const QDir databaseWorkingBackupDirectory(databaseWorkingBackupDirectoryPath);
auto removeDatabaseFiles = [&] (const QDir &dir) {
return std::all_of(databaseFiles.begin(), databaseFiles.cend(),
[&] (const QString &fileName) {
const auto filePath = dir.filePath(fileName);
return !QFile::exists(filePath) || QFile::remove(filePath);
});
};
auto copyDatabaseFiles = [&] (const QDir &fromDir, const QDir& toDir) {
return removeDatabaseFiles(toDir) &&
std::all_of(databaseFiles.begin(), databaseFiles.cend(),
[&] (const QString &fileName) {
const auto fromFilePath = fromDir.filePath(fileName);
const auto toFilePath = toDir.filePath(fileName);
return QFile::copy(fromFilePath, toFilePath);
});
};
auto databaseFilesExistIn = [&] (const QDir &dir) {
return dir.exists() &&
std::all_of(databaseFiles.begin(), databaseFiles.cend(),
[&] (const QString &fileName) {
const auto filePath = dir.filePath(fileName);
return QFile::exists(filePath);
});
};
// First, let's move the files from `resources-test-backup` to
// `resources-working-backup` (if they exist)
if (databaseFilesExistIn(databaseTestBackupDirectory)) {
qCDebug(KAMD_LOG_RESOURCES) << "Marking the test backup as working...";
if (copyDatabaseFiles(databaseTestBackupDirectory, databaseWorkingBackupDirectory)) {
removeDatabaseFiles(databaseTestBackupDirectory);
} else {
qCWarning(KAMD_LOG_RESOURCES) << "Marking the test backup as working failed!";
removeDatabaseFiles(databaseWorkingBackupDirectory);
}
}
// Next, copy the current database files to `resources-test-backup`
if (databaseFilesExistIn(databaseDirectory)) {
qCDebug(KAMD_LOG_RESOURCES) << "Creating the backup of the current database...";
if (!copyDatabaseFiles(databaseDirectory, databaseTestBackupDirectory)) {
qCWarning(KAMD_LOG_RESOURCES) << "Creating the backup of the current database failed!";
removeDatabaseFiles(databaseTestBackupDirectory);
}
}
// Now we can try to open the database
d->database = Common::Database::instance(
Common::Database::ResourcesDatabase,
Common::Database::ReadWrite);
if (d->database) {
qCDebug(KAMD_LOG_RESOURCES) << "Database opened successfully";
QObject::connect(d->database.get(), &Common::Database::error,
[=] (const QSqlError &error) {
const QString errorLog =
QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation)
+ QStringLiteral("/kactivitymanagerd/resources/errors.log");
QFile file(errorLog);
if (file.open(QIODevice::Append)) {
QTextStream out(&file);
out << QDateTime::currentDateTime().toString(Qt::ISODate) << " error: " << error.text() << "\n";
} else {
qCWarning(KAMD_LOG_RESOURCES) << QDateTime::currentDateTime().toString(Qt::ISODate) << " error: " << error.text();
}
removeDatabaseFiles(databaseTestBackupDirectory);
});
Common::ResourcesDatabaseSchema::initSchema(*d->database);
} else {
// The current database can not be opened, delete the
// backup we just created
removeDatabaseFiles(databaseTestBackupDirectory);
if (databaseFilesExistIn(databaseWorkingBackupDirectoryPath)) {
qCWarning(KAMD_LOG_RESOURCES) << "The database seems to be corrupted, trying to load the latest working version";
const auto success = copyDatabaseFiles(databaseWorkingBackupDirectory, databaseDirectory);
if (success && retryOnFail) {
// Avoid infinite recursion
initDatabase(false);
}
} else {
qCWarning(KAMD_LOG_RESOURCES) << "The database might be corrupted and there is no working backup";
}
}
}
ResourcesDatabaseMigrator::~ResourcesDatabaseMigrator()
ResourcesDatabaseInitializer::ResourcesDatabaseInitializer()
{
initDatabase(true);
}
ResourcesDatabaseInitializer::~ResourcesDatabaseInitializer()
{
}
......@@ -41,17 +41,15 @@ namespace Common {
class Database;
} // namespace Common
class ResourcesDatabaseMigrator : public QObject {
Q_OBJECT
class ResourcesDatabaseInitializer {
public:
// static Database *self();
private:
ResourcesDatabaseMigrator();
~ResourcesDatabaseMigrator() override;
ResourcesDatabaseInitializer();
~ResourcesDatabaseInitializer();
void migrateDatabase(const QString &newDatabaseFile) const;
void initDatabase(bool retryOnFail = true);
D_PTR;
......
......@@ -100,7 +100,7 @@ void ResourceLinking::LinkResourceToActivity(QString initiatingAgent,
DATABASE_TRANSACTION(*resourcesDatabase());
Utils::exec(Utils::FailOnError, *linkResourceToActivityQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *linkResourceToActivityQuery,
":usedActivity" , usedActivity,
":initiatingAgent" , initiatingAgent,
":targettedResource" , targettedResource
......@@ -169,7 +169,7 @@ void ResourceLinking::UnlinkResourceFromActivity(QString initiatingAgent,
DATABASE_TRANSACTION(*resourcesDatabase());
Utils::exec(Utils::FailOnError, *query,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *query,
":usedActivity" , usedActivity,
":initiatingAgent" , initiatingAgent,
":targettedResource" , targettedResource
......@@ -222,7 +222,7 @@ bool ResourceLinking::IsResourceLinkedToActivity(QString initiatingAgent,
"targettedResource = COALESCE(:targettedResource, '') "
));
Utils::exec(Utils::FailOnError, *isResourceLinkedToActivityQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *isResourceLinkedToActivityQuery,
":usedActivity" , usedActivity,
":initiatingAgent" , initiatingAgent,
":targettedResource" , targettedResource
......
......@@ -158,7 +158,7 @@ void ResourceScoreCache::update()
qCDebug(KAMD_LOG_RESOURCES) << "Creating the cache for: " << d->resource;
// This can fail if we have the cache already made
auto isCacheNew = Utils::exec(
auto isCacheNew = Utils::exec(*resourcesDatabase(),
Utils::IgnoreError, Queries::self().createResourceScoreCacheQuery,
":usedActivity", d->activity,
":initiatingAgent", d->application,
......@@ -167,7 +167,7 @@ void ResourceScoreCache::update()
);
// Getting the old score
Utils::exec(
Utils::exec(*resourcesDatabase(),
Utils::FailOnError, Queries::self().getResourceScoreCacheQuery,
":usedActivity", d->activity,
":initiatingAgent", d->application,
......@@ -205,7 +205,7 @@ void ResourceScoreCache::update()
qCDebug(KAMD_LOG_RESOURCES) << " First update : " << firstUpdate;
qCDebug(KAMD_LOG_RESOURCES) << " Last update : " << lastUpdate;
Utils::exec(Utils::FailOnError, Queries::self().getScoreAdditionQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, Queries::self().getScoreAdditionQuery,
":usedActivity", d->activity,
":initiatingAgent", d->application,
":targettedResource", d->resource,
......@@ -236,7 +236,7 @@ void ResourceScoreCache::update()
// Updating the score
Utils::exec(Utils::FailOnError, Queries::self().updateResourceScoreCacheQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, Queries::self().updateResourceScoreCacheQuery,
":usedActivity", d->activity,
":initiatingAgent", d->application,
":targettedResource", d->resource,
......
......@@ -171,7 +171,7 @@ void StatsPlugin::openResourceEvent(const QString &usedActivity,
"VALUES (:usedActivity, :initiatingAgent, :targettedResource, :start, :end)"
));
Utils::exec(Utils::FailOnError, *openResourceEventQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *openResourceEventQuery,
":usedActivity" , usedActivity ,
":initiatingAgent" , initiatingAgent ,
":targettedResource" , targettedResource ,
......@@ -205,7 +205,7 @@ void StatsPlugin::closeResourceEvent(const QString &usedActivity,
"end IS NULL"
));
Utils::exec(Utils::FailOnError, *closeResourceEventQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *closeResourceEventQuery,
":usedActivity" , usedActivity ,
":initiatingAgent" , initiatingAgent ,
":targettedResource" , targettedResource ,
......@@ -242,7 +242,7 @@ bool StatsPlugin::insertResourceInfo(const QString &uri)
));
getResourceInfoQuery->bindValue(":targettedResource", uri);
Utils::exec(Utils::FailOnError, *getResourceInfoQuery);
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *getResourceInfoQuery);
if (getResourceInfoQuery->next()) {
return false;
......@@ -264,7 +264,7 @@ bool StatsPlugin::insertResourceInfo(const QString &uri)
")"
));
Utils::exec(Utils::FailOnError, *insertResourceInfoQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *insertResourceInfoQuery,
":targettedResource", uri
);
......@@ -286,7 +286,7 @@ void StatsPlugin::saveResourceTitle(const QString &uri, const QString &title,
"targettedResource = :targettedResource "
));
Utils::exec(Utils::FailOnError, *saveResourceTitleQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *saveResourceTitleQuery,
":targettedResource" , uri ,
":title" , title ,
":autoTitle" , (autoTitle ? "1" : "0")
......@@ -309,7 +309,7 @@ void StatsPlugin::saveResourceMimetype(const QString &uri,
"targettedResource = :targettedResource "
));
Utils::exec(Utils::FailOnError, *saveResourceMimetypeQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, *saveResourceMimetypeQuery,
":targettedResource" , uri ,
":mimetype" , mimetype ,
":autoMimetype" , (autoMimetype ? "1" : "0")
......@@ -461,8 +461,8 @@ void StatsPlugin::DeleteRecentStats(const QString &activity, int count,
"DELETE FROM ResourceScoreCache "
"WHERE usedActivity = COALESCE(:usedActivity, usedActivity)");
Utils::exec(Utils::FailOnError, removeEventsQuery, ":usedActivity", usedActivity);
Utils::exec(Utils::FailOnError, removeScoreCachesQuery, ":usedActivity", usedActivity);
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeEventsQuery, ":usedActivity", usedActivity);
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeScoreCachesQuery, ":usedActivity", usedActivity);
} else {
......@@ -493,12 +493,12 @@ void StatsPlugin::DeleteRecentStats(const QString &activity, int count,
"WHERE usedActivity = COALESCE(:usedActivity, usedActivity) "
"AND firstUpdate > :since");
Utils::exec(Utils::FailOnError, removeEventsQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeEventsQuery,
":usedActivity", usedActivity,
":since", since.toSecsSinceEpoch()
);
Utils::exec(Utils::FailOnError, removeScoreCachesQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeScoreCachesQuery,
":usedActivity", usedActivity,
":since", since.toSecsSinceEpoch()
);
......@@ -534,12 +534,12 @@ void StatsPlugin::DeleteEarlierStats(const QString &activity, int months)
"WHERE usedActivity = COALESCE(:usedActivity, usedActivity) "
"AND lastUpdate < :time");
Utils::exec(Utils::FailOnError, removeEventsQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeEventsQuery,
":usedActivity", usedActivity,
":time", time.toSecsSinceEpoch()
);
Utils::exec(Utils::FailOnError, removeScoreCachesQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeScoreCachesQuery,
":usedActivity", usedActivity,
":time", time.toSecsSinceEpoch()
);
......@@ -600,10 +600,10 @@ void StatsPlugin::DeleteStatsForResource(const QString &activity,
const auto pattern = Common::starPatternToLike(resource);
Utils::exec(Utils::FailOnError, removeEventsQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeEventsQuery,
":targettedResource", pattern);
Utils::exec(Utils::FailOnError, removeScoreCachesQuery,
Utils::exec(*resourcesDatabase(), Utils::FailOnError, removeScoreCachesQuery,
":targettedResource", pattern);
emit ResourceScoreDeleted(activity, client, resource);
......
......@@ -21,6 +21,7 @@
// Qt
#include <QObject>
#include <QTimer>
#include <QSqlQuery>
// Boost and STL
#include <memory>
......@@ -29,7 +30,6 @@
// Local
#include <Plugin.h>
class QSqlQuery;
class ResourceLinking;
/**
......
......@@ -21,6 +21,7 @@
#define PLUGINS_SQLITE_DATABASE_UTILS_H
#include <QSqlQuery>
#include <QSqlError>
#include <common/database/schema/ResourcesDatabaseSchema.h>
#include <memory>
......@@ -57,7 +58,7 @@ namespace Utils {
FailOnError
};
inline bool exec(ErrorHandling eh, QSqlQuery &query)
inline bool exec(Common::Database &database, ErrorHandling eh, QSqlQuery &query)
{
bool success = query.exec();
......@@ -67,18 +68,22 @@ namespace Utils {
qCWarning(KAMD_LOG_RESOURCES) << query.lastError();
}
Q_ASSERT_X(success, "Uils::exec", qPrintable(QStringLiteral("Query failed:") + query.lastError().text()));
if (!success) {
database.reportError(query.lastError());
}
}
return success;
}
template <typename T1, typename T2, typename... Ts>
inline bool exec(ErrorHandling eh, QSqlQuery &query,
inline bool exec(Common::Database &database, ErrorHandling eh, QSqlQuery &query,
const T1 &variable, const T2 &value, Ts... ts)
{
query.bindValue(variable, value);
return exec(eh, query, ts...);
return exec(database, eh, query, ts...);
}
} // namespace Utils
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment