Verified Commit e8750bad authored by Linus Jahn's avatar Linus Jahn

Rewrite database models to not block the GUI

This rewrites parts of the main database class. All classes working on
the database have been moved to the new database thread, so inserting
records into the database isn't blocking the user interface anymore.
What also improved the performance *massively* is the use of SQLite
transactions, when inserting multiple records. So inserting is so fast
now that it actually wouldn't necessarily require another thread, but
with this we're safe in the future.

The message model and the roster model have been splitted up into a
database manager and the list model that is used to present cached data
to the user. The XMPP managers are connected to both the model and the
db classes, so both are updated in parallel.

There are also two new classes for the RosterItem and the Message,
because we need to cache those in the models in a vector. The Message
class inherits from QXmppMessage, so we can now use the same class for
sending messages and inserting messages into the database.

Closes #273.
parent a6ba2a0a
......@@ -5,11 +5,15 @@ set(KAIDAN_SOURCES
src/ClientWorker.cpp
src/AvatarFileStorage.cpp
src/Database.cpp
src/RosterItem.cpp
src/RosterModel.cpp
src/RosterDb.cpp
src/RosterManager.cpp
src/RegistrationManager.cpp
src/MessageHandler.cpp
src/Message.cpp
src/MessageModel.cpp
src/MessageDb.cpp
src/MessageHandler.cpp
src/Notifications.cpp
src/PresenceCache.cpp
src/DiscoveryManager.cpp
......@@ -21,9 +25,11 @@ set(KAIDAN_SOURCES
src/TransferCache.cpp
src/DownloadManager.cpp
src/QmlUtils.cpp
src/Utils.cpp
# needed to trigger moc generation
# needed to trigger moc generation / to be displayed in IDEs
src/Enums.h
src/Globals.h
# kaidan QXmpp extensions (need to be merged into QXmpp upstream)
src/qxmpp-exts/QXmppHttpUploadIq.cpp
......@@ -31,7 +37,7 @@ set(KAIDAN_SOURCES
src/qxmpp-exts/QXmppUploadManager.cpp
src/qxmpp-exts/QXmppColorGenerator.cpp
# hsluv-c required for color generation
# hsluv-c required for color generation
src/hsluv-c/hsluv.c
)
......
......@@ -80,24 +80,12 @@ ClientWorker::ClientWorker(Caches *caches, Kaidan *kaidan, bool enableLogging, Q
client->versionManager().setClientVersion(VERSION_STRING);
client->versionManager().setClientOs(QSysInfo::prettyProductName());
#if QXMPP_VERSION >= 0x000904
#if QXMPP_VERSION >= QT_VERSION_CHECK(1, 0, 0)
// Client State Indication
connect(app, &QGuiApplication::applicationStateChanged, this, &ClientWorker::setCsiState);
#endif
}
ClientWorker::~ClientWorker()
{
delete client;
delete logger;
delete rosterManager;
delete msgHandler;
delete discoManager;
delete vCardManager;
delete uploadManager;
delete downloadManager;
}
void ClientWorker::main()
{
// initialize random generator
......@@ -199,7 +187,7 @@ QString ClientWorker::generateRandomString(unsigned int length) const
return randomString;
}
#if QXMPP_VERSION >= 0x000904 // after QXmpp v0.9.4
#if QXMPP_VERSION >= QT_VERSION_CHECK(1, 0, 0)
void ClientWorker::setCsiState(Qt::ApplicationState state)
{
if (state == Qt::ApplicationActive)
......
......@@ -34,7 +34,6 @@
// Qt
#include <QObject>
#include <QTimer>
#include <QThread>
#include <QSettings>
class QGuiApplication;
// QXmpp
......@@ -61,24 +60,6 @@ class DownloadManager;
using namespace Enums;
class ClientThread : public QThread
{
Q_OBJECT
friend ClientWorker;
public:
ClientThread()
{
setObjectName("QXmppClient");
}
protected:
void run() override
{
exec();
}
};
/**
* The ClientWorker is used as a QObject-based worker on the ClientThread.
*/
......@@ -88,14 +69,16 @@ class ClientWorker : public QObject
public:
struct Caches {
Caches(Database *database, QObject *parent = nullptr)
: msgModel(new MessageModel(database->getDatabase(), parent)),
rosterModel(new RosterModel(database->getDatabase(), parent)),
Caches(Kaidan *kaidan, RosterDb *rosterDb, MessageDb *msgDb,
QObject *parent = nullptr)
: msgModel(new MessageModel(kaidan, msgDb, parent)),
rosterModel(new RosterModel(rosterDb, parent)),
avatarStorage(new AvatarFileStorage(parent)),
presCache(new PresenceCache(parent)),
transferCache(new TransferCache(parent)),
settings(new QSettings(APPLICATION_NAME, APPLICATION_NAME))
{
rosterModel->setMessageModel(msgModel);
}
~Caches()
......@@ -134,8 +117,6 @@ public:
ClientWorker(Caches *caches, Kaidan *kaidan, bool enableLogging, QGuiApplication *app,
QObject *parent = nullptr);
~ClientWorker();
public slots:
/**
* Main function of the client thread
......@@ -178,7 +159,7 @@ private slots:
*/
void onConnectionError(QXmppClient::Error error);
#if QXMPP_VERSION >= 0x000904 // after QXmpp v0.9.4
#if QXMPP_VERSION >= QT_VERSION_CHECK(1, 0, 0)
/**
* Uses the QGuiApplication state to reduce network traffic when window is minimized
*/
......
......@@ -29,142 +29,169 @@
*/
#include "Database.h"
#include "Globals.h"
#include "Utils.h"
#include <QDebug>
#include <QDir>
#include <QFile>
#include <QStandardPaths>
#include <QString>
#include <QStringList>
#include <QSqlDatabase>
#include <QSqlDriver>
#include <QSqlError>
#include <QSqlField>
#include <QSqlQuery>
#include <QSqlRecord>
#include <QStandardPaths>
#include <QString>
#include <QStringList>
static const int DATABASE_LATEST_VERSION = 10;
static const char *DATABASE_TABLE_INFO = "dbinfo";
static const char *DATABASE_TABLE_MESSAGES = "Messages";
static const char *DATABASE_TABLE_ROSTER = "Roster";
Database::Database(QObject *parent) : QObject(parent)
Database::Database(QObject *parent)
: QObject(parent)
{
version = -1;
database = QSqlDatabase::addDatabase("QSQLITE", "kaidan_default_db");
if (!database.isValid()) {
qFatal("Cannot add database: %s", qPrintable(database.lastError().text()));
}
}
Database::~Database()
{
database.close();
}
QSqlDatabase* Database::getDatabase()
{
return &database;
m_database.close();
}
void Database::openDatabase()
{
m_database = QSqlDatabase::addDatabase("QSQLITE", DB_CONNECTION);
if (!m_database.isValid())
qFatal("Cannot add database: %s", qPrintable(m_database.lastError().text()));
const QDir writeDir = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation);
if (!writeDir.mkpath(".")) {
qFatal("Failed to create writable directory at %s", qPrintable(writeDir.absolutePath()));
}
// Ensure that we have a writable location on all devices.
const QString fileName = writeDir.absoluteFilePath("messages.sqlite3");
const QString fileName = writeDir.absoluteFilePath(DB_FILENAME);
// open() will create the SQLite database if it doesn't exist.
database.setDatabaseName(fileName);
if (!database.open()) {
qFatal("Cannot open database: %s", qPrintable(database.lastError().text()));
QFile::remove(fileName);
m_database.setDatabaseName(fileName);
if (!m_database.open()) {
qFatal("Cannot open database: %s", qPrintable(m_database.lastError().text()));
}
loadDatabaseInfo();
if (needToConvert())
convertDatabase();
}
void Database::transaction()
{
if (!m_transactions) {
// currently no transactions running
if (!m_database.transaction()) {
qWarning() << "Could not begin transaction on database:"
<< m_database.lastError().text();
}
}
// increase counter
m_transactions++;
}
void Database::commit()
{
// reduce counter
m_transactions--;
Q_ASSERT(m_transactions >= 0);
if (!m_transactions) {
// no transaction requested anymore
if (!m_database.commit()) {
qWarning() << "Could not commit transaction on database:"
<< m_database.lastError().text();
}
}
}
void Database::loadDatabaseInfo()
{
QStringList tables = database.tables();
if (!tables.contains(DATABASE_TABLE_INFO)) {
if (tables.contains(DATABASE_TABLE_MESSAGES) &&
tables.contains(DATABASE_TABLE_ROSTER)) {
QStringList tables = m_database.tables();
if (!tables.contains(DB_TABLE_INFO)) {
if (tables.contains(DB_TABLE_MESSAGES) &&
tables.contains(DB_TABLE_ROSTER))
// old Kaidan v0.1/v0.2 table
version = 1;
} else {
version = 0;
}
m_version = 1;
else
m_version = 0;
// we've got all we want; do not query for a db version
return;
}
QSqlQuery query(database);
query.prepare("SELECT version FROM dbinfo");
if (!query.exec()) {
qWarning("Cannot query database info: %s", qPrintable(database.lastError().text()));
}
QSqlQuery query(m_database);
Utils::execQuery(query, "SELECT version FROM dbinfo");
QSqlRecord record = query.record();
int versionCol = record.indexOf("version");
while (query.next()) {
version = query.value(versionCol).toInt();
m_version = query.value(versionCol).toInt();
}
}
bool Database::needToConvert()
{
if (version < DATABASE_LATEST_VERSION) {
return true;
}
return false;
return m_version < DATABASE_LATEST_VERSION;
}
void Database::convertDatabase()
{
qDebug() << "[database] Converting database to latest version from version" << version;
while (version < DATABASE_LATEST_VERSION) {
switch (version) {
qDebug() << "[database] Converting database to latest version from version" << m_version;
transaction();
while (m_version < DATABASE_LATEST_VERSION) {
switch (m_version) {
case 0:
createNewDatabase(); version = DATABASE_LATEST_VERSION; break;
createNewDatabase(); m_version = DATABASE_LATEST_VERSION; break;
case 1:
convertDatabaseToV2(); version = 2; break;
convertDatabaseToV2(); m_version = 2; break;
case 2:
convertDatabaseToV3(); version = 3; break;
convertDatabaseToV3(); m_version = 3; break;
case 3:
convertDatabaseToV4(); version = 4; break;
convertDatabaseToV4(); m_version = 4; break;
case 4:
convertDatabaseToV5(); version = 5; break;
convertDatabaseToV5(); m_version = 5; break;
case 5:
convertDatabaseToV6(); version = 6; break;
convertDatabaseToV6(); m_version = 6; break;
case 6:
convertDatabaseToV7(); version = 7; break;
convertDatabaseToV7(); m_version = 7; break;
case 7:
convertDatabaseToV8(); version = 8; break;
convertDatabaseToV8(); m_version = 8; break;
case 8:
convertDatabaseToV9(); version = 9; break;
convertDatabaseToV9(); m_version = 9; break;
case 9:
convertDatabaseToV10(); version = 10; break;
convertDatabaseToV10(); m_version = 10; break;
default:
break;
}
}
QSqlQuery query(database);
query.prepare(QString("UPDATE dbinfo SET version = %1").arg(DATABASE_LATEST_VERSION));
if (!query.exec()) {
qDebug("Failed to query database: %s", qPrintable(query.lastError().text()));
}
database.commit();
version = DATABASE_LATEST_VERSION;
QSqlRecord updateRecord;
updateRecord.append(Utils::createSqlField("version", DATABASE_LATEST_VERSION));
QSqlQuery query(m_database);
Utils::execQuery(
query,
m_database.driver()->sqlStatement(
QSqlDriver::UpdateStatement,
DB_TABLE_INFO,
updateRecord,
false
)
);
commit();
m_version = DATABASE_LATEST_VERSION;
}
void Database::createNewDatabase()
{
QSqlQuery query(database);
QSqlQuery query(m_database);
//
// DB info
......@@ -223,13 +250,25 @@ void Database::createNewDatabase()
void Database::createDbInfoTable()
{
QSqlQuery query(database);
query.prepare("CREATE TABLE IF NOT EXISTS 'dbinfo' (version INTEGER NOT NULL)");
execQuery(query);
query.prepare(QString("INSERT INTO 'dbinfo' (version) VALUES (%1)")
.arg(DATABASE_LATEST_VERSION));
execQuery(query);
QSqlQuery query(m_database);
Utils::execQuery(
query,
"CREATE TABLE IF NOT EXISTS 'dbinfo' (version INTEGER NOT NULL)"
);
QSqlRecord insertRecord;
insertRecord.append(Utils::createSqlField("version", DATABASE_LATEST_VERSION));
Utils::execQuery(
query,
m_database.driver()->sqlStatement(
QSqlDriver::InsertStatement,
DB_TABLE_INFO,
insertRecord,
false
)
);
}
void Database::convertDatabaseToV2()
......@@ -240,123 +279,78 @@ void Database::convertDatabaseToV2()
void Database::convertDatabaseToV3()
{
QSqlQuery query(database);
query.prepare("ALTER TABLE Roster ADD avatarHash TEXT");
execQuery(query);
QSqlQuery query(m_database);
Utils::execQuery(query, "ALTER TABLE Roster ADD avatarHash TEXT");
}
void Database::convertDatabaseToV4()
{
QSqlQuery query(database);
QSqlQuery query(m_database);
// SQLite doesn't support the ALTER TABLE drop columns feature, so we have to use a workaround.
// we copy all rows into a back-up table (but without `avatarHash`), and then delete the old table
// and copy everything to the normal table again
query.prepare("CREATE TEMPORARY TABLE roster_backup(jid,name,lastExchanged,"
"unreadMessages,lastMessage,lastOnline,activity,status,mood);");
execQuery(query);
query.prepare("INSERT INTO roster_backup SELECT jid,name,lastExchanged,unreadMessages,"
"lastMessage,lastOnline,activity,status,mood FROM Roster;");
execQuery(query);
query.prepare("DROP TABLE Roster;");
execQuery(query);
query.prepare("CREATE TABLE Roster('jid' TEXT NOT NULL,'name' TEXT NOT NULL,"
"'lastExchanged' TEXT NOT NULL,'unreadMessages' INTEGER,'lastMessage' TEXT,"
"'lastOnline' TEXT,'activity' TEXT,'status' TEXT,'mood' TEXT);");
execQuery(query);
query.prepare("INSERT INTO Roster SELECT jid,name,lastExchanged,unreadMessages,"
"lastMessage,lastOnline,activity,status,mood FROM Roster_backup;");
execQuery(query);
query.prepare("DROP TABLE Roster_backup;");
execQuery(query);
Utils::execQuery(query, "CREATE TEMPORARY TABLE roster_backup(jid,name,lastExchanged,"
"unreadMessages,lastMessage,lastOnline,activity,status,mood);");
Utils::execQuery(query, "INSERT INTO roster_backup SELECT jid,name,lastExchanged,unreadMessages,"
"lastMessage,lastOnline,activity,status,mood FROM Roster;");
Utils::execQuery(query, "DROP TABLE Roster;");
Utils::execQuery(query, "CREATE TABLE Roster('jid' TEXT NOT NULL,'name' TEXT NOT NULL,"
"'lastExchanged' TEXT NOT NULL,'unreadMessages' INTEGER,'lastMessage' TEXT,"
"'lastOnline' TEXT,'activity' TEXT,'status' TEXT,'mood' TEXT);");
Utils::execQuery(query, "INSERT INTO Roster SELECT jid,name,lastExchanged,unreadMessages,"
"lastMessage,lastOnline,activity,status,mood FROM Roster_backup;");
Utils::execQuery(query, "DROP TABLE Roster_backup;");
}
void Database::convertDatabaseToV5()
{
QSqlQuery query(database);
query.prepare("ALTER TABLE 'Messages' ADD 'type' INTEGER");
execQuery(query);
query.prepare("UPDATE Messages SET type = 0 WHERE type IS NULL");
execQuery(query);
query.prepare("ALTER TABLE 'Messages' ADD 'mediaUrl' TEXT");
execQuery(query);
QSqlQuery query(m_database);
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'type' INTEGER");
Utils::execQuery(query, "UPDATE Messages SET type = 0 WHERE type IS NULL");
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'mediaUrl' TEXT");
}
void Database::convertDatabaseToV6()
{
QSqlQuery query(database);
QSqlQuery query(m_database);
for (QString column : {"'mediaSize' INTEGER", "'mediaContentType' TEXT",
"'mediaLastModified' INTEGER", "'mediaLocation' TEXT"}) {
query.prepare(QString("ALTER TABLE 'Messages' ADD ").append(column));
execQuery(query);
Utils::execQuery(query, QString("ALTER TABLE 'Messages' ADD ").append(column));
}
}
void Database::convertDatabaseToV7()
{
QSqlQuery query(database);
query.prepare(QString("ALTER TABLE 'Messages' ADD 'mediaThumb' BLOB"));
execQuery(query);
query.prepare(QString("ALTER TABLE 'Messages' ADD 'mediaHashes' TEXT"));
execQuery(query);
QSqlQuery query(m_database);
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'mediaThumb' BLOB");
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'mediaHashes' TEXT");
}
void Database::convertDatabaseToV8()
{
QSqlQuery query(database);
query.prepare("CREATE TEMPORARY TABLE roster_backup(jid, name, lastExchanged, "
"unreadMessages, lastMessage);");
execQuery(query);
query.prepare("INSERT INTO roster_backup SELECT jid, name, lastExchanged, unreadMessages, "
"lastMessage FROM Roster;");
execQuery(query);
query.prepare("DROP TABLE Roster;");
execQuery(query);
query.prepare("CREATE TABLE IF NOT EXISTS Roster ('jid' TEXT NOT NULL,'name' TEXT,"
"'lastExchanged' TEXT NOT NULL, 'unreadMessages' INTEGER,"
"'lastMessage' TEXT);");
execQuery(query);
query.prepare("INSERT INTO Roster SELECT jid, name, lastExchanged, unreadMessages, "
"lastMessage FROM Roster_backup;");
execQuery(query);
query.prepare("DROP TABLE roster_backup;");
execQuery(query);
QSqlQuery query(m_database);
Utils::execQuery(query, "CREATE TEMPORARY TABLE roster_backup(jid, name, lastExchanged, "
"unreadMessages, lastMessage);");
Utils::execQuery(query, "INSERT INTO roster_backup SELECT jid, name, lastExchanged, unreadMessages, "
"lastMessage FROM Roster;");
Utils::execQuery(query, "DROP TABLE Roster;");
Utils::execQuery(query, "CREATE TABLE IF NOT EXISTS Roster ('jid' TEXT NOT NULL,'name' TEXT,"
"'lastExchanged' TEXT NOT NULL, 'unreadMessages' INTEGER,"
"'lastMessage' TEXT);");
Utils::execQuery(query, "INSERT INTO Roster SELECT jid, name, lastExchanged, unreadMessages, "
"lastMessage FROM Roster_backup;");
Utils::execQuery(query, "DROP TABLE roster_backup;");
}
void Database::convertDatabaseToV9()
{
QSqlQuery query(database);
query.prepare("ALTER TABLE 'Messages' ADD 'edited' BOOL");
execQuery(query);
QSqlQuery query(m_database);
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'edited' BOOL");
}
void Database::convertDatabaseToV10()
{
QSqlQuery query(database);
query.prepare("ALTER TABLE 'Messages' ADD 'isSpoiler' BOOL");
execQuery(query);
query.prepare("ALTER TABLE 'Messages' ADD 'spoilerHint' TEXT");
execQuery(query);
}
void Database::execQuery(QSqlQuery &query)
{
if (!query.exec()) {
qDebug() << query.executedQuery();
qFatal("Failed to query database: %s", qPrintable(query.lastError().text()));
}
QSqlQuery query(m_database);
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'isSpoiler' BOOL");
Utils::execQuery(query, "ALTER TABLE 'Messages' ADD 'spoilerHint' TEXT");
}
......@@ -36,6 +36,10 @@
class QSqlQuery;
/**
* The Database class manages the SQL database. It opens the database and converts old
* formats.
*/
class Database : public QObject
{
Q_OBJECT
......@@ -44,15 +48,52 @@ public:
Database(QObject *parent = nullptr);
~Database();
QSqlDatabase* getDatabase();
bool needToConvert();
void convertDatabase();
/**
* Opens the database for reading and writing and guarantees the database to be
* up-to-date.
*/
void openDatabase();
/**
* Begins a transaction if none has been started.
*/
void transaction();
/**
* Commits the transaction if every transaction has been finished.
*/
void commit();
private:
/**
* @return true if the database has to be converted using @c convertDatabase()
* because the database is not up-to-date.
*/
bool needToConvert();
/**
* Converts the database to latest model.
*/
void convertDatabase();
/**
* Loads the database information and detects the database version.
*/
void loadDatabaseInfo();
/**
* Creates the database information table which contains the database version.
*/
void createDbInfoTable();
/**
* Creates a new database without content.
*/
void createNewDatabase();
/*
* Upgrades the database to the next version.
*/
void convertDatabaseToV2();
void convertDatabaseToV3();
void convertDatabaseToV4();
......@@ -62,10 +103,18 @@ private:
void convertDatabaseToV8();
void convertDatabaseToV9();
void convertDatabaseToV10();
void execQuery(QSqlQuery &query);
QSqlDatabase database;
int version;
QSqlDatabase m_database;
/**
* -1 : Database not loaded.
* 0 : Database not existent.
* 1 : Old database before Kaidan v0.3.
* > 1 : Database version.
*/
int m_version = -1;
int m_transactions = 0;
};
#endif // DATABASE_H
......@@ -34,6 +34,8 @@
#include "TransferCache.h"
#include "MessageModel.h"
#include "Globals.h"
// C++
#include <utility>
// Qt
#include "QDir"
#include "QStandardPaths"
......@@ -64,7 +66,7 @@ DownloadManager::~DownloadManager()
delete thread;
}
void DownloadManager::startDownload(const QString msgId, const QString url)
void DownloadManager::startDownload(const QString &msgId, const QString &url)
{
// don't download the same file twice and in parallel
if (downloads.keys().contains(msgId)) {
......@@ -83,35 +85,43 @@ void DownloadManager::startDownload(const QString msgId, const QString url)
dl->moveToThread(thread);
downloads[msgId] = dl;
connect(dl, &DownloadJob::finished, this, [this, dl, msgId]() {
MessageModel::Message msgUpdate;
msgUpdate.mediaLocation = dl->downloadLocation();
emit model->updateMessageRequested(msgId, msgUpdate);
connect(dl, &DownloadJob::finished, this, [=]() {
const QString mediaLocation = dl->downloadLocation();
emit model->updateMessageRequested(msgId, [=] (Message &msg) {
msg.setMediaLocation(mediaLocation);
});
abortDownload(msgId);
});
connect(dl, &DownloadJob::failed, this, [this, msgId]() {
connect(dl, &DownloadJob::failed, this, [=]() {
abortDownload(msgId);
});
emit dl->startDownloadRequested();
}
void DownloadManager::abortDownload(const QString msgId)
void DownloadManager::abortDownload(const QString &msgId)
{
DownloadJob *job = downloads.value(msgId);
if (job != nullptr)
delete job;
delete job;
downloads.remove(msgId);
emit transferCache->removeJobRequested(msgId);
}
DownloadJob::DownloadJob(QString msgId, QUrl source, QString filePath,
DownloadJob::DownloadJob(QString msgId,
QUrl source,
QString filePath,
QNetworkAccessManager *netMngr,
TransferCache *transferCache, Kaidan *kaidan)
: QObject(nullptr), msgId(msgId), source(source), filePath(filePath),
netMngr(netMngr), transferCache(transferCache), kaidan(kaidan), file()
TransferCache *transferCache,
Kaidan *kaidan)
: QObject(nullptr),
msgId(std::move(msgId)),
source(std::move(source)),
filePath(std::move(filePath)),
netMngr(netMngr),
transferCache(transferCache),
kaidan(kaidan)
{
connect(this, &DownloadJob::startDownloadRequested,
this, &DownloadJob::startDownload);
......@@ -149,19 +159,19 @@ void DownloadJob::startDownload()
this, [this] (qint64 bytesReceived, qint64 bytesTotal) {
emit transferCache->setJobProgressRequested(msgId, bytesReceived, bytesTotal);
});
connect(reply, &QNetworkReply::finished, this, [this] () {
connect(reply, &QNetworkReply::finished, this, [=] () {
emit transferCache->removeJobRequested(msgId);