1 /***************************************************************************
2 * Copyright (C) 2005-2020 by the Quassel Project *
3 * devel@quassel-irc.org *
5 * This program is free software; you can redistribute it and/or modify *
6 * it under the terms of the GNU General Public License as published by *
7 * the Free Software Foundation; either version 2 of the License, or *
8 * (at your option) version 3. *
10 * This program is distributed in the hope that it will be useful, *
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13 * GNU General Public License for more details. *
15 * You should have received a copy of the GNU General Public License *
16 * along with this program; if not, write to the *
17 * Free Software Foundation, Inc., *
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. *
19 ***************************************************************************/
21 #include "abstractsqlstorage.h"
25 #include <QMutexLocker>
34 int AbstractSqlStorage::_nextConnectionId = 0;
35 AbstractSqlStorage::AbstractSqlStorage(QObject* parent)
39 AbstractSqlStorage::~AbstractSqlStorage()
41 // disconnect the connections, so their deletion is no longer interessting for us
42 QHash<QThread*, Connection*>::iterator conIter;
43 for (conIter = _connectionPool.begin(); conIter != _connectionPool.end(); ++conIter) {
44 QSqlDatabase::removeDatabase(conIter.value()->name());
45 disconnect(conIter.value(), nullptr, this, nullptr);
49 QSqlDatabase AbstractSqlStorage::logDb()
51 if (!_connectionPool.contains(QThread::currentThread()))
52 addConnectionToPool();
54 QSqlDatabase db = QSqlDatabase::database(_connectionPool[QThread::currentThread()]->name(), false);
57 qWarning() << "Database connection" << displayName() << "for thread" << QThread::currentThread()
58 << "was lost, attempting to reconnect...";
65 void AbstractSqlStorage::addConnectionToPool()
67 QMutexLocker locker(&_connectionPoolMutex);
68 // we have to recheck if the connection pool already contains a connection for
69 // this thread. Since now (after the lock) we can only tell for sure
70 if (_connectionPool.contains(QThread::currentThread()))
73 QThread* currentThread = QThread::currentThread();
75 int connectionId = _nextConnectionId++;
77 Connection* connection = new Connection(QLatin1String(QString("quassel_%1_con_%2").arg(driverName()).arg(connectionId).toLatin1()));
78 connection->moveToThread(currentThread);
79 connect(this, &QObject::destroyed, connection, &QObject::deleteLater);
80 connect(currentThread, &QObject::destroyed, connection, &QObject::deleteLater);
81 connect(connection, &QObject::destroyed, this, &AbstractSqlStorage::connectionDestroyed);
82 _connectionPool[currentThread] = connection;
84 QSqlDatabase db = QSqlDatabase::addDatabase(driverName(), connection->name());
85 db.setDatabaseName(databaseName());
87 if (!hostName().isEmpty())
88 db.setHostName(hostName());
93 if (!userName().isEmpty()) {
94 db.setUserName(userName());
95 db.setPassword(password());
101 void AbstractSqlStorage::dbConnect(QSqlDatabase& db)
104 qWarning() << "Unable to open database" << displayName() << "for thread" << QThread::currentThread();
105 qWarning() << "-" << db.lastError().text();
108 if (!initDbSession(db)) {
109 qWarning() << "Unable to initialize database" << displayName() << "for thread" << QThread::currentThread();
115 Storage::State AbstractSqlStorage::init(const QVariantMap& settings, const QProcessEnvironment& environment, bool loadFromEnvironment)
117 setConnectionProperties(settings, environment, loadFromEnvironment);
119 _debug = Quassel::isOptionSet("debug");
121 QSqlDatabase db = logDb();
122 if (!db.isValid() || !db.isOpen())
125 if (installedSchemaVersion() == -1) {
126 qCritical() << "Storage Schema is missing!";
130 if (installedSchemaVersion() > schemaVersion()) {
131 qCritical() << "Installed Schema is newer then any known Version.";
135 if (installedSchemaVersion() < schemaVersion()) {
136 qInfo() << qPrintable(tr("Installed database schema (version %1) is not up to date. Upgrading to "
137 "version %2... This may take a while for major upgrades.")
138 .arg(installedSchemaVersion())
139 .arg(schemaVersion()));
140 emit dbUpgradeInProgress(true);
141 auto upgradeResult = upgradeDb();
142 emit dbUpgradeInProgress(false);
143 if (!upgradeResult) {
144 qWarning() << qPrintable(tr("Upgrade failed..."));
147 // Add a message when migration succeeds to avoid confusing folks by implying the schema upgrade failed if
148 // later functionality does not work.
149 qInfo() << qPrintable(tr("Installed database schema successfully upgraded to version %1.").arg(schemaVersion()));
152 qInfo() << qPrintable(displayName()) << "storage backend is ready. Schema version:" << installedSchemaVersion();
156 QString AbstractSqlStorage::queryString(const QString& queryName, int version)
160 // The current schema is stored in the root folder, while upgrade queries are stored in the
161 // 'versions/##' subfolders.
163 // Use the current SQL schema, not a versioned request
164 queryInfo = QFileInfo(QString(":/SQL/%1/%2.sql").arg(displayName()).arg(queryName));
165 // If version is needed later, get it via version = schemaVersion();
168 // Use the specified schema version, not the general folder
169 queryInfo = QFileInfo(QString(":/SQL/%1/version/%2/%3.sql").arg(displayName()).arg(version).arg(queryName));
172 if (!queryInfo.exists() || !queryInfo.isFile() || !queryInfo.isReadable()) {
173 qCritical() << "Unable to read SQL-Query" << queryName << "for engine" << displayName();
177 QFile queryFile(queryInfo.filePath());
178 if (!queryFile.open(QIODevice::ReadOnly | QIODevice::Text))
180 QString query = QTextStream(&queryFile).readAll();
183 return query.trimmed();
186 std::vector<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::setupQueries()
188 std::vector<SqlQueryResource> queries;
189 // The current schema is stored in the root folder, including setup scripts.
190 QDir dir = QDir(QString(":/SQL/%1/").arg(displayName()));
191 foreach (QFileInfo fileInfo, dir.entryInfoList(QStringList() << "setup*", QDir::NoFilter, QDir::Name)) {
192 queries.emplace_back(queryString(fileInfo.baseName()), fileInfo.baseName());
197 bool AbstractSqlStorage::setup(const QVariantMap& settings, const QProcessEnvironment& environment, bool loadFromEnvironment)
199 setConnectionProperties(settings, environment, loadFromEnvironment);
200 QSqlDatabase db = logDb();
202 qCritical() << "Unable to setup Logging Backend!";
207 foreach (auto queryResource, setupQueries()) {
208 QSqlQuery query = db.exec(queryResource.queryString);
209 if (!watchQuery(query)) {
210 qCritical() << qPrintable(QString("Unable to setup Logging Backend! Setup query failed (step: %1).")
211 .arg(queryResource.queryFilename));
216 bool success = setupSchemaVersion(schemaVersion());
224 std::vector<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::upgradeQueries(int version)
226 std::vector<SqlQueryResource> queries;
227 // Upgrade queries are stored in the 'version/##' subfolders.
228 QDir dir = QDir(QString(":/SQL/%1/version/%2/").arg(displayName()).arg(version));
229 foreach (QFileInfo fileInfo, dir.entryInfoList(QStringList() << "upgrade*", QDir::NoFilter, QDir::Name)) {
230 queries.emplace_back(queryString(fileInfo.baseName(), version), fileInfo.baseName());
235 bool AbstractSqlStorage::upgradeDb()
237 if (schemaVersion() <= installedSchemaVersion())
240 QSqlDatabase db = logDb();
242 // TODO: For databases that support it (e.g. almost only PostgreSQL), wrap upgrades in a
243 // transaction. This will need careful testing of potential additional space requirements and
244 // any database modifications that might not be allowed in a transaction.
246 // Check if we're resuming an interrupted multi-step upgrade: is an upgrade step stored?
247 const QString previousLaunchUpgradeStep = schemaVersionUpgradeStep();
248 bool resumingUpgrade = !previousLaunchUpgradeStep.isEmpty();
250 for (int ver = installedSchemaVersion() + 1; ver <= schemaVersion(); ver++) {
251 foreach (auto queryResource, upgradeQueries(ver)) {
252 if (resumingUpgrade) {
253 // An upgrade was interrupted. Check if this matches the the last successful query.
254 if (previousLaunchUpgradeStep == queryResource.queryFilename) {
255 // Found the matching query!
256 qInfo() << qPrintable(QString("Resuming interrupted upgrade for schema version %1 (last step: %2)")
257 .arg(QString::number(ver), previousLaunchUpgradeStep));
259 // Stop searching for queries
260 resumingUpgrade = false;
261 // Continue past the previous query with the next not-yet-tried query
265 // Not yet matched, keep looking
270 // Run the upgrade query
271 QSqlQuery query = db.exec(queryResource.queryString);
272 if (!watchQuery(query)) {
273 // Individual upgrade query failed, bail out
274 qCritical() << qPrintable(QString("Unable to upgrade Logging Backend! Upgrade query in schema version %1 failed (step: %2).")
275 .arg(QString::number(ver), queryResource.queryFilename));
279 // Mark as successful
280 setSchemaVersionUpgradeStep(queryResource.queryFilename);
284 if (resumingUpgrade) {
285 // Something went wrong and the last successful SQL query to resume from couldn't be
287 // 1. The storage of successful query glitched, or the database was manually changed
288 // 2. Quassel changed the filenames of upgrade queries, and the local Quassel core
289 // version was replaced during an interrupted schema upgrade
291 // Both are unlikely, but it's a good idea to handle it anyways.
293 qCritical() << qPrintable(QString("Unable to resume interrupted upgrade in Logging "
294 "Backend! Missing upgrade step in schema version %1 "
295 "(expected step: %2)")
296 .arg(QString::number(ver), previousLaunchUpgradeStep));
300 // Update the schema version for each intermediate step and mark the step as done. This
301 // ensures that any interrupted upgrades have a greater chance of resuming correctly after
304 // Almost all databases make single queries atomic (fully works or fully fails, no partial),
305 // and with many of the longest migrations being a single query, this makes upgrade
306 // interruptions much more likely to leave the database in a valid intermediate schema
308 if (!updateSchemaVersion(ver, true)) {
309 // Updating the schema version failed, bail out
310 qCritical() << "Unable to upgrade Logging Backend! Setting schema version" << ver << "failed.";
315 // If we made it here, everything seems to have worked!
319 int AbstractSqlStorage::schemaVersion()
321 // returns the newest Schema Version!
322 // not the currently used one! (though it can be the same)
323 if (_schemaVersion > 0)
324 return _schemaVersion;
328 // Schema versions are stored in the 'version/##' subfolders.
329 QDir dir = QDir(QString(":/SQL/%1/version/").arg(displayName()));
330 foreach (QFileInfo fileInfo, dir.entryInfoList()) {
331 if (!fileInfo.isDir())
334 version = fileInfo.fileName().toInt(&ok);
338 if (version > _schemaVersion)
339 _schemaVersion = version;
341 return _schemaVersion;
345 QString AbstractSqlStorage::schemaVersionUpgradeStep()
347 // By default, assume there's no pending upgrade
352 bool AbstractSqlStorage::watchQuery(QSqlQuery& query)
354 bool queryError = query.lastError().isValid();
355 if (queryError || _debug) {
357 qCritical() << "unhandled Error in QSqlQuery!";
358 qCritical() << " last Query:\n" << qPrintable(query.lastQuery());
359 qCritical() << " executed Query:\n" << qPrintable(query.executedQuery());
360 QVariantMap boundValues = query.boundValues();
361 QStringList valueStrings;
362 QVariantMap::const_iterator iter;
363 for (iter = boundValues.constBegin(); iter != boundValues.constEnd(); ++iter) {
366 if (query.driver()) {
367 // let the driver do the formatting
368 field.setType(iter.value().type());
369 if (iter.value().isNull())
372 field.setValue(iter.value());
373 value = query.driver()->formatValue(field);
376 switch (iter.value().type()) {
377 case QVariant::Invalid:
381 value = iter.value().toString();
384 value = QString("'%1'").arg(iter.value().toString());
387 valueStrings << QString("%1=%2").arg(iter.key(), value);
389 qCritical() << " bound Values:" << qPrintable(valueStrings.join(", "));
390 qCritical() << " Error Code:" << qPrintable(query.lastError().nativeErrorCode());
391 qCritical() << " Error Message:" << qPrintable(query.lastError().text());
392 qCritical() << " Driver Message:" << qPrintable(query.lastError().driverText());
393 qCritical() << " DB Message:" << qPrintable(query.lastError().databaseText());
400 void AbstractSqlStorage::connectionDestroyed()
402 QMutexLocker locker(&_connectionPoolMutex);
403 _connectionPool.remove(sender()->thread());
406 // ========================================
407 // AbstractSqlStorage::Connection
408 // ========================================
409 AbstractSqlStorage::Connection::Connection(const QString& name, QObject* parent)
411 , _name(name.toLatin1())
414 AbstractSqlStorage::Connection::~Connection()
417 QSqlDatabase db = QSqlDatabase::database(name(), false);
423 QSqlDatabase::removeDatabase(name());
426 // ========================================
427 // AbstractSqlMigrator
428 // ========================================
430 void AbstractSqlMigrator::newQuery(const QString& query, QSqlDatabase db)
433 _query = new QSqlQuery(db);
434 _query->prepare(query);
437 void AbstractSqlMigrator::resetQuery()
443 bool AbstractSqlMigrator::exec()
447 return !_query->lastError().isValid();
450 QString AbstractSqlMigrator::migrationObject(MigrationObject moType)
454 return "QuasselUser";
460 return "IdentityNick";
470 return "UserSetting";
477 QVariantList AbstractSqlMigrator::boundValues()
483 int numValues = _query->boundValues().count();
484 for (int i = 0; i < numValues; i++) {
485 values << _query->boundValue(i);
490 void AbstractSqlMigrator::dumpStatus()
492 qWarning() << " executed Query:";
493 qWarning() << qPrintable(executedQuery());
494 qWarning() << " bound Values:";
495 QList<QVariant> list = boundValues();
496 for (int i = 0; i < list.size(); ++i)
497 qWarning() << i << ": " << list.at(i).toString().toLatin1().data();
498 qWarning() << " Error Code:" << qPrintable(lastError().nativeErrorCode());
499 qWarning() << " Error Message:" << lastError().text();
502 // ========================================
503 // AbstractSqlMigrationReader
504 // ========================================
505 AbstractSqlMigrationReader::AbstractSqlMigrationReader()
506 : AbstractSqlMigrator()
509 bool AbstractSqlMigrationReader::migrateTo(AbstractSqlMigrationWriter* writer)
511 if (!transaction()) {
512 qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start reader's transaction!";
515 if (!writer->transaction()) {
516 qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start writer's transaction!";
517 rollback(); // close the reader transaction;
523 // due to the incompatibility across Migration objects we can't run this in a loop... :/
524 QuasselUserMO quasselUserMo;
525 if (!transferMo(QuasselUser, quasselUserMo))
528 IdentityMO identityMo;
529 if (!transferMo(Identity, identityMo))
532 IdentityNickMO identityNickMo;
533 if (!transferMo(IdentityNick, identityNickMo))
537 if (!transferMo(Network, networkMo))
541 if (!transferMo(Buffer, bufferMo))
545 if (!transferMo(Sender, senderMo))
549 if (!transferMo(Backlog, backlogMo))
552 IrcServerMO ircServerMo;
553 if (!transferMo(IrcServer, ircServerMo))
556 UserSettingMO userSettingMo;
557 if (!transferMo(UserSetting, userSettingMo))
560 CoreStateMO coreStateMO;
561 if (!transferMo(CoreState, coreStateMO))
564 if (!_writer->postProcess())
566 return finalizeMigration();
569 void AbstractSqlMigrationReader::abortMigration(const QString& errorMsg)
571 qWarning() << "Migration Failed!";
572 if (!errorMsg.isNull()) {
573 qWarning() << qPrintable(errorMsg);
575 if (lastError().isValid()) {
576 qWarning() << "ReaderError:";
580 if (_writer->lastError().isValid()) {
581 qWarning() << "WriterError:";
582 _writer->dumpStatus();
590 bool AbstractSqlMigrationReader::finalizeMigration()
593 _writer->resetQuery();
596 if (!_writer->commit()) {
605 bool AbstractSqlMigrationReader::transferMo(MigrationObject moType, T& mo)
608 _writer->resetQuery();
610 if (!prepareQuery(moType)) {
611 abortMigration(QString("AbstractSqlMigrationReader::migrateTo(): unable to prepare reader query of type %1!")
612 .arg(AbstractSqlMigrator::migrationObject(moType)));
615 if (!_writer->prepareQuery(moType)) {
616 abortMigration(QString("AbstractSqlMigrationReader::migrateTo(): unable to prepare writer query of type %1!")
617 .arg(AbstractSqlMigrator::migrationObject(moType)));
621 qDebug() << qPrintable(QString("Transferring %1...").arg(AbstractSqlMigrator::migrationObject(moType)));
624 file.open(stdout, QIODevice::WriteOnly);
627 if (!_writer->writeMo(mo)) {
628 abortMigration(QString("AbstractSqlMigrationReader::transferMo(): unable to transfer Migratable Object of type %1!")
629 .arg(AbstractSqlMigrator::migrationObject(moType)));
647 uint qHash(const SenderData& key)
649 return qHash(QString(key.sender + "\n" + key.realname + "\n" + key.avatarurl));
652 bool operator==(const SenderData& a, const SenderData& b)
654 return a.sender == b.sender && a.realname == b.realname && a.avatarurl == b.avatarurl;