core: Track upgrade step within schema version
[quassel.git] / src / core / abstractsqlstorage.cpp
1 /***************************************************************************
2  *   Copyright (C) 2005-2019 by the Quassel Project                        *
3  *   devel@quassel-irc.org                                                 *
4  *                                                                         *
5  *   This program is free software; you can redistribute it and/or modify  *
6  *   it under the terms of the GNU General Public License as published by  *
7  *   the Free Software Foundation; either version 2 of the License, or     *
8  *   (at your option) version 3.                                           *
9  *                                                                         *
10  *   This program is distributed in the hope that it will be useful,       *
11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
13  *   GNU General Public License for more details.                          *
14  *                                                                         *
15  *   You should have received a copy of the GNU General Public License     *
16  *   along with this program; if not, write to the                         *
17  *   Free Software Foundation, Inc.,                                       *
18  *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.         *
19  ***************************************************************************/
20
21 #include "abstractsqlstorage.h"
22
23 #include <QMutexLocker>
24 #include <QSqlDriver>
25 #include <QSqlError>
26 #include <QSqlField>
27 #include <QSqlQuery>
28
29 #include "logmessage.h"
30 #include "quassel.h"
31
32 int AbstractSqlStorage::_nextConnectionId = 0;
33 AbstractSqlStorage::AbstractSqlStorage(QObject *parent)
34     : Storage(parent),
35     _schemaVersion(0)
36 {
37 }
38
39
40 AbstractSqlStorage::~AbstractSqlStorage()
41 {
42     // disconnect the connections, so their deletion is no longer interessting for us
43     QHash<QThread *, Connection *>::iterator conIter;
44     for (conIter = _connectionPool.begin(); conIter != _connectionPool.end(); ++conIter) {
45         QSqlDatabase::removeDatabase(conIter.value()->name());
46         disconnect(conIter.value(), 0, this, 0);
47     }
48 }
49
50
51 QSqlDatabase AbstractSqlStorage::logDb()
52 {
53     if (!_connectionPool.contains(QThread::currentThread()))
54         addConnectionToPool();
55
56     QSqlDatabase db = QSqlDatabase::database(_connectionPool[QThread::currentThread()]->name(),false);
57
58     if (!db.isOpen()) {
59         qWarning() << "Database connection" << displayName() << "for thread" << QThread::currentThread() << "was lost, attempting to reconnect...";
60         dbConnect(db);
61     }
62
63     return db;
64 }
65
66
67 void AbstractSqlStorage::addConnectionToPool()
68 {
69     QMutexLocker locker(&_connectionPoolMutex);
70     // we have to recheck if the connection pool already contains a connection for
71     // this thread. Since now (after the lock) we can only tell for sure
72     if (_connectionPool.contains(QThread::currentThread()))
73         return;
74
75     QThread *currentThread = QThread::currentThread();
76
77     int connectionId = _nextConnectionId++;
78
79     Connection *connection = new Connection(QLatin1String(QString("quassel_%1_con_%2").arg(driverName()).arg(connectionId).toLatin1()));
80     connection->moveToThread(currentThread);
81     connect(this, SIGNAL(destroyed()), connection, SLOT(deleteLater()));
82     connect(currentThread, SIGNAL(destroyed()), connection, SLOT(deleteLater()));
83     connect(connection, SIGNAL(destroyed()), this, SLOT(connectionDestroyed()));
84     _connectionPool[currentThread] = connection;
85
86     QSqlDatabase db = QSqlDatabase::addDatabase(driverName(), connection->name());
87     db.setDatabaseName(databaseName());
88
89     if (!hostName().isEmpty())
90         db.setHostName(hostName());
91
92     if (port() != -1)
93         db.setPort(port());
94
95     if (!userName().isEmpty()) {
96         db.setUserName(userName());
97         db.setPassword(password());
98     }
99
100     dbConnect(db);
101 }
102
103
104 void AbstractSqlStorage::dbConnect(QSqlDatabase &db)
105 {
106     if (!db.open()) {
107         quWarning() << "Unable to open database" << displayName() << "for thread" << QThread::currentThread();
108         quWarning() << "-" << db.lastError().text();
109     }
110     else {
111         if (!initDbSession(db)) {
112             quWarning() << "Unable to initialize database" << displayName() << "for thread" << QThread::currentThread();
113             db.close();
114         }
115     }
116 }
117
118
119 Storage::State AbstractSqlStorage::init(const QVariantMap &settings,
120                                         const QProcessEnvironment &environment,
121                                         bool loadFromEnvironment)
122 {
123     setConnectionProperties(settings, environment, loadFromEnvironment);
124
125     _debug = Quassel::isOptionSet("debug");
126
127     QSqlDatabase db = logDb();
128     if (!db.isValid() || !db.isOpen())
129         return NotAvailable;
130
131     if (installedSchemaVersion() == -1) {
132         qCritical() << "Storage Schema is missing!";
133         return NeedsSetup;
134     }
135
136     if (installedSchemaVersion() > schemaVersion()) {
137         qCritical() << "Installed Schema is newer then any known Version.";
138         return NotAvailable;
139     }
140
141     if (installedSchemaVersion() < schemaVersion()) {
142         quInfo() << qPrintable(tr("Installed database schema (version %1) is not up to date. Upgrading to "
143                                   "version %2...  This may take a while for major upgrades."
144                                  ).arg(installedSchemaVersion()).arg(schemaVersion()));
145         emit dbUpgradeInProgress(true);
146         auto upgradeResult = upgradeDb();
147         emit dbUpgradeInProgress(false);
148         if (!upgradeResult) {
149             qWarning() << qPrintable(tr("Upgrade failed..."));
150             return NotAvailable;
151         }
152         // Add a message when migration succeeds to avoid confusing folks by implying the schema upgrade failed if
153         // later functionality does not work.
154         quInfo() << qPrintable(tr("Installed database schema successfully upgraded to version %1.").arg(schemaVersion()));
155     }
156
157     quInfo() << qPrintable(displayName()) << "storage backend is ready. Schema version:" << installedSchemaVersion();
158     return IsReady;
159 }
160
161
162 QString AbstractSqlStorage::queryString(const QString &queryName, int version)
163 {
164     QFileInfo queryInfo;
165
166     // The current schema is stored in the root folder, while upgrade queries are stored in the
167     // 'versions/##' subfolders.
168     if (version == 0) {
169         // Use the current SQL schema, not a versioned request
170         queryInfo = QFileInfo(QString(":/SQL/%1/%2.sql").arg(displayName()).arg(queryName));
171         // If version is needed later, get it via version = schemaVersion();
172     } else {
173         // Use the specified schema version, not the general folder
174         queryInfo = QFileInfo(QString(":/SQL/%1/version/%2/%3.sql")
175                               .arg(displayName()).arg(version).arg(queryName));
176     }
177
178     if (!queryInfo.exists() || !queryInfo.isFile() || !queryInfo.isReadable()) {
179         qCritical() << "Unable to read SQL-Query" << queryName << "for engine" << displayName();
180         return QString();
181     }
182
183     QFile queryFile(queryInfo.filePath());
184     if (!queryFile.open(QIODevice::ReadOnly | QIODevice::Text))
185         return QString();
186     QString query = QTextStream(&queryFile).readAll();
187     queryFile.close();
188
189     return query.trimmed();
190 }
191
192
193 QStringList AbstractSqlStorage::setupQueries()
194 {
195     QStringList queries;
196     // The current schema is stored in the root folder, including setup scripts.
197     QDir dir = QDir(QString(":/SQL/%1/").arg(displayName()));
198     foreach(QFileInfo fileInfo, dir.entryInfoList(QStringList() << "setup*", QDir::NoFilter, QDir::Name)) {
199         queries << queryString(fileInfo.baseName());
200     }
201     return queries;
202 }
203
204
205 bool AbstractSqlStorage::setup(const QVariantMap &settings, const QProcessEnvironment &environment,
206                                bool loadFromEnvironment)
207 {
208     setConnectionProperties(settings, environment, loadFromEnvironment);
209     QSqlDatabase db = logDb();
210     if (!db.isOpen()) {
211         qCritical() << "Unable to setup Logging Backend!";
212         return false;
213     }
214
215     db.transaction();
216     foreach(QString queryString, setupQueries()) {
217         QSqlQuery query = db.exec(queryString);
218         if (!watchQuery(query)) {
219             qCritical() << "Unable to setup Logging Backend!";
220             db.rollback();
221             return false;
222         }
223     }
224     bool success = setupSchemaVersion(schemaVersion());
225     if (success)
226         db.commit();
227     else
228         db.rollback();
229     return success;
230 }
231
232
233 QList<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::upgradeQueries(int version)
234 {
235     QList<SqlQueryResource> queries;
236     // Upgrade queries are stored in the 'version/##' subfolders.
237     QDir dir = QDir(QString(":/SQL/%1/version/%2/").arg(displayName()).arg(version));
238     foreach(QFileInfo fileInfo, dir.entryInfoList(QStringList() << "upgrade*", QDir::NoFilter, QDir::Name)) {
239         queries << SqlQueryResource(queryString(fileInfo.baseName(), version), fileInfo.baseName());
240     }
241     return queries;
242 }
243
244
245 bool AbstractSqlStorage::upgradeDb()
246 {
247     if (schemaVersion() <= installedSchemaVersion())
248         return true;
249
250     QSqlDatabase db = logDb();
251
252     // TODO: For databases that support it (e.g. almost only PostgreSQL), wrap upgrades in a
253     // transaction.  This will need careful testing of potential additional space requirements and
254     // any database modifications that might not be allowed in a transaction.
255
256     // Check if we're resuming an interrupted multi-step upgrade: is an upgrade step stored?
257     const QString previousLaunchUpgradeStep = schemaVersionUpgradeStep();
258     bool resumingUpgrade = !previousLaunchUpgradeStep.isEmpty();
259
260     for (int ver = installedSchemaVersion() + 1; ver <= schemaVersion(); ver++) {
261         foreach (auto queryResource, upgradeQueries(ver)) {
262             if (resumingUpgrade) {
263                 // An upgrade was interrupted.  Check if this matches the the last successful query.
264                 if (previousLaunchUpgradeStep == queryResource.queryFilename) {
265                     // Found the matching query!
266                     quInfo() << qPrintable(QString("Resuming interrupted upgrade for schema version %1 (last step: %2)")
267                                            .arg(QString::number(ver), previousLaunchUpgradeStep));
268
269                     // Stop searching for queries
270                     resumingUpgrade = false;
271                     // Continue past the previous query with the next not-yet-tried query
272                     continue;
273                 }
274                 else {
275                     // Not yet matched, keep looking
276                     continue;
277                 }
278             }
279
280             // Run the upgrade query
281             QSqlQuery query = db.exec(queryResource.queryString);
282             if (!watchQuery(query)) {
283                 // Individual upgrade query failed, bail out
284                 qCritical() << qPrintable(QString("Unable to upgrade Logging Backend!  Upgrade query in schema version %1 failed (step: %2).")
285                                           .arg(QString::number(ver), queryResource.queryFilename));
286                 return false;
287             }
288             else {
289                 // Mark as successful
290                 setSchemaVersionUpgradeStep(queryResource.queryFilename);
291             }
292         }
293
294         if (resumingUpgrade) {
295             // Something went wrong and the last successful SQL query to resume from couldn't be
296             // found.
297             // 1.  The storage of successful query glitched, or the database was manually changed
298             // 2.  Quassel changed the filenames of upgrade queries, and the local Quassel core
299             //     version was replaced during an interrupted schema upgrade
300             //
301             // Both are unlikely, but it's a good idea to handle it anyways.
302
303             qCritical() << qPrintable(QString("Unable to resume interrupted upgrade in Logging "
304                                               "Backend!  Missing upgrade step in schema version %1 "
305                                               "(expected step: %2)")
306                                       .arg(QString::number(ver), previousLaunchUpgradeStep));
307             return false;
308         }
309
310         // Update the schema version for each intermediate step and mark the step as done.  This
311         // ensures that any interrupted upgrades have a greater chance of resuming correctly after
312         // core restart.
313         //
314         // Almost all databases make single queries atomic (fully works or fully fails, no partial),
315         // and with many of the longest migrations being a single query, this makes upgrade
316         // interruptions much more likely to leave the database in a valid intermediate schema
317         // version.
318         if (!updateSchemaVersion(ver, true)) {
319             // Updating the schema version failed, bail out
320             qCritical() << "Unable to upgrade Logging Backend!  Setting schema version"
321                         << ver << "failed.";
322             return false;
323         }
324     }
325
326     // Update the schema version for the final step.  Split this out to offer more informative
327     // logging (though setting schema version really should not fail).
328     if (!updateSchemaVersion(schemaVersion())) {
329         // Updating the final schema version failed, bail out
330         qCritical() << "Unable to upgrade Logging Backend!  Setting final schema version"
331                     << schemaVersion() << "failed.";
332         return false;
333     }
334
335     // If we made it here, everything seems to have worked!
336     return true;
337 }
338
339
340 int AbstractSqlStorage::schemaVersion()
341 {
342     // returns the newest Schema Version!
343     // not the currently used one! (though it can be the same)
344     if (_schemaVersion > 0)
345         return _schemaVersion;
346
347     int version;
348     bool ok;
349     // Schema versions are stored in the 'version/##' subfolders.
350     QDir dir = QDir(QString(":/SQL/%1/version/").arg(displayName()));
351     foreach(QFileInfo fileInfo, dir.entryInfoList()) {
352         if (!fileInfo.isDir())
353             continue;
354
355         version = fileInfo.fileName().toInt(&ok);
356         if (!ok)
357             continue;
358
359         if (version > _schemaVersion)
360             _schemaVersion = version;
361     }
362     return _schemaVersion;
363 }
364
365
366 QString AbstractSqlStorage::schemaVersionUpgradeStep()
367 {
368     // By default, assume there's no pending upgrade
369     return {};
370 }
371
372
373 bool AbstractSqlStorage::watchQuery(QSqlQuery &query)
374 {
375     bool queryError = query.lastError().isValid();
376     if (queryError || _debug) {
377         if (queryError)
378             qCritical() << "unhandled Error in QSqlQuery!";
379         qCritical() << "                  last Query:\n" << qPrintable(query.lastQuery());
380         qCritical() << "              executed Query:\n" << qPrintable(query.executedQuery());
381         QVariantMap boundValues = query.boundValues();
382         QStringList valueStrings;
383         QVariantMap::const_iterator iter;
384         for (iter = boundValues.constBegin(); iter != boundValues.constEnd(); ++iter) {
385             QString value;
386             QSqlField field;
387             if (query.driver()) {
388                 // let the driver do the formatting
389                 field.setType(iter.value().type());
390                 if (iter.value().isNull())
391                     field.clear();
392                 else
393                     field.setValue(iter.value());
394                 value =  query.driver()->formatValue(field);
395             }
396             else {
397                 switch (iter.value().type()) {
398                 case QVariant::Invalid:
399                     value = "NULL";
400                     break;
401                 case QVariant::Int:
402                     value = iter.value().toString();
403                     break;
404                 default:
405                     value = QString("'%1'").arg(iter.value().toString());
406                 }
407             }
408             valueStrings << QString("%1=%2").arg(iter.key(), value);
409         }
410         qCritical() << "                bound Values:" << qPrintable(valueStrings.join(", "));
411         qCritical() << "                Error Number:" << query.lastError().number();
412         qCritical() << "               Error Message:" << qPrintable(query.lastError().text());
413         qCritical() << "              Driver Message:" << qPrintable(query.lastError().driverText());
414         qCritical() << "                  DB Message:" << qPrintable(query.lastError().databaseText());
415
416         return !queryError;
417     }
418     return true;
419 }
420
421
422 void AbstractSqlStorage::connectionDestroyed()
423 {
424     QMutexLocker locker(&_connectionPoolMutex);
425     _connectionPool.remove(sender()->thread());
426 }
427
428
429 // ========================================
430 //  AbstractSqlStorage::Connection
431 // ========================================
432 AbstractSqlStorage::Connection::Connection(const QString &name, QObject *parent)
433     : QObject(parent),
434     _name(name.toLatin1())
435 {
436 }
437
438
439 AbstractSqlStorage::Connection::~Connection()
440 {
441     {
442         QSqlDatabase db = QSqlDatabase::database(name(), false);
443         if (db.isOpen()) {
444             db.commit();
445             db.close();
446         }
447     }
448     QSqlDatabase::removeDatabase(name());
449 }
450
451
452 // ========================================
453 //  AbstractSqlMigrator
454 // ========================================
455 AbstractSqlMigrator::AbstractSqlMigrator()
456     : _query(0)
457 {
458 }
459
460
461 void AbstractSqlMigrator::newQuery(const QString &query, QSqlDatabase db)
462 {
463     Q_ASSERT(!_query);
464     _query = new QSqlQuery(db);
465     _query->prepare(query);
466 }
467
468
469 void AbstractSqlMigrator::resetQuery()
470 {
471     delete _query;
472     _query = 0;
473 }
474
475
476 bool AbstractSqlMigrator::exec()
477 {
478     Q_ASSERT(_query);
479     _query->exec();
480     return !_query->lastError().isValid();
481 }
482
483
484 QString AbstractSqlMigrator::migrationObject(MigrationObject moType)
485 {
486     switch (moType) {
487     case QuasselUser:
488         return "QuasselUser";
489     case Sender:
490         return "Sender";
491     case Identity:
492         return "Identity";
493     case IdentityNick:
494         return "IdentityNick";
495     case Network:
496         return "Network";
497     case Buffer:
498         return "Buffer";
499     case Backlog:
500         return "Backlog";
501     case IrcServer:
502         return "IrcServer";
503     case UserSetting:
504         return "UserSetting";
505     case CoreState:
506         return "CoreState";
507     };
508     return QString();
509 }
510
511
512 QVariantList AbstractSqlMigrator::boundValues()
513 {
514     QVariantList values;
515     if (!_query)
516         return values;
517
518     int numValues = _query->boundValues().count();
519     for (int i = 0; i < numValues; i++) {
520         values << _query->boundValue(i);
521     }
522     return values;
523 }
524
525
526 void AbstractSqlMigrator::dumpStatus()
527 {
528     qWarning() << "  executed Query:";
529     qWarning() << qPrintable(executedQuery());
530     qWarning() << "  bound Values:";
531     QList<QVariant> list = boundValues();
532     for (int i = 0; i < list.size(); ++i)
533         qWarning() << i << ": " << list.at(i).toString().toLatin1().data();
534     qWarning() << "  Error Number:"   << lastError().number();
535     qWarning() << "  Error Message:"   << lastError().text();
536 }
537
538
539 // ========================================
540 //  AbstractSqlMigrationReader
541 // ========================================
542 AbstractSqlMigrationReader::AbstractSqlMigrationReader()
543     : AbstractSqlMigrator(),
544     _writer(0)
545 {
546 }
547
548
549 bool AbstractSqlMigrationReader::migrateTo(AbstractSqlMigrationWriter *writer)
550 {
551     if (!transaction()) {
552         qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start reader's transaction!";
553         return false;
554     }
555     if (!writer->transaction()) {
556         qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start writer's transaction!";
557         rollback(); // close the reader transaction;
558         return false;
559     }
560
561     _writer = writer;
562
563     // due to the incompatibility across Migration objects we can't run this in a loop... :/
564     QuasselUserMO quasselUserMo;
565     if (!transferMo(QuasselUser, quasselUserMo))
566         return false;
567
568     IdentityMO identityMo;
569     if (!transferMo(Identity, identityMo))
570         return false;
571
572     IdentityNickMO identityNickMo;
573     if (!transferMo(IdentityNick, identityNickMo))
574         return false;
575
576     NetworkMO networkMo;
577     if (!transferMo(Network, networkMo))
578         return false;
579
580     BufferMO bufferMo;
581     if (!transferMo(Buffer, bufferMo))
582         return false;
583
584     SenderMO senderMo;
585     if (!transferMo(Sender, senderMo))
586         return false;
587
588     BacklogMO backlogMo;
589     if (!transferMo(Backlog, backlogMo))
590         return false;
591
592     IrcServerMO ircServerMo;
593     if (!transferMo(IrcServer, ircServerMo))
594         return false;
595
596     UserSettingMO userSettingMo;
597     if (!transferMo(UserSetting, userSettingMo))
598         return false;
599
600     CoreStateMO coreStateMO;
601     if (!transferMo(CoreState, coreStateMO))
602         return false;
603
604     if (!_writer->postProcess())
605         abortMigration();
606     return finalizeMigration();
607 }
608
609
610 void AbstractSqlMigrationReader::abortMigration(const QString &errorMsg)
611 {
612     qWarning() << "Migration Failed!";
613     if (!errorMsg.isNull()) {
614         qWarning() << qPrintable(errorMsg);
615     }
616     if (lastError().isValid()) {
617         qWarning() << "ReaderError:";
618         dumpStatus();
619     }
620
621     if (_writer->lastError().isValid()) {
622         qWarning() << "WriterError:";
623         _writer->dumpStatus();
624     }
625
626     rollback();
627     _writer->rollback();
628     _writer = 0;
629 }
630
631
632 bool AbstractSqlMigrationReader::finalizeMigration()
633 {
634     resetQuery();
635     _writer->resetQuery();
636
637     commit();
638     if (!_writer->commit()) {
639         _writer = 0;
640         return false;
641     }
642     _writer = 0;
643     return true;
644 }
645
646
647 template<typename T>
648 bool AbstractSqlMigrationReader::transferMo(MigrationObject moType, T &mo)
649 {
650     resetQuery();
651     _writer->resetQuery();
652
653     if (!prepareQuery(moType)) {
654         abortMigration(QString("AbstractSqlMigrationReader::migrateTo(): unable to prepare reader query of type %1!").arg(AbstractSqlMigrator::migrationObject(moType)));
655         return false;
656     }
657     if (!_writer->prepareQuery(moType)) {
658         abortMigration(QString("AbstractSqlMigrationReader::migrateTo(): unable to prepare writer query of type %1!").arg(AbstractSqlMigrator::migrationObject(moType)));
659         return false;
660     }
661
662     qDebug() << qPrintable(QString("Transferring %1...").arg(AbstractSqlMigrator::migrationObject(moType)));
663     int i = 0;
664     QFile file;
665     file.open(stdout, QIODevice::WriteOnly);
666
667     while (readMo(mo)) {
668         if (!_writer->writeMo(mo)) {
669             abortMigration(QString("AbstractSqlMigrationReader::transferMo(): unable to transfer Migratable Object of type %1!").arg(AbstractSqlMigrator::migrationObject(moType)));
670             return false;
671         }
672         i++;
673         if (i % 1000 == 0) {
674             file.write("*");
675             file.flush();
676         }
677     }
678     if (i > 1000) {
679         file.write("\n");
680         file.flush();
681     }
682
683     qDebug() << "Done.";
684     return true;
685 }
686
687 uint qHash(const SenderData &key) {
688     return qHash(QString(key.sender + "\n" + key.realname + "\n" + key.avatarurl));
689 }
690
691 bool operator==(const SenderData &a, const SenderData &b) {
692     return a.sender == b.sender &&
693         a.realname == b.realname &&
694         a.avatarurl == b.avatarurl;
695 }