src/core/abstractsqlstorage.cpp

   1 /***************************************************************************
   2  *   Copyright (C) 2005-2019 by the Quassel Project                        *
   3  *   devel@quassel-irc.org                                                 *
   4  *                                                                         *
   5  *   This program is free software; you can redistribute it and/or modify  *
   6  *   it under the terms of the GNU General Public License as published by  *
   7  *   the Free Software Foundation; either version 2 of the License, or     *
   8  *   (at your option) version 3.                                           *
   9  *                                                                         *
  10  *   This program is distributed in the hope that it will be useful,       *
  11  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  12  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  13  *   GNU General Public License for more details.                          *
  14  *                                                                         *
  15  *   You should have received a copy of the GNU General Public License     *
  16  *   along with this program; if not, write to the                         *
  17  *   Free Software Foundation, Inc.,                                       *
  18  *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.         *
  19  ***************************************************************************/
  20
  21 #include "abstractsqlstorage.h"
  22
  23 #include <QMutexLocker>
  24 #include <QSqlDriver>
  25 #include <QSqlError>
  26 #include <QSqlField>
  27 #include <QSqlQuery>
  28
  29 #include "quassel.h"
  30
  31 int AbstractSqlStorage::_nextConnectionId = 0;
  32 AbstractSqlStorage::AbstractSqlStorage(QObject* parent)
  33     : Storage(parent)
  34 {}
  35
  36 AbstractSqlStorage::~AbstractSqlStorage()
  37 {
  38     // disconnect the connections, so their deletion is no longer interessting for us
  39     QHash<QThread*, Connection*>::iterator conIter;
  40     for (conIter = _connectionPool.begin(); conIter != _connectionPool.end(); ++conIter) {
  41         QSqlDatabase::removeDatabase(conIter.value()->name());
  42         disconnect(conIter.value(), nullptr, this, nullptr);
  43     }
  44 }
  45
  46 QSqlDatabase AbstractSqlStorage::logDb()
  47 {
  48     if (!_connectionPool.contains(QThread::currentThread()))
  49         addConnectionToPool();
  50
  51     QSqlDatabase db = QSqlDatabase::database(_connectionPool[QThread::currentThread()]->name(), false);
  52
  53     if (!db.isOpen()) {
  54         qWarning() << "Database connection" << displayName() << "for thread" << QThread::currentThread()
  55                    << "was lost, attempting to reconnect...";
  56         dbConnect(db);
  57     }
  58
  59     return db;
  60 }
  61
  62 void AbstractSqlStorage::addConnectionToPool()
  63 {
  64     QMutexLocker locker(&_connectionPoolMutex);
  65     // we have to recheck if the connection pool already contains a connection for
  66     // this thread. Since now (after the lock) we can only tell for sure
  67     if (_connectionPool.contains(QThread::currentThread()))
  68         return;
  69
  70     QThread* currentThread = QThread::currentThread();
  71
  72     int connectionId = _nextConnectionId++;
  73
  74     Connection* connection = new Connection(QLatin1String(QString("quassel_%1_con_%2").arg(driverName()).arg(connectionId).toLatin1()));
  75     connection->moveToThread(currentThread);
  76     connect(this, &QObject::destroyed, connection, &QObject::deleteLater);
  77     connect(currentThread, &QObject::destroyed, connection, &QObject::deleteLater);
  78     connect(connection, &QObject::destroyed, this, &AbstractSqlStorage::connectionDestroyed);
  79     _connectionPool[currentThread] = connection;
  80
  81     QSqlDatabase db = QSqlDatabase::addDatabase(driverName(), connection->name());
  82     db.setDatabaseName(databaseName());
  83
  84     if (!hostName().isEmpty())
  85         db.setHostName(hostName());
  86
  87     if (port() != -1)
  88         db.setPort(port());
  89
  90     if (!userName().isEmpty()) {
  91         db.setUserName(userName());
  92         db.setPassword(password());
  93     }
  94
  95     dbConnect(db);
  96 }
  97
  98 void AbstractSqlStorage::dbConnect(QSqlDatabase& db)
  99 {
 100     if (!db.open()) {
 101         qWarning() << "Unable to open database" << displayName() << "for thread" << QThread::currentThread();
 102         qWarning() << "-" << db.lastError().text();
 103     }
 104     else {
 105         if (!initDbSession(db)) {
 106             qWarning() << "Unable to initialize database" << displayName() << "for thread" << QThread::currentThread();
 107             db.close();
 108         }
 109     }
 110 }
 111
 112 Storage::State AbstractSqlStorage::init(const QVariantMap& settings, const QProcessEnvironment& environment, bool loadFromEnvironment)
 113 {
 114     setConnectionProperties(settings, environment, loadFromEnvironment);
 115
 116     _debug = Quassel::isOptionSet("debug");
 117
 118     QSqlDatabase db = logDb();
 119     if (!db.isValid() || !db.isOpen())
 120         return NotAvailable;
 121
 122     if (installedSchemaVersion() == -1) {
 123         qCritical() << "Storage Schema is missing!";
 124         return NeedsSetup;
 125     }
 126
 127     if (installedSchemaVersion() > schemaVersion()) {
 128         qCritical() << "Installed Schema is newer then any known Version.";
 129         return NotAvailable;
 130     }
 131
 132     if (installedSchemaVersion() < schemaVersion()) {
 133         qInfo() << qPrintable(tr("Installed database schema (version %1) is not up to date. Upgrading to "
 134                                   "version %2...  This may take a while for major upgrades.")
 135                                    .arg(installedSchemaVersion())
 136                                    .arg(schemaVersion()));
 137         emit dbUpgradeInProgress(true);
 138         auto upgradeResult = upgradeDb();
 139         emit dbUpgradeInProgress(false);
 140         if (!upgradeResult) {
 141             qWarning() << qPrintable(tr("Upgrade failed..."));
 142             return NotAvailable;
 143         }
 144         // Add a message when migration succeeds to avoid confusing folks by implying the schema upgrade failed if
 145         // later functionality does not work.
 146         qInfo() << qPrintable(tr("Installed database schema successfully upgraded to version %1.").arg(schemaVersion()));
 147     }
 148
 149     qInfo() << qPrintable(displayName()) << "storage backend is ready. Schema version:" << installedSchemaVersion();
 150     return IsReady;
 151 }
 152
 153 QString AbstractSqlStorage::queryString(const QString& queryName, int version)
 154 {
 155     QFileInfo queryInfo;
 156
 157     // The current schema is stored in the root folder, while upgrade queries are stored in the
 158     // 'versions/##' subfolders.
 159     if (version == 0) {
 160         // Use the current SQL schema, not a versioned request
 161         queryInfo = QFileInfo(QString(":/SQL/%1/%2.sql").arg(displayName()).arg(queryName));
 162         // If version is needed later, get it via version = schemaVersion();
 163     }
 164     else {
 165         // Use the specified schema version, not the general folder
 166         queryInfo = QFileInfo(QString(":/SQL/%1/version/%2/%3.sql").arg(displayName()).arg(version).arg(queryName));
 167     }
 168
 169     if (!queryInfo.exists() || !queryInfo.isFile() || !queryInfo.isReadable()) {
 170         qCritical() << "Unable to read SQL-Query" << queryName << "for engine" << displayName();
 171         return QString();
 172     }
 173
 174     QFile queryFile(queryInfo.filePath());
 175     if (!queryFile.open(QIODevice::ReadOnly | QIODevice::Text))
 176         return QString();
 177     QString query = QTextStream(&queryFile).readAll();
 178     queryFile.close();
 179
 180     return query.trimmed();
 181 }
 182
 183 QList<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::setupQueries()
 184 {
 185     QList<SqlQueryResource> queries;
 186     // The current schema is stored in the root folder, including setup scripts.
 187     QDir dir = QDir(QString(":/SQL/%1/").arg(displayName()));
 188     foreach (QFileInfo fileInfo, dir.entryInfoList(QStringList() << "setup*", QDir::NoFilter, QDir::Name)) {
 189         queries << SqlQueryResource(queryString(fileInfo.baseName()), fileInfo.baseName());
 190     }
 191     return queries;
 192 }
 193
 194 bool AbstractSqlStorage::setup(const QVariantMap& settings, const QProcessEnvironment& environment, bool loadFromEnvironment)
 195 {
 196     setConnectionProperties(settings, environment, loadFromEnvironment);
 197     QSqlDatabase db = logDb();
 198     if (!db.isOpen()) {
 199         qCritical() << "Unable to setup Logging Backend!";
 200         return false;
 201     }
 202
 203     db.transaction();
 204     foreach (auto queryResource, setupQueries()) {
 205         QSqlQuery query = db.exec(queryResource.queryString);
 206         if (!watchQuery(query)) {
 207             qCritical() << qPrintable(QString("Unable to setup Logging Backend!  Setup query failed (step: %1).")
 208                                       .arg(queryResource.queryFilename));
 209             db.rollback();
 210             return false;
 211         }
 212     }
 213     bool success = setupSchemaVersion(schemaVersion());
 214     if (success)
 215         db.commit();
 216     else
 217         db.rollback();
 218     return success;
 219 }
 220
 221 QList<AbstractSqlStorage::SqlQueryResource> AbstractSqlStorage::upgradeQueries(int version)
 222 {
 223     QList<SqlQueryResource> queries;
 224     // Upgrade queries are stored in the 'version/##' subfolders.
 225     QDir dir = QDir(QString(":/SQL/%1/version/%2/").arg(displayName()).arg(version));
 226     foreach (QFileInfo fileInfo, dir.entryInfoList(QStringList() << "upgrade*", QDir::NoFilter, QDir::Name)) {
 227         queries << SqlQueryResource(queryString(fileInfo.baseName(), version), fileInfo.baseName());
 228     }
 229     return queries;
 230 }
 231
 232 bool AbstractSqlStorage::upgradeDb()
 233 {
 234     if (schemaVersion() <= installedSchemaVersion())
 235         return true;
 236
 237     QSqlDatabase db = logDb();
 238
 239     // TODO: For databases that support it (e.g. almost only PostgreSQL), wrap upgrades in a
 240     // transaction.  This will need careful testing of potential additional space requirements and
 241     // any database modifications that might not be allowed in a transaction.
 242
 243     // Check if we're resuming an interrupted multi-step upgrade: is an upgrade step stored?
 244     const QString previousLaunchUpgradeStep = schemaVersionUpgradeStep();
 245     bool resumingUpgrade = !previousLaunchUpgradeStep.isEmpty();
 246
 247     for (int ver = installedSchemaVersion() + 1; ver <= schemaVersion(); ver++) {
 248         foreach (auto queryResource, upgradeQueries(ver)) {
 249             if (resumingUpgrade) {
 250                 // An upgrade was interrupted.  Check if this matches the the last successful query.
 251                 if (previousLaunchUpgradeStep == queryResource.queryFilename) {
 252                     // Found the matching query!
 253                     qInfo() << qPrintable(QString("Resuming interrupted upgrade for schema version %1 (last step: %2)")
 254                                           .arg(QString::number(ver), previousLaunchUpgradeStep));
 255
 256                     // Stop searching for queries
 257                     resumingUpgrade = false;
 258                     // Continue past the previous query with the next not-yet-tried query
 259                     continue;
 260                 }
 261                 else {
 262                     // Not yet matched, keep looking
 263                     continue;
 264                 }
 265             }
 266
 267             // Run the upgrade query
 268             QSqlQuery query = db.exec(queryResource.queryString);
 269             if (!watchQuery(query)) {
 270                 // Individual upgrade query failed, bail out
 271                 qCritical() << qPrintable(QString("Unable to upgrade Logging Backend!  Upgrade query in schema version %1 failed (step: %2).")
 272                                           .arg(QString::number(ver), queryResource.queryFilename));
 273                 return false;
 274             }
 275             else {
 276                 // Mark as successful
 277                 setSchemaVersionUpgradeStep(queryResource.queryFilename);
 278             }
 279         }
 280
 281         if (resumingUpgrade) {
 282             // Something went wrong and the last successful SQL query to resume from couldn't be
 283             // found.
 284             // 1.  The storage of successful query glitched, or the database was manually changed
 285             // 2.  Quassel changed the filenames of upgrade queries, and the local Quassel core
 286             //     version was replaced during an interrupted schema upgrade
 287             //
 288             // Both are unlikely, but it's a good idea to handle it anyways.
 289
 290             qCritical() << qPrintable(QString("Unable to resume interrupted upgrade in Logging "
 291                                               "Backend!  Missing upgrade step in schema version %1 "
 292                                               "(expected step: %2)")
 293                                       .arg(QString::number(ver), previousLaunchUpgradeStep));
 294             return false;
 295         }
 296
 297         // Update the schema version for each intermediate step and mark the step as done.  This
 298         // ensures that any interrupted upgrades have a greater chance of resuming correctly after
 299         // core restart.
 300         //
 301         // Almost all databases make single queries atomic (fully works or fully fails, no partial),
 302         // and with many of the longest migrations being a single query, this makes upgrade
 303         // interruptions much more likely to leave the database in a valid intermediate schema
 304         // version.
 305         if (!updateSchemaVersion(ver, true)) {
 306             // Updating the schema version failed, bail out
 307             qCritical() << "Unable to upgrade Logging Backend!  Setting schema version" << ver << "failed.";
 308             return false;
 309         }
 310     }
 311
 312     // If we made it here, everything seems to have worked!
 313     return true;
 314 }
 315
 316 int AbstractSqlStorage::schemaVersion()
 317 {
 318     // returns the newest Schema Version!
 319     // not the currently used one! (though it can be the same)
 320     if (_schemaVersion > 0)
 321         return _schemaVersion;
 322
 323     int version;
 324     bool ok;
 325     // Schema versions are stored in the 'version/##' subfolders.
 326     QDir dir = QDir(QString(":/SQL/%1/version/").arg(displayName()));
 327     foreach (QFileInfo fileInfo, dir.entryInfoList()) {
 328         if (!fileInfo.isDir())
 329             continue;
 330
 331         version = fileInfo.fileName().toInt(&ok);
 332         if (!ok)
 333             continue;
 334
 335         if (version > _schemaVersion)
 336             _schemaVersion = version;
 337     }
 338     return _schemaVersion;
 339 }
 340
 341
 342 QString AbstractSqlStorage::schemaVersionUpgradeStep()
 343 {
 344     // By default, assume there's no pending upgrade
 345     return {};
 346 }
 347
 348
 349 bool AbstractSqlStorage::watchQuery(QSqlQuery& query)
 350 {
 351     bool queryError = query.lastError().isValid();
 352     if (queryError || _debug) {
 353         if (queryError)
 354             qCritical() << "unhandled Error in QSqlQuery!";
 355         qCritical() << "                  last Query:\n" << qPrintable(query.lastQuery());
 356         qCritical() << "              executed Query:\n" << qPrintable(query.executedQuery());
 357         QVariantMap boundValues = query.boundValues();
 358         QStringList valueStrings;
 359         QVariantMap::const_iterator iter;
 360         for (iter = boundValues.constBegin(); iter != boundValues.constEnd(); ++iter) {
 361             QString value;
 362             QSqlField field;
 363             if (query.driver()) {
 364                 // let the driver do the formatting
 365                 field.setType(iter.value().type());
 366                 if (iter.value().isNull())
 367                     field.clear();
 368                 else
 369                     field.setValue(iter.value());
 370                 value = query.driver()->formatValue(field);
 371             }
 372             else {
 373                 switch (iter.value().type()) {
 374                 case QVariant::Invalid:
 375                     value = "NULL";
 376                     break;
 377                 case QVariant::Int:
 378                     value = iter.value().toString();
 379                     break;
 380                 default:
 381                     value = QString("'%1'").arg(iter.value().toString());
 382                 }
 383             }
 384             valueStrings << QString("%1=%2").arg(iter.key(), value);
 385         }
 386         qCritical() << "                bound Values:" << qPrintable(valueStrings.join(", "));
 387         qCritical() << "                Error Number:" << query.lastError().number();
 388         qCritical() << "               Error Message:" << qPrintable(query.lastError().text());
 389         qCritical() << "              Driver Message:" << qPrintable(query.lastError().driverText());
 390         qCritical() << "                  DB Message:" << qPrintable(query.lastError().databaseText());
 391
 392         return !queryError;
 393     }
 394     return true;
 395 }
 396
 397 void AbstractSqlStorage::connectionDestroyed()
 398 {
 399     QMutexLocker locker(&_connectionPoolMutex);
 400     _connectionPool.remove(sender()->thread());
 401 }
 402
 403 // ========================================
 404 //  AbstractSqlStorage::Connection
 405 // ========================================
 406 AbstractSqlStorage::Connection::Connection(const QString& name, QObject* parent)
 407     : QObject(parent)
 408     , _name(name.toLatin1())
 409 {}
 410
 411 AbstractSqlStorage::Connection::~Connection()
 412 {
 413     {
 414         QSqlDatabase db = QSqlDatabase::database(name(), false);
 415         if (db.isOpen()) {
 416             db.commit();
 417             db.close();
 418         }
 419     }
 420     QSqlDatabase::removeDatabase(name());
 421 }
 422
 423 // ========================================
 424 //  AbstractSqlMigrator
 425 // ========================================
 426
 427 void AbstractSqlMigrator::newQuery(const QString& query, QSqlDatabase db)
 428 {
 429     Q_ASSERT(!_query);
 430     _query = new QSqlQuery(db);
 431     _query->prepare(query);
 432 }
 433
 434 void AbstractSqlMigrator::resetQuery()
 435 {
 436     delete _query;
 437     _query = nullptr;
 438 }
 439
 440 bool AbstractSqlMigrator::exec()
 441 {
 442     Q_ASSERT(_query);
 443     _query->exec();
 444     return !_query->lastError().isValid();
 445 }
 446
 447 QString AbstractSqlMigrator::migrationObject(MigrationObject moType)
 448 {
 449     switch (moType) {
 450     case QuasselUser:
 451         return "QuasselUser";
 452     case Sender:
 453         return "Sender";
 454     case Identity:
 455         return "Identity";
 456     case IdentityNick:
 457         return "IdentityNick";
 458     case Network:
 459         return "Network";
 460     case Buffer:
 461         return "Buffer";
 462     case Backlog:
 463         return "Backlog";
 464     case IrcServer:
 465         return "IrcServer";
 466     case UserSetting:
 467         return "UserSetting";
 468     case CoreState:
 469         return "CoreState";
 470     };
 471     return QString();
 472 }
 473
 474 QVariantList AbstractSqlMigrator::boundValues()
 475 {
 476     QVariantList values;
 477     if (!_query)
 478         return values;
 479
 480     int numValues = _query->boundValues().count();
 481     for (int i = 0; i < numValues; i++) {
 482         values << _query->boundValue(i);
 483     }
 484     return values;
 485 }
 486
 487 void AbstractSqlMigrator::dumpStatus()
 488 {
 489     qWarning() << "  executed Query:";
 490     qWarning() << qPrintable(executedQuery());
 491     qWarning() << "  bound Values:";
 492     QList<QVariant> list = boundValues();
 493     for (int i = 0; i < list.size(); ++i)
 494         qWarning() << i << ": " << list.at(i).toString().toLatin1().data();
 495     qWarning() << "  Error Number:" << lastError().number();
 496     qWarning() << "  Error Message:" << lastError().text();
 497 }
 498
 499 // ========================================
 500 //  AbstractSqlMigrationReader
 501 // ========================================
 502 AbstractSqlMigrationReader::AbstractSqlMigrationReader()
 503     : AbstractSqlMigrator()
 504 {}
 505
 506 bool AbstractSqlMigrationReader::migrateTo(AbstractSqlMigrationWriter* writer)
 507 {
 508     if (!transaction()) {
 509         qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start reader's transaction!";
 510         return false;
 511     }
 512     if (!writer->transaction()) {
 513         qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start writer's transaction!";
 514         rollback();  // close the reader transaction;
 515         return false;
 516     }
 517
 518     _writer = writer;
 519
 520     // due to the incompatibility across Migration objects we can't run this in a loop... :/
 521     QuasselUserMO quasselUserMo;
 522     if (!transferMo(QuasselUser, quasselUserMo))
 523         return false;
 524
 525     IdentityMO identityMo;
 526     if (!transferMo(Identity, identityMo))
 527         return false;
 528
 529     IdentityNickMO identityNickMo;
 530     if (!transferMo(IdentityNick, identityNickMo))
 531         return false;
 532
 533     NetworkMO networkMo;
 534     if (!transferMo(Network, networkMo))
 535         return false;
 536
 537     BufferMO bufferMo;
 538     if (!transferMo(Buffer, bufferMo))
 539         return false;
 540
 541     SenderMO senderMo;
 542     if (!transferMo(Sender, senderMo))
 543         return false;
 544
 545     BacklogMO backlogMo;
 546     if (!transferMo(Backlog, backlogMo))
 547         return false;
 548
 549     IrcServerMO ircServerMo;
 550     if (!transferMo(IrcServer, ircServerMo))
 551         return false;
 552
 553     UserSettingMO userSettingMo;
 554     if (!transferMo(UserSetting, userSettingMo))
 555         return false;
 556
 557     CoreStateMO coreStateMO;
 558     if (!transferMo(CoreState, coreStateMO))
 559         return false;
 560
 561     if (!_writer->postProcess())
 562         abortMigration();
 563     return finalizeMigration();
 564 }
 565
 566 void AbstractSqlMigrationReader::abortMigration(const QString& errorMsg)
 567 {
 568     qWarning() << "Migration Failed!";
 569     if (!errorMsg.isNull()) {
 570         qWarning() << qPrintable(errorMsg);
 571     }
 572     if (lastError().isValid()) {
 573         qWarning() << "ReaderError:";
 574         dumpStatus();
 575     }
 576
 577     if (_writer->lastError().isValid()) {
 578         qWarning() << "WriterError:";
 579         _writer->dumpStatus();
 580     }
 581
 582     rollback();
 583     _writer->rollback();
 584     _writer = nullptr;
 585 }
 586
 587 bool AbstractSqlMigrationReader::finalizeMigration()
 588 {
 589     resetQuery();
 590     _writer->resetQuery();
 591
 592     commit();
 593     if (!_writer->commit()) {
 594         _writer = nullptr;
 595         return false;
 596     }
 597     _writer = nullptr;
 598     return true;
 599 }
 600
 601 template<typename T>
 602 bool AbstractSqlMigrationReader::transferMo(MigrationObject moType, T& mo)
 603 {
 604     resetQuery();
 605     _writer->resetQuery();
 606
 607     if (!prepareQuery(moType)) {
 608         abortMigration(QString("AbstractSqlMigrationReader::migrateTo(): unable to prepare reader query of type %1!")
 609                            .arg(AbstractSqlMigrator::migrationObject(moType)));
 610         return false;
 611     }
 612     if (!_writer->prepareQuery(moType)) {
 613         abortMigration(QString("AbstractSqlMigrationReader::migrateTo(): unable to prepare writer query of type %1!")
 614                            .arg(AbstractSqlMigrator::migrationObject(moType)));
 615         return false;
 616     }
 617
 618     qDebug() << qPrintable(QString("Transferring %1...").arg(AbstractSqlMigrator::migrationObject(moType)));
 619     int i = 0;
 620     QFile file;
 621     file.open(stdout, QIODevice::WriteOnly);
 622
 623     while (readMo(mo)) {
 624         if (!_writer->writeMo(mo)) {
 625             abortMigration(QString("AbstractSqlMigrationReader::transferMo(): unable to transfer Migratable Object of type %1!")
 626                                .arg(AbstractSqlMigrator::migrationObject(moType)));
 627             return false;
 628         }
 629         i++;
 630         if (i % 1000 == 0) {
 631             file.write("*");
 632             file.flush();
 633         }
 634     }
 635     if (i > 1000) {
 636         file.write("\n");
 637         file.flush();
 638     }
 639
 640     qDebug() << "Done.";
 641     return true;
 642 }
 643
 644 uint qHash(const SenderData& key)
 645 {
 646     return qHash(QString(key.sender + "\n" + key.realname + "\n" + key.avatarurl));
 647 }
 648
 649 bool operator==(const SenderData& a, const SenderData& b)
 650 {
 651     return a.sender == b.sender && a.realname == b.realname && a.avatarurl == b.avatarurl;
 652 }