From: Marcus Eggenberger <egs@quassel-irc.org>
Date: Tue, 3 Mar 2009 11:54:38 +0000 (+0100)
Subject: migration no longer eats memory
X-Git-Tag: 0.5-rc1~327
X-Git-Url: https://git.quassel-irc.org/?p=quassel.git;a=commitdiff_plain;h=f6781dc095957d65e8fc1683fda1c5a5b9cbedbb

migration no longer eats memory
---

diff --git a/src/common/main.cpp b/src/common/main.cpp
index 2a33a3d8..40922b65 100644
--- a/src/common/main.cpp
+++ b/src/common/main.cpp
@@ -99,6 +99,8 @@ int main(int argc, char **argv) {
   cliParser->addOption("loglevel <level>", 'L', "Loglevel Debug|Info|Warning|Error", "Info");
   cliParser->addOption("configdir <path>", 'c', "Specify the directory holding configuration files, the SQlite database and the SSL Cert");
   cliParser->addOption("datadir <path>", 0, "DEPRECATED - Use --configdir instead");
+  cliParser->addOption("migrate-backend <backendidentifier>", 0, "Starts an interactive session and attempts to migrate your current storage backend to the new one");
+  cliParser->addOption("switch-backend <backendidentifier>", 0, "Starts an interactive session and switches your current storage backend to the new one. No migration will be done!");
 #endif
 
 #ifdef HAVE_KDE
diff --git a/src/core/SQL/SQLite/14/migrate_read_backlog.sql b/src/core/SQL/SQLite/14/migrate_read_backlog.sql
index 6db9d234..6c0f2995 100644
--- a/src/core/SQL/SQLite/14/migrate_read_backlog.sql
+++ b/src/core/SQL/SQLite/14/migrate_read_backlog.sql
@@ -1,3 +1,4 @@
 SELECT messageid, time, bufferid, type, flags, senderid, message
 FROM backlog
-
+WHERE messageid > ? AND messageid <= ?
+ORDER BY messageid ASC
diff --git a/src/core/SQL/SQLite/14/migrate_read_sender.sql b/src/core/SQL/SQLite/14/migrate_read_sender.sql
index c74d2e94..9cd5ce12 100644
--- a/src/core/SQL/SQLite/14/migrate_read_sender.sql
+++ b/src/core/SQL/SQLite/14/migrate_read_sender.sql
@@ -1,2 +1,5 @@
 SELECT senderid, sender
 FROM sender
+WHERE senderid > ? AND senderid <= ?
+ORDER BY senderid ASC
+
diff --git a/src/core/abstractsqlstorage.cpp b/src/core/abstractsqlstorage.cpp
index 618c929b..8523d506 100644
--- a/src/core/abstractsqlstorage.cpp
+++ b/src/core/abstractsqlstorage.cpp
@@ -26,10 +26,10 @@
 #include <QSqlError>
 #include <QSqlQuery>
 
+int AbstractSqlStorage::_nextConnectionId = 0;
 AbstractSqlStorage::AbstractSqlStorage(QObject *parent)
   : Storage(parent),
-    _schemaVersion(0),
-    _nextConnectionId(0)
+    _schemaVersion(0)
 {
 }
 
@@ -37,6 +37,7 @@ AbstractSqlStorage::~AbstractSqlStorage() {
   // disconnect the connections, so their deletion is no longer interessting for us
   QHash<QThread *, Connection *>::iterator conIter;
   for(conIter = _connectionPool.begin(); conIter != _connectionPool.end(); conIter++) {
+    QSqlDatabase::removeDatabase(conIter.value()->name());
     disconnect(conIter.value(), 0, this, 0);
   }
 }
@@ -350,11 +351,11 @@ AbstractSqlMigrationReader::AbstractSqlMigrationReader()
 
 bool AbstractSqlMigrationReader::migrateTo(AbstractSqlMigrationWriter *writer) {
   if(!transaction()) {
-    qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start reader stransaction!";
+    qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start reader's transaction!";
     return false;
   }
   if(!writer->transaction()) {
-    qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start writer stransaction!";
+    qWarning() << "AbstractSqlMigrationReader::migrateTo(): unable to start writer's transaction!";
     rollback(); // close the reader transaction;
     return false;
   }
@@ -366,10 +367,6 @@ bool AbstractSqlMigrationReader::migrateTo(AbstractSqlMigrationWriter *writer) {
   if(!transferMo(QuasselUser, quasselUserMo))
      return false;
 
-  SenderMO senderMo;
-  if(!transferMo(Sender, senderMo))
-    return false;
-
   IdentityMO identityMo;
   if(!transferMo(Identity, identityMo))
     return false;
@@ -386,6 +383,10 @@ bool AbstractSqlMigrationReader::migrateTo(AbstractSqlMigrationWriter *writer) {
   if(!transferMo(Buffer, bufferMo))
     return false;
 
+  SenderMO senderMo;
+  if(!transferMo(Sender, senderMo))
+    return false;
+
   BacklogMO backlogMo;
   if(!transferMo(Backlog, backlogMo))
     return false;
@@ -455,9 +456,12 @@ bool AbstractSqlMigrationReader::transferMo(MigrationObject moType, T &mo) {
   int i = 0;
   QFile file;
   file.open(stdout, QIODevice::WriteOnly);
+
   while(readMo(mo)) {
     if(!_writer->writeMo(mo)) {
       abortMigration(QString("AbstractSqlMigrationReader::transferMo(): unable to transfer Migratable Object of type %1!").arg(AbstractSqlMigrator::migrationObject(moType)));
+      rollback();
+      _writer->rollback();
       return false;
     }
     i++;
@@ -470,6 +474,7 @@ bool AbstractSqlMigrationReader::transferMo(MigrationObject moType, T &mo) {
     file.write("\n");
     file.flush();
   }
+
   qDebug() << "Done.";
   return true;
 }
diff --git a/src/core/abstractsqlstorage.h b/src/core/abstractsqlstorage.h
index 28942913..cc9f619a 100644
--- a/src/core/abstractsqlstorage.h
+++ b/src/core/abstractsqlstorage.h
@@ -80,7 +80,7 @@ private:
 
   int _schemaVersion;
 
-  int _nextConnectionId;
+  static int _nextConnectionId;
   QMutex _connectionPoolMutex;
   // we let a Connection Object manage each actual db connection
   // those objects reside in the thread the connection belongs to
@@ -121,6 +121,7 @@ public:
   struct SenderMO {
     int senderId;
     QString sender;
+    SenderMO() : senderId(0) {}
   };
 
   struct IdentityMO {
@@ -269,11 +270,11 @@ public:
   AbstractSqlMigrationReader();
 
   virtual bool readMo(QuasselUserMO &user) = 0;
-  virtual bool readMo(SenderMO &sender) = 0;
   virtual bool readMo(IdentityMO &identity) = 0;
   virtual bool readMo(IdentityNickMO &identityNick) = 0;
   virtual bool readMo(NetworkMO &network) = 0;
   virtual bool readMo(BufferMO &buffer) = 0;
+  virtual bool readMo(SenderMO &sender) = 0;
   virtual bool readMo(BacklogMO &backlog) = 0;
   virtual bool readMo(IrcServerMO &ircserver) = 0;
   virtual bool readMo(UserSettingMO &userSetting) = 0;
@@ -292,11 +293,11 @@ private:
 class AbstractSqlMigrationWriter : public AbstractSqlMigrator {
 public:
   virtual bool writeMo(const QuasselUserMO &user) = 0;
-  virtual bool writeMo(const SenderMO &sender) = 0;
   virtual bool writeMo(const IdentityMO &identity) = 0;
   virtual bool writeMo(const IdentityNickMO &identityNick) = 0;
   virtual bool writeMo(const NetworkMO &network) = 0;
   virtual bool writeMo(const BufferMO &buffer) = 0;
+  virtual bool writeMo(const SenderMO &sender) = 0;
   virtual bool writeMo(const BacklogMO &backlog) = 0;
   virtual bool writeMo(const IrcServerMO &ircserver) = 0;
   virtual bool writeMo(const UserSettingMO &userSetting) = 0;
diff --git a/src/core/core.cpp b/src/core/core.cpp
index 5dd97210..d859c2a9 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -131,37 +131,18 @@ Core::Core()
     exit(EXIT_FAILURE);
   }
 
-  // Register storage backends here!
-  registerStorageBackend(new SqliteStorage(this));
-  registerStorageBackend(new PostgreSqlStorage(this));
+  registerStorageBackends();
 
   connect(&_storageSyncTimer, SIGNAL(timeout()), this, SLOT(syncStorage()));
   _storageSyncTimer.start(10 * 60 * 1000); // 10 minutes
 }
 
 void Core::init() {
-  CoreSettings cs2;
-  QVariantMap connectionProperties = cs2.storageSettings().toMap()["ConnectionProperties"].toMap();
-  qDebug() << connectionProperties;
-  SqliteMigrationReader *reader = new SqliteMigrationReader();
-  qDebug() << "reader:" << reader->init();
-  PostgreSqlMigrationWriter *writer = new PostgreSqlMigrationWriter();
-  qDebug() << "writer:" << writer->init(connectionProperties);
-  qDebug() << qPrintable(QString("Migrating Storage backend %1 to %2...").arg(reader->displayName(), writer->displayName()));
-  if(reader->migrateTo(writer))
-    qDebug() << "Migration finished!";
-  return;
-
-
-
-
-
-
-
-
-
+  if(Quassel::isOptionSet("switch-backend")) {
+    switchBackend(Quassel::optionValue("switch-backend"));
+    exit(0);
+  }
 
-  
   CoreSettings cs;
   _configured = initStorage(cs.storageSettings().toMap());
 
@@ -176,6 +157,11 @@ void Core::init() {
     qWarning() << "Core is currently not configured! Please connect with a Quassel Client for basic setup.";
   }
 
+  if(Quassel::isOptionSet("migrate-backend")) {
+    migrateBackend(Quassel::optionValue("migrate-backend"));
+    exit(0);
+  }
+
   connect(&_server, SIGNAL(newConnection()), this, SLOT(incomingConnection()));
   connect(&_v6server, SIGNAL(newConnection()), this, SLOT(incomingConnection()));
   if(!startListening()) exit(1); // TODO make this less brutal
@@ -263,6 +249,11 @@ QString Core::setupCore(QVariantMap setupData) {
 }
 
 /*** Storage Handling ***/
+void Core::registerStorageBackends() {
+  // Register storage backends here!
+  registerStorageBackend(new SqliteStorage(this));
+  registerStorageBackend(new PostgreSqlStorage(this));
+}
 
 bool Core::registerStorageBackend(Storage *backend) {
   if(backend->isAvailable()) {
@@ -274,6 +265,13 @@ bool Core::registerStorageBackend(Storage *backend) {
   }
 }
 
+void Core::unregisterStorageBackends() {
+  foreach(Storage *s, _storageBackends.values()) {
+    s->deleteLater();
+  }
+  _storageBackends.clear();
+}
+
 void Core::unregisterStorageBackend(Storage *backend) {
   _storageBackends.remove(backend->displayName());
   backend->deleteLater();
@@ -281,10 +279,9 @@ void Core::unregisterStorageBackend(Storage *backend) {
 
 // old db settings:
 // "Type" => "sqlite"
-bool Core::initStorage(QVariantMap dbSettings, bool setup) {
+bool Core::initStorage(const QString &backend, QVariantMap settings, bool setup) {
   _storage = 0;
 
-  QString backend = dbSettings["Backend"].toString();
   if(backend.isEmpty()) {
     return false;
   }
@@ -297,15 +294,13 @@ bool Core::initStorage(QVariantMap dbSettings, bool setup) {
     return false;
   }
 
-  QVariantMap connectionProperties = dbSettings["ConnectionProperties"].toMap();
-
-  Storage::State storageState = storage->init(connectionProperties);
+  Storage::State storageState = storage->init(settings);
   switch(storageState) {
   case Storage::NeedsSetup:
     if(!setup)
       return false; // trigger setup process
-    if(storage->setup(connectionProperties))
-      return initStorage(dbSettings, false);
+    if(storage->setup(settings))
+      return initStorage(backend, settings, false);
     // if setup wasn't successfull we mark the backend as unavailable
   case Storage::NotAvailable:
     qCritical() << "Selected storage backend is not available:" << backend;
@@ -315,16 +310,19 @@ bool Core::initStorage(QVariantMap dbSettings, bool setup) {
     return false;
   case Storage::IsReady:
     // delete all other backends
-    foreach(Storage *s, _storageBackends.values()) {
-      if(s != storage) s->deleteLater();
-    }
-    _storageBackends.clear();
+    _storageBackends.remove(backend);
+    unregisterStorageBackends();
     connect(storage, SIGNAL(bufferInfoUpdated(UserId, const BufferInfo &)), this, SIGNAL(bufferInfoUpdated(UserId, const BufferInfo &)));
   }
   _storage = storage;
   return true;
 }
 
+bool Core::initStorage(QVariantMap dbSettings, bool setup) {
+  return initStorage(dbSettings["Backend"].toString(), dbSettings["ConnectionProperties"].toMap(), setup);
+}
+
+
 void Core::syncStorage() {
   if(_storage)
     _storage->sync();
@@ -696,3 +694,135 @@ void Core::socketError(QAbstractSocket::SocketError err) {
   if(socket && err != QAbstractSocket::RemoteHostClosedError)
     qWarning() << "Core::socketError()" << socket << err << socket->errorString();
 }
+
+bool Core::migrateBackend(const QString &backend) {
+  AbstractSqlStorage *sqlStorage = qobject_cast<AbstractSqlStorage *>(_storage);
+  if(!sqlStorage) {
+    qWarning() << "Core::migrateDb(): only SQL based backends can be migrated!";
+    return false;
+  }
+
+  AbstractSqlMigrationReader *reader = sqlStorage->createMigrationReader();
+  if(!reader) {
+    qWarning() << qPrintable(QString("Core::migrateDb(): unable to migrate storage backend! (No migration reader for %1)").arg(sqlStorage->displayName()));
+    return false;
+  }
+
+
+  // reregister all storage backends
+  registerStorageBackends();
+  if(_storageBackends.contains(sqlStorage->displayName())) {
+    unregisterStorageBackend(_storageBackends[sqlStorage->displayName()]);
+  }
+  if(!_storageBackends.contains(backend)) {
+    qWarning() << qPrintable(QString("Core::migrateBackend(): unsupported migration target: %1").arg(backend));
+    qWarning() << "    supported Targets:" << qPrintable(QStringList(_storageBackends.keys()).join(", "));
+    return false;
+  }
+
+  Storage *storage = _storageBackends[backend];
+  QVariantMap settings = promptForSettings(storage->setupKeys());
+
+  Storage::State storageState = storage->init(settings);
+  switch(storageState) {
+  case Storage::NotAvailable:
+    qCritical() << "Core::migrateBackend(): selected storage backend is not available:" << backend;
+    return false;
+  case Storage::NeedsSetup:
+    if(!storage->setup(settings)) {
+      qWarning() << qPrintable(QString("Core::migrateBackend(): unable to setup target: %1").arg(backend));
+      return false;
+    }
+
+    if(storage->init(settings) != Storage::IsReady) {
+      qWarning() << qPrintable(QString("Core::migrateBackend(): unable to initialize target: %1").arg(backend));
+      return false;
+    }
+  case Storage::IsReady:
+    break;
+  }
+
+  sqlStorage = qobject_cast<AbstractSqlStorage *>(storage);
+  if(!sqlStorage) {
+    qWarning() << "Core::migrateDb(): only SQL based backends can be migrated!";
+    return false;
+  }
+  AbstractSqlMigrationWriter *writer = sqlStorage->createMigrationWriter();
+  if(!writer) {
+    qWarning() << qPrintable(QString("Core::migrateDb(): unable to migrate storage backend! (No migration writer for %1)").arg(backend));
+    return false;
+  }
+
+  qDebug() << qPrintable(QString("Migrating Storage backend %1 to %2...").arg(_storage->displayName(), storage->displayName()));
+  delete _storage;
+  _storage = 0;
+  delete storage;
+  storage = 0;
+  if(reader->migrateTo(writer)) {
+    qDebug() << "Migration finished!";
+    saveBackendSettings(backend, settings);
+    return true;
+  }
+  return false;
+}
+
+bool Core::switchBackend(const QString &backend) {
+  if(!_storageBackends.contains(backend)) {
+    qWarning() << qPrintable(QString("Core::switchBackend(): unsupported backend: %1").arg(backend));
+    qWarning() << "    supported backends:" << qPrintable(QStringList(_storageBackends.keys()).join(", "));
+    return false;
+  }
+
+  Storage *storage = _storageBackends[backend];
+  QVariantMap settings = promptForSettings(storage->setupKeys());
+
+  bool ok = initStorage(backend, settings, true /* initial setup is allowed */);
+  if(ok) {
+    saveBackendSettings(backend, settings);
+    qWarning() << "Switched backend to:" << qPrintable(backend);
+  } else {
+    qWarning() << "Failed to switch backend to:" << qPrintable(backend);
+  }
+  return ok;
+}
+
+void Core::saveBackendSettings(const QString &backend, const QVariantMap &settings) {
+  QVariantMap dbsettings;
+  dbsettings["Backend"] = backend;
+  dbsettings["ConnectionProperties"] = settings;
+  CoreSettings().setStorageSettings(dbsettings);
+}
+
+QVariantMap Core::promptForSettings(const QVariantMap &map) {
+  QVariantMap settings;
+  if(map.isEmpty())
+    return settings;
+
+  QTextStream out(stdout);
+  QTextStream in(stdin);
+  out << "!!!Warning: echo mode is always on even if asked for a password!!!" << endl;
+
+  QVariantMap::const_iterator iter;
+  QString value;
+  for(iter = map.constBegin(); iter != map.constEnd(); iter++) {
+    out << iter.key() << " (" << iter.value().toString() << "): ";
+    out.flush();
+    value = in.readLine();
+
+    if(value.isEmpty()) {
+      settings[iter.key()] = iter.value();
+    } else {
+      value = value.trimmed();
+      QVariant val;
+      switch(iter.value().type()) {
+      case QVariant::Int:
+	val = QVariant(value.toInt());
+	break;
+      default:
+	val = QVariant(value);
+      }
+      settings[iter.key()] = val;
+    }
+  }
+  return settings;
+}
diff --git a/src/core/core.h b/src/core/core.h
index 3e19474f..de579d5b 100644
--- a/src/core/core.h
+++ b/src/core/core.h
@@ -399,6 +399,7 @@ private slots:
   void clientHasData();
   void clientDisconnected();
 
+  bool initStorage(const QString &backend, QVariantMap settings, bool setup = false);
   bool initStorage(QVariantMap dbSettings, bool setup = false);
 
 #ifdef HAVE_SSL
@@ -419,8 +420,14 @@ private:
   QString setupCoreForInternalUsage();
   QString setupCore(QVariantMap setupData);
 
+  void registerStorageBackends();
   bool registerStorageBackend(Storage *);
+  void unregisterStorageBackends();
   void unregisterStorageBackend(Storage *);
+  bool migrateBackend(const QString &backend);
+  bool switchBackend(const QString &backend);
+  void saveBackendSettings(const QString &backend, const QVariantMap &settings);
+  QVariantMap promptForSettings(const QVariantMap &map);
 
   QHash<UserId, SessionThread *> sessions;
   Storage *_storage;
diff --git a/src/core/coresession.cpp b/src/core/coresession.cpp
index 32429a3a..be1e6ef3 100644
--- a/src/core/coresession.cpp
+++ b/src/core/coresession.cpp
@@ -230,7 +230,6 @@ void CoreSession::customEvent(QEvent *event) {
 }
 
 void CoreSession::processMessages() {
-  qDebug() << "processing" << _messageQueue.count() << "messages..";
   if(_messageQueue.count() == 1) {
     const RawMessage &rawMsg = _messageQueue.first();
     BufferInfo bufferInfo = Core::bufferInfo(user(), rawMsg.networkId, rawMsg.bufferType, rawMsg.target);
diff --git a/src/core/postgresqlstorage.cpp b/src/core/postgresqlstorage.cpp
index 96a60d3c..8fe4cdb4 100644
--- a/src/core/postgresqlstorage.cpp
+++ b/src/core/postgresqlstorage.cpp
@@ -35,6 +35,18 @@ PostgreSqlStorage::PostgreSqlStorage(QObject *parent)
 PostgreSqlStorage::~PostgreSqlStorage() {
 }
 
+AbstractSqlMigrationWriter *PostgreSqlStorage::createMigrationWriter() {
+  PostgreSqlMigrationWriter *writer = new PostgreSqlMigrationWriter();
+  QVariantMap properties;
+  properties["Username"] = _userName;
+  properties["Password"] = _password;
+  properties["Hostname"] = _hostName;
+  properties["Port"] = _port;
+  properties["Database"] = _databaseName;
+  writer->setConnectionProperties(properties);
+  return writer;
+}
+
 bool PostgreSqlStorage::isAvailable() const {
   if(!QSqlDatabase::isDriverAvailable("QPSQL")) return false;
   return true;
diff --git a/src/core/postgresqlstorage.h b/src/core/postgresqlstorage.h
index 16fc1c81..af8aa9c3 100644
--- a/src/core/postgresqlstorage.h
+++ b/src/core/postgresqlstorage.h
@@ -178,10 +178,4 @@ private:
   };
 };
 
-inline AbstractSqlMigrationWriter *PostgreSqlStorage::createMigrationWriter() {
-  return new PostgreSqlMigrationWriter();
-}
-
-
-
 #endif
diff --git a/src/core/sql.qrc b/src/core/sql.qrc
index 307fb947..823dcc02 100644
--- a/src/core/sql.qrc
+++ b/src/core/sql.qrc
@@ -66,6 +66,8 @@
     <file>./SQL/PostgreSQL/14/setup_080_ircservers.sql</file>
     <file>./SQL/PostgreSQL/14/setup_090_backlog_idx.sql</file>
     <file>./SQL/PostgreSQL/14/setup_100_user_setting.sql</file>
+    <file>./SQL/PostgreSQL/14/setup_110_alter_sender_seq.sql</file>
+    <file>./SQL/PostgreSQL/14/setup_120_alter_messageid_seq.sql</file>
     <file>./SQL/PostgreSQL/14/update_backlog_bufferid.sql</file>
     <file>./SQL/PostgreSQL/14/update_buffer_lastseen.sql</file>
     <file>./SQL/PostgreSQL/14/update_buffer_name.sql</file>
diff --git a/src/core/sqlitestorage.cpp b/src/core/sqlitestorage.cpp
index c2c9f0c7..3e4a40e5 100644
--- a/src/core/sqlitestorage.cpp
+++ b/src/core/sqlitestorage.cpp
@@ -1078,46 +1078,68 @@ bool SqliteStorage::safeExec(QSqlQuery &query, int retryCount) {
 //  SqliteMigration
 // ========================================
 SqliteMigrationReader::SqliteMigrationReader()
-  : SqliteStorage()
+  : SqliteStorage(),
+    _maxId(0)
 {
 }
 
+void SqliteMigrationReader::setMaxId(MigrationObject mo) {
+  QString queryString;
+  switch(mo) {
+  case Sender:
+    queryString = "SELECT max(senderid) FROM sender";
+    break;
+  case Backlog:
+    queryString = "SELECT max(messageid) FROM backlog";
+    break;
+  default:
+    _maxId = 0;
+    return;
+  }
+  QSqlQuery query = logDb().exec(queryString);
+  query.first();
+  _maxId = query.value(0).toInt();
+}
+
 bool SqliteMigrationReader::prepareQuery(MigrationObject mo) {
-  QString query;
+  setMaxId(mo);
+
   switch(mo) {
   case QuasselUser:
-    query = queryString("migrate_read_quasseluser");
-    break;
-  case Sender:
-    query = queryString("migrate_read_sender");
+    newQuery(queryString("migrate_read_quasseluser"), logDb());
     break;
   case Identity:
-    query = queryString("migrate_read_identity");
+    newQuery(queryString("migrate_read_identity"), logDb());
     break;
   case IdentityNick:
-    query = queryString("migrate_read_identity_nick");
+    newQuery(queryString("migrate_read_identity_nick"), logDb());
     break;
   case Network:
-    query = queryString("migrate_read_network");
+    newQuery(queryString("migrate_read_network"), logDb());
     break;
   case Buffer:
-    query = queryString("migrate_read_buffer");
+    newQuery(queryString("migrate_read_buffer"), logDb());
+    break;
+  case Sender:
+    newQuery(queryString("migrate_read_sender"), logDb());
+    bindValue(0, 0);
+    bindValue(1, stepSize());
     break;
   case Backlog:
-    query = queryString("migrate_read_backlog");
+    newQuery(queryString("migrate_read_backlog"), logDb());
+    bindValue(0, 0);
+    bindValue(1, stepSize());
     break;
   case IrcServer:
-    query = queryString("migrate_read_ircserver");
+    newQuery(queryString("migrate_read_ircserver"), logDb());
     break;
   case UserSetting:
-    query = queryString("migrate_read_usersetting");
+    newQuery(queryString("migrate_read_usersetting"), logDb());
     break;
   }
-  newQuery(query, logDb());
   return exec();
 }
 
-//bool SqliteMigrationReader::readUser(QuasselUserMO &user) {
 bool SqliteMigrationReader::readMo(QuasselUserMO &user) {
   if(!next())
     return false;
@@ -1128,17 +1150,6 @@ bool SqliteMigrationReader::readMo(QuasselUserMO &user) {
   return true;
 }
 
-//bool SqliteMigrationReader::readSender(SenderMO &sender) {
-bool SqliteMigrationReader::readMo(SenderMO &sender) {
-  if(!next())
-    return false;
-
-  sender.senderId = value(0).toInt();
-  sender.sender = value(1).toString();
-  return true;
-}
-
-//bool SqliteMigrationReader::readIdentity(IdentityMO &identity) {
 bool SqliteMigrationReader::readMo(IdentityMO &identity) {
   if(!next())
     return false;
@@ -1167,7 +1178,6 @@ bool SqliteMigrationReader::readMo(IdentityMO &identity) {
   return true;
 }
 
-//bool SqliteMigrationReader::readIdentityNick(IdentityNickMO &identityNick) {
 bool SqliteMigrationReader::readMo(IdentityNickMO &identityNick) {
   if(!next())
     return false;
@@ -1178,7 +1188,6 @@ bool SqliteMigrationReader::readMo(IdentityNickMO &identityNick) {
   return true;
 }
 
-//bool SqliteMigrationReader::readNetwork(NetworkMO &network) {
 bool SqliteMigrationReader::readMo(NetworkMO &network) {
   if(!next())
     return false;
@@ -1208,7 +1217,6 @@ bool SqliteMigrationReader::readMo(NetworkMO &network) {
   return true;
 }
 
-//bool SqliteMigrationReader::readBuffer(BufferMO &buffer) {
 bool SqliteMigrationReader::readMo(BufferMO &buffer) {
   if(!next())
     return false;
@@ -1226,10 +1234,38 @@ bool SqliteMigrationReader::readMo(BufferMO &buffer) {
   return true;
 }
 
-//bool SqliteMigrationReader::readBacklog(BacklogMO &backlog) {
+bool SqliteMigrationReader::readMo(SenderMO &sender) {
+  int skipSteps = 0;
+  while(!next()) {
+    if(sender.senderId < _maxId) {
+      bindValue(0, sender.senderId + (skipSteps * stepSize()));
+      bindValue(1, sender.senderId + ((skipSteps + 1) * stepSize()));
+      skipSteps++;
+      if(!exec())
+	return false;
+    } else {
+      return false;
+    }
+  }
+
+  sender.senderId = value(0).toInt();
+  sender.sender = value(1).toString();
+  return true;
+}
+
 bool SqliteMigrationReader::readMo(BacklogMO &backlog) {
-  if(!next())
-    return false;
+  int skipSteps = 0;
+  while(!next()) {
+    if(backlog.messageid < _maxId) {
+      bindValue(0, backlog.messageid.toInt() + (skipSteps * stepSize()));
+      bindValue(1, backlog.messageid.toInt() + ((skipSteps + 1) * stepSize()));
+      skipSteps++;
+      if(!exec())
+	return false;
+    } else {
+      return false;
+    }
+  }
 
   backlog.messageid = value(0).toInt();
   backlog.time = QDateTime::fromTime_t(value(1).toInt());
@@ -1241,7 +1277,6 @@ bool SqliteMigrationReader::readMo(BacklogMO &backlog) {
   return true;
 }
 
-//bool SqliteMigrationReader::readIrcServer(IrcServerMO &ircserver) {
 bool SqliteMigrationReader::readMo(IrcServerMO &ircserver) {
   if(!next())
     return false;
@@ -1263,7 +1298,6 @@ bool SqliteMigrationReader::readMo(IrcServerMO &ircserver) {
   return true;
 }
 
-//bool SqliteMigrationReader::readUserSetting(UserSettingMO &userSetting) {
 bool SqliteMigrationReader::readMo(UserSettingMO &userSetting) {
   if(!next())
     return false;
@@ -1274,5 +1308,3 @@ bool SqliteMigrationReader::readMo(UserSettingMO &userSetting) {
 
   return true;
 }
-
-
diff --git a/src/core/sqlitestorage.h b/src/core/sqlitestorage.h
index 1185f9dc..1df9ea10 100644
--- a/src/core/sqlitestorage.h
+++ b/src/core/sqlitestorage.h
@@ -139,10 +139,16 @@ public:
 
   virtual bool prepareQuery(MigrationObject mo);
 
+  inline int stepSize() { return 50000; }
+
 protected:
   virtual inline bool transaction() { return logDb().transaction(); }
   virtual inline void rollback() { logDb().rollback(); }
   virtual inline bool commit() { return logDb().commit(); }
+
+private:
+  void setMaxId(MigrationObject mo);
+  int _maxId;
 };
 
 inline AbstractSqlMigrationReader *SqliteStorage::createMigrationReader() {