X-Git-Url: https://git.quassel-irc.org/?a=blobdiff_plain;f=src%2Fcommon%2Fcompressor.cpp;h=e13c1b8c4285a97e20432e599a04d50cec888fe9;hb=8f92b3f08df9f4eb8fd243ccec6aa9d4b563ec23;hp=9ee877c2d4310d7b59cd52228305bafe5a98320b;hpb=9ad83fb2c64caf43b3f565cc79def6d43d30a5c1;p=quassel.git diff --git a/src/common/compressor.cpp b/src/common/compressor.cpp index 9ee877c2..e13c1b8c 100644 --- a/src/common/compressor.cpp +++ b/src/common/compressor.cpp @@ -1,5 +1,5 @@ /*************************************************************************** - * Copyright (C) 2005-2014 by the Quassel Project * + * Copyright (C) 2005-2019 by the Quassel Project * * devel@quassel-irc.org * * * * This program is free software; you can redistribute it and/or modify * @@ -23,31 +23,89 @@ #include #include -const qint64 maxBufferSize = 64 * 1024 * 1024; // protect us from zip bombs +const int maxBufferSize = 64 * 1024 * 1024; // protect us from zip bombs +const int ioBufferSize = 64 * 1024; // chunk size for inflate/deflate; should not be too large as we preallocate that space! -Compressor::Compressor(QTcpSocket *socket, Compressor::CompressionLevel level, QObject *parent) - : QObject(parent), - _socket(socket), - _level(level) +Compressor::Compressor(QTcpSocket* socket, Compressor::CompressionLevel level, QObject* parent) + : QObject(parent) + , _socket(socket) + , _level(level) + , _inflater(nullptr) + , _deflater(nullptr) { - _level = NoCompression; // compression not implemented yet + connect(socket, &QIODevice::readyRead, this, &Compressor::readData); - connect(socket, SIGNAL(readyRead()), SLOT(readData())); + bool ok = true; + if (level != NoCompression) + ok = initStreams(); + + if (!ok) { + // something went wrong during initialization... but we can only emit an error after RemotePeer has connected its signal + QTimer::singleShot(0, this, [this]() { emit error(); }); + return; + } // It's possible that more data has already arrived during the handshake, so readyRead() wouldn't be triggered. // However, we want to give RemotePeer a chance to connect to our signals, so trigger this asynchronously. if (socket->bytesAvailable()) - QTimer::singleShot(0, this, SLOT(readData())); + QTimer::singleShot(0, this, &Compressor::readData); } +Compressor::~Compressor() +{ + // release resources allocated by zlib + if (_inflater) { + inflateEnd(_inflater); + delete _inflater; + } + if (_deflater) { + deflateEnd(_deflater); + delete _deflater; + } +} + +bool Compressor::initStreams() +{ + int zlevel; + switch (compressionLevel()) { + case BestCompression: + zlevel = 9; + break; + case BestSpeed: + zlevel = 1; + break; + default: + zlevel = Z_DEFAULT_COMPRESSION; + } + + _inflater = new z_stream; + memset(_inflater, 0, sizeof(z_stream)); + if (Z_OK != inflateInit(_inflater)) { + qWarning() << "Could not initialize the inflate stream!"; + return false; + } + + _deflater = new z_stream; + memset(_deflater, 0, sizeof(z_stream)); + if (Z_OK != deflateInit(_deflater, zlevel)) { + qWarning() << "Could not initialize the deflate stream!"; + return false; + } + + _inputBuffer.reserve(ioBufferSize); // pre-allocate space + _outputBuffer.resize(ioBufferSize); // not a typo; we never change the size of this buffer anyway (we *do* for _inputBuffer!) + + qDebug() << "Enabling compression..."; + + return true; +} qint64 Compressor::bytesAvailable() const { return _readBuffer.size(); } - -qint64 Compressor::read(char *data, qint64 maxSize) +qint64 Compressor::read(char* data, qint64 maxSize) { if (maxSize <= 0) maxSize = _readBuffer.size(); @@ -63,16 +121,15 @@ qint64 Compressor::read(char *data, qint64 maxSize) // If there's still data left in the socket buffer, make sure to schedule a read if (_socket->bytesAvailable()) - QTimer::singleShot(0, this, SLOT(readData())); + QTimer::singleShot(0, this, &Compressor::readData); return n; } - // The usual usage pattern is to write a blocksize first, followed by the actual data. // By setting NoFlush, one can indicate that the write buffer should not immediately be // written, which should make things a bit more efficient. -qint64 Compressor::write(const char *data, qint64 count, WriteBufferHint flush) +qint64 Compressor::write(const char* data, qint64 count, WriteBufferHint flush) { int pos = _writeBuffer.size(); _writeBuffer.resize(pos + count); @@ -84,35 +141,117 @@ qint64 Compressor::write(const char *data, qint64 count, WriteBufferHint flush) return count; } - void Compressor::readData() { // don't try to read more data if we're already closing - if (_socket->state() != QAbstractSocket::ConnectedState) + if (_socket->state() != QAbstractSocket::ConnectedState) return; if (!_socket->bytesAvailable() || _readBuffer.size() >= maxBufferSize) return; - if (compressionLevel() == NoCompression) + if (compressionLevel() == NoCompression) { _readBuffer.append(_socket->read(maxBufferSize - _readBuffer.size())); + emit readyRead(); + return; + } - emit readyRead(); -} + // We let zlib directly append to the readBuffer, which means we pre-allocate extra space for ioBufferSize. + // Afterwards, we'll shrink the buffer appropriately. Since shrinking should not reallocate, the readBuffer's + // capacity should over time adapt to the largest message sizes we encounter. However, this is not a bad thing + // considering that otherwise (using an intermediate buffer) we'd copy around data for every single message. + // TODO: Benchmark if it would still make sense to squeeze the buffer from time to time (e.g. after initial sync)! + + while (_socket->bytesAvailable() && _readBuffer.size() + ioBufferSize < maxBufferSize && _inputBuffer.size() < ioBufferSize) { + _readBuffer.resize(_readBuffer.size() + ioBufferSize); + _inputBuffer.append(_socket->read(ioBufferSize - _inputBuffer.size())); + + _inflater->next_in = reinterpret_cast(_inputBuffer.data()); + _inflater->avail_in = _inputBuffer.size(); + _inflater->next_out = reinterpret_cast(_readBuffer.data() + _readBuffer.size() - ioBufferSize); + _inflater->avail_out = ioBufferSize; + + const unsigned char* orig_out = _inflater->next_out; // so we see if we have actually produced any output + int status = inflate(_inflater, Z_SYNC_FLUSH); // get as much data as possible + + // adjust input and output buffers + _readBuffer.resize(_inflater->next_out - reinterpret_cast(_readBuffer.data())); + if (_inflater->avail_in > 0) + memmove(_inputBuffer.data(), _inflater->next_in, _inflater->avail_in); + _inputBuffer.resize(_inflater->avail_in); + + if (_inflater->next_out != orig_out) + emit readyRead(); + + switch (status) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + case Z_STREAM_ERROR: + qWarning() << "Error while decompressing stream:" << status; + emit error(StreamError); + return; + case Z_BUF_ERROR: + // means that we need more input to continue, so this is not an actual error + return; + case Z_STREAM_END: + qWarning() << "Reached end of zlib stream!"; // this should really never happen + return; + default: + // just try to get more out of the stream + break; + } + } + // qDebug() << "inflate in:" << _inflater->total_in << "out:" << _inflater->total_out << "ratio:" << (double)_inflater->total_in/_inflater->total_out; +} void Compressor::writeData() { if (compressionLevel() == NoCompression) { _socket->write(_writeBuffer); _writeBuffer.clear(); + return; } -} + _deflater->next_in = reinterpret_cast(_writeBuffer.data()); + _deflater->avail_in = _writeBuffer.size(); + + int status; + do { + _deflater->next_out = reinterpret_cast(_outputBuffer.data()); + _deflater->avail_out = ioBufferSize; + status = deflate(_deflater, Z_PARTIAL_FLUSH); + if (status != Z_OK && status != Z_BUF_ERROR) { + qWarning() << "Error while compressing stream:" << status; + emit error(StreamError); + return; + } + + if (_deflater->avail_out == static_cast(ioBufferSize)) + continue; // nothing to write here + + if (!_socket->write(_outputBuffer.constData(), ioBufferSize - _deflater->avail_out)) { + qWarning() << "Error while writing to socket:" << _socket->errorString(); + emit error(DeviceError); + return; + } + } while (_deflater->avail_out == 0); // the output buffer being full is the only reason we should have to loop here! + + if (_deflater->avail_in > 0) { + qWarning() << "Oops, something weird happened: data still remaining in write buffer!"; + emit error(StreamError); + } + + _writeBuffer.resize(0); + + // qDebug() << "deflate in:" << _deflater->total_in << "out:" << _deflater->total_out << "ratio:" << (double)_deflater->total_out/_deflater->total_in; +} void Compressor::flush() { if (compressionLevel() == NoCompression && _socket->state() == QAbstractSocket::ConnectedState) _socket->flush(); + // FIXME: missing impl for enabled compression; but then we're not using this method yet }