From 2b5dcfcee118265ef7930cc7c5c16bff22d580fd Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 6 Jul 2011 16:08:59 +0200 Subject: QNAM HTTP: Re-write compression code This eliminates some code (header parsing) that can be done by zlib already. Add support for 'deflate' encoding. Also do less memory copying while uncompressing. Change-Id: I94de21e3c58b904dd91d004c375ed8cbea56cb0b Task-Number: QTBUG-13191 Reviewed-on: http://codereview.qt.nokia.com/1314 Reviewed-by: Qt Sanity Bot Reviewed-by: Martin Petersson Reviewed-by: Peter Hartmann Reviewed-by: Markus Goetz --- src/network/access/qhttpnetworkconnection.cpp | 2 +- src/network/access/qhttpnetworkconnection_p.h | 3 - .../access/qhttpnetworkconnectionchannel.cpp | 82 +------ .../access/qhttpnetworkconnectionchannel_p.h | 1 - src/network/access/qhttpnetworkreply.cpp | 235 +++++++-------------- src/network/access/qhttpnetworkreply_p.h | 26 +-- 6 files changed, 100 insertions(+), 249 deletions(-) diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp index a8a4fd9ae7..0f1132e2a8 100644 --- a/src/network/access/qhttpnetworkconnection.cpp +++ b/src/network/access/qhttpnetworkconnection.cpp @@ -271,7 +271,7 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair) value = request.headerField("accept-encoding"); if (value.isEmpty()) { #ifndef QT_NO_COMPRESS - request.setHeaderField("Accept-Encoding", "gzip"); + request.setHeaderField("Accept-Encoding", "gzip, deflate"); request.d->autoDecompress = true; #else // if zlib is not available set this to false always diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h index 0c86fd94b9..0652436b35 100644 --- a/src/network/access/qhttpnetworkconnection_p.h +++ b/src/network/access/qhttpnetworkconnection_p.h @@ -202,9 +202,6 @@ public: QString errorDetail(QNetworkReply::NetworkError errorCode, QAbstractSocket *socket, const QString &extraDetail = QString()); -#ifndef QT_NO_COMPRESS - bool expand(QAbstractSocket *socket, QHttpNetworkReply *reply, bool dataComplete); -#endif void removeReply(QHttpNetworkReply *reply); QString hostName; diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp index aafdbf7774..8be876dd48 100644 --- a/src/network/access/qhttpnetworkconnectionchannel.cpp +++ b/src/network/access/qhttpnetworkconnectionchannel.cpp @@ -403,7 +403,7 @@ void QHttpNetworkConnectionChannel::_q_receiveReply() bytes += headerBytes; // If headers were parsed successfully now it is the ReadingDataState if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) { - if (replyPrivate->isGzipped() && replyPrivate->autoDecompress) { + if (replyPrivate->isCompressed() && replyPrivate->autoDecompress) { // remove the Content-Length from header replyPrivate->removeAutoDecompressHeader(); } else { @@ -475,30 +475,18 @@ void QHttpNetworkConnectionChannel::_q_receiveReply() { // use the traditional slower reading (for compressed encoding, chunked encoding, // no content-length etc) - QByteDataBuffer byteDatas; - qint64 haveRead = replyPrivate->readBody(socket, &byteDatas); - if (haveRead) { + qint64 haveRead = replyPrivate->readBody(socket, &replyPrivate->responseData); + if (haveRead > 0) { bytes += haveRead; - if (replyPrivate->autoDecompress) - replyPrivate->appendCompressedReplyData(byteDatas); - else - replyPrivate->appendUncompressedReplyData(byteDatas); - - if (!replyPrivate->autoDecompress) { - replyPrivate->totalProgress += bytes; - if (replyPrivate->shouldEmitSignals()) { - // important: At the point of this readyRead(), the byteDatas list must be empty, - // else implicit sharing will trigger memcpy when the user is reading data! - emit reply->readyRead(); - emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); - } + replyPrivate->totalProgress += haveRead; + if (replyPrivate->shouldEmitSignals()) { + emit reply->readyRead(); + emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength); } -#ifndef QT_NO_COMPRESS - else if (!expand(false)) { // expand a chunk if possible - // If expand() failed we can just return, it had already called connection->emitReplyError() - return; - } -#endif + } else if (haveRead == -1) { + // Some error occured + connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure); + break; } } // still in ReadingDataState? This function will be called again by the socket's readyRead @@ -638,57 +626,9 @@ bool QHttpNetworkConnectionChannel::ensureConnection() return true; } - -#ifndef QT_NO_COMPRESS -bool QHttpNetworkConnectionChannel::expand(bool dataComplete) -{ - Q_ASSERT(socket); - Q_ASSERT(reply); - - qint64 total = reply->d_func()->compressedData.size(); - if (total >= CHUNK || dataComplete) { - // uncompress the data - QByteArray content, inflated; - content = reply->d_func()->compressedData; - reply->d_func()->compressedData.clear(); - - int ret = Z_OK; - if (content.size()) - ret = reply->d_func()->gunzipBodyPartially(content, inflated); - int retCheck = (dataComplete) ? Z_STREAM_END : Z_OK; - if (ret >= retCheck) { - if (inflated.size()) { - reply->d_func()->totalProgress += inflated.size(); - reply->d_func()->appendUncompressedReplyData(inflated); - if (reply->d_func()->shouldEmitSignals()) { - // important: At the point of this readyRead(), inflated must be cleared, - // else implicit sharing will trigger memcpy when the user is reading data! - emit reply->readyRead(); - emit reply->dataReadProgress(reply->d_func()->totalProgress, 0); - } - } - } else { - connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure); - return false; - } - } - return true; -} -#endif - - void QHttpNetworkConnectionChannel::allDone() { Q_ASSERT(reply); -#ifndef QT_NO_COMPRESS - // expand the whole data. - if (reply->d_func()->expectContent() && reply->d_func()->autoDecompress && !reply->d_func()->streamEnd) { - bool expandResult = expand(true); - // If expand() failed we can just return, it had already called connection->emitReplyError() - if (!expandResult) - return; - } -#endif if (!reply) { qWarning() << "QHttpNetworkConnectionChannel::allDone() called without reply. Please report at http://bugreports.qt.nokia.com/"; diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h index c159f1a0c5..7a4dd07db1 100644 --- a/src/network/access/qhttpnetworkconnectionchannel_p.h +++ b/src/network/access/qhttpnetworkconnectionchannel_p.h @@ -149,7 +149,6 @@ public: bool ensureConnection(); - bool expand(bool dataComplete); void allDone(); // reply header + body have been read void handleStatus(); // called from allDone() diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp index 04bcd06908..5b174dbc05 100644 --- a/src/network/access/qhttpnetworkreply.cpp +++ b/src/network/access/qhttpnetworkreply.cpp @@ -65,6 +65,11 @@ QHttpNetworkReply::~QHttpNetworkReply() if (d->connection) { d->connection->d_func()->removeReply(this); } + +#ifndef QT_NO_COMPRESS + if (d->autoDecompress && d->isCompressed()) + inflateEnd(&d->inflateStrm); +#endif } QUrl QHttpNetworkReply::url() const @@ -252,7 +257,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl) chunkedTransferEncoding(false), connectionCloseEnabled(true), forceConnectionCloseEnabled(false), - currentChunkSize(0), currentChunkRead(0), connection(0), initInflate(false), + currentChunkSize(0), currentChunkRead(0), connection(0), autoDecompress(false), responseData(), requestIsPrepared(false) ,pipeliningUsed(false), downstreamLimited(false) ,userProvidedDownloadBuffer(0) @@ -274,11 +279,9 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation() currentChunkRead = 0; connectionCloseEnabled = true; #ifndef QT_NO_COMPRESS - if (initInflate) + if (autoDecompress) inflateEnd(&inflateStrm); #endif - initInflate = false; - streamEnd = false; fields.clear(); } @@ -297,10 +300,10 @@ qint64 QHttpNetworkReplyPrivate::bytesAvailable() const return (state != ReadingDataState ? 0 : fragment.size()); } -bool QHttpNetworkReplyPrivate::isGzipped() +bool QHttpNetworkReplyPrivate::isCompressed() { QByteArray encoding = headerField("content-encoding"); - return qstricmp(encoding.constData(), "gzip") == 0; + return qstricmp(encoding.constData(), "gzip") == 0 || qstricmp(encoding.constData(), "deflate") == 0; } void QHttpNetworkReplyPrivate::removeAutoDecompressHeader() @@ -358,120 +361,6 @@ QAuthenticatorPrivate::Method QHttpNetworkReplyPrivate::authenticationMethod(boo return method; } -#ifndef QT_NO_COMPRESS -bool QHttpNetworkReplyPrivate::gzipCheckHeader(QByteArray &content, int &pos) -{ - int method = 0; // method byte - int flags = 0; // flags byte - bool ret = false; - - // Assure two bytes in the buffer so we can peek ahead -- handle case - // where first byte of header is at the end of the buffer after the last - // gzip segment - pos = -1; - QByteArray &body = content; - int maxPos = body.size()-1; - if (maxPos < 1) { - return ret; - } - - // Peek ahead to check the gzip magic header - if (body[0] != char(gz_magic[0]) || - body[1] != char(gz_magic[1])) { - return ret; - } - pos += 2; - // Check the rest of the gzip header - if (++pos <= maxPos) - method = body[pos]; - if (pos++ <= maxPos) - flags = body[pos]; - if (method != Z_DEFLATED || (flags & RESERVED) != 0) { - return ret; - } - - // Discard time, xflags and OS code: - pos += 6; - if (pos > maxPos) - return ret; - if ((flags & EXTRA_FIELD) && ((pos+2) <= maxPos)) { // skip the extra field - unsigned len = (unsigned)body[++pos]; - len += ((unsigned)body[++pos])<<8; - pos += len; - if (pos > maxPos) - return ret; - } - if ((flags & ORIG_NAME) != 0) { // skip the original file name - while(++pos <= maxPos && body[pos]) {} - } - if ((flags & COMMENT) != 0) { // skip the .gz file comment - while(++pos <= maxPos && body[pos]) {} - } - if ((flags & HEAD_CRC) != 0) { // skip the header crc - pos += 2; - if (pos > maxPos) - return ret; - } - ret = (pos < maxPos); // return failed, if no more bytes left - return ret; -} - -int QHttpNetworkReplyPrivate::gunzipBodyPartially(QByteArray &compressed, QByteArray &inflated) -{ - int ret = Z_DATA_ERROR; - unsigned have; - unsigned char out[CHUNK]; - int pos = -1; - - if (!initInflate) { - // check the header - if (!gzipCheckHeader(compressed, pos)) - return ret; - // allocate inflate state - inflateStrm.zalloc = Z_NULL; - inflateStrm.zfree = Z_NULL; - inflateStrm.opaque = Z_NULL; - inflateStrm.avail_in = 0; - inflateStrm.next_in = Z_NULL; - ret = inflateInit2(&inflateStrm, -MAX_WBITS); - if (ret != Z_OK) - return ret; - initInflate = true; - streamEnd = false; - } - - //remove the header. - compressed.remove(0, pos+1); - // expand until deflate stream ends - inflateStrm.next_in = (unsigned char *)compressed.data(); - inflateStrm.avail_in = compressed.size(); - do { - inflateStrm.avail_out = sizeof(out); - inflateStrm.next_out = out; - ret = inflate(&inflateStrm, Z_NO_FLUSH); - switch (ret) { - case Z_NEED_DICT: - ret = Z_DATA_ERROR; - // and fall through - case Z_DATA_ERROR: - case Z_MEM_ERROR: - inflateEnd(&inflateStrm); - initInflate = false; - return ret; - } - have = sizeof(out) - inflateStrm.avail_out; - inflated.append(QByteArray((const char *)out, have)); - } while (inflateStrm.avail_out == 0); - // clean up and return - if (ret <= Z_ERRNO || ret == Z_STREAM_END) { - inflateEnd(&inflateStrm); - initInflate = false; - } - streamEnd = (ret == Z_STREAM_END); - return ret; -} -#endif - qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket) { if (fragment.isEmpty()) { @@ -616,6 +505,24 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket) connectionCloseEnabled = (connectionHeaderField.toLower().contains("close") || headerField("proxy-connection").toLower().contains("close")) || (majorVersion == 1 && minorVersion == 0 && connectionHeaderField.isEmpty()); + +#ifndef QT_NO_COMPRESS + if (autoDecompress && isCompressed()) { + // allocate inflate state + inflateStrm.zalloc = Z_NULL; + inflateStrm.zfree = Z_NULL; + inflateStrm.opaque = Z_NULL; + inflateStrm.avail_in = 0; + inflateStrm.next_in = Z_NULL; + // "windowBits can also be greater than 15 for optional gzip decoding. + // Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection" + // http://www.zlib.net/manual.html + int ret = inflateInit2(&inflateStrm, MAX_WBITS+32); + if (ret != Z_OK) + return -1; + } +#endif + } return bytes; } @@ -712,22 +619,74 @@ qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteData qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuffer *out) { qint64 bytes = 0; + +#ifndef QT_NO_COMPRESS + // for gzip we'll allocate a temporary one that we then decompress + QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out); +#else + QByteDataBuffer *tempOutDataBuffer = out; +#endif + + if (isChunked()) { // chunked transfer encoding (rfc 2616, sec 3.6) - bytes += readReplyBodyChunked(socket, out); + bytes += readReplyBodyChunked(socket, tempOutDataBuffer); } else if (bodyLength > 0) { // we have a Content-Length - bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead); + bytes += readReplyBodyRaw(socket, tempOutDataBuffer, bodyLength - contentRead); if (contentRead + bytes == bodyLength) state = AllDoneState; } else { // no content length. just read what's possible - bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable()); + bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable()); + } + +#ifndef QT_NO_COMPRESS + // This is true if there is compressed encoding and we're supposed to use it. + if (autoDecompress) { + qint64 uncompressRet = uncompressBodyData(tempOutDataBuffer, out); + delete tempOutDataBuffer; + if (uncompressRet < 0) + return -1; } +#endif + contentRead += bytes; return bytes; } +#ifndef QT_NO_COMPRESS +qint64 QHttpNetworkReplyPrivate::uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out) +{ + for (int i = 0; i < in->bufferCount(); i++) { + QByteArray &bIn = (*in)[i]; + + inflateStrm.avail_in = bIn.size(); + inflateStrm.next_in = reinterpret_cast(bIn.data()); + + do { + QByteArray bOut; + // make a wild guess about the uncompressed size. + bOut.reserve(inflateStrm.avail_in * 3 + 512); + inflateStrm.avail_out = bOut.capacity(); + inflateStrm.next_out = reinterpret_cast(bOut.data()); + + int ret = inflate(&inflateStrm, Z_NO_FLUSH); + switch (ret) { + case Z_NEED_DICT: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + return -1; + } + bOut.resize(bOut.capacity() - inflateStrm.avail_out); + out->append(bOut); + } while (inflateStrm.avail_in > 0); + } + + return out->byteAmount(); +} +#endif + qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QAbstractSocket *socket, QByteDataBuffer *out, qint64 size) { // FIXME get rid of this function and just use readBodyFast and give it socket->bytesAvailable() @@ -841,36 +800,6 @@ qint64 QHttpNetworkReplyPrivate::getChunkSize(QAbstractSocket *socket, qint64 *c return bytes; } -void QHttpNetworkReplyPrivate::appendUncompressedReplyData(QByteArray &qba) -{ - responseData.append(qba); - - // clear the original! helps with implicit sharing and - // avoiding memcpy when the user is reading the data - qba.clear(); -} - -void QHttpNetworkReplyPrivate::appendUncompressedReplyData(QByteDataBuffer &data) -{ - responseData.append(data); - - // clear the original! helps with implicit sharing and - // avoiding memcpy when the user is reading the data - data.clear(); -} - -void QHttpNetworkReplyPrivate::appendCompressedReplyData(QByteDataBuffer &data) -{ - // Work in progress: Later we will directly use a list of QByteArray or a QRingBuffer - // instead of one QByteArray. - for(int i = 0; i < data.bufferCount(); i++) { - QByteArray &byteData = data[i]; - compressedData.append(byteData.constData(), byteData.size()); - } - data.clear(); -} - - bool QHttpNetworkReplyPrivate::shouldEmitSignals() { // for 401 & 407 don't emit the data signals. Content along with these diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h index 583a256ada..14219d484b 100644 --- a/src/network/access/qhttpnetworkreply_p.h +++ b/src/network/access/qhttpnetworkreply_p.h @@ -56,15 +56,7 @@ #ifndef QT_NO_HTTP #ifndef QT_NO_COMPRESS -# include -static const unsigned char gz_magic[2] = {0x1f, 0x8b}; // gzip magic header -// gzip flag byte -#define HEAD_CRC 0x02 // bit 1 set: header CRC present -#define EXTRA_FIELD 0x04 // bit 2 set: extra field present -#define ORIG_NAME 0x08 // bit 3 set: original file name present -#define COMMENT 0x10 // bit 4 set: file comment present -#define RESERVED 0xE0 // bits 5..7: reserved -#define CHUNK 16384 +#include #endif #include @@ -192,10 +184,6 @@ public: qint64 readReplyBodyChunked(QAbstractSocket *in, QByteDataBuffer *out); qint64 getChunkSize(QAbstractSocket *in, qint64 *chunkSize); - void appendUncompressedReplyData(QByteArray &qba); - void appendUncompressedReplyData(QByteDataBuffer &data); - void appendCompressedReplyData(QByteDataBuffer &data); - bool shouldEmitSignals(); bool expectContent(); void eraseData(); @@ -203,11 +191,8 @@ public: qint64 bytesAvailable() const; bool isChunked(); bool isConnectionCloseEnabled(); - bool isGzipped(); -#ifndef QT_NO_COMPRESS - bool gzipCheckHeader(QByteArray &content, int &pos); - int gunzipBodyPartially(QByteArray &compressed, QByteArray &inflated); -#endif + + bool isCompressed(); void removeAutoDecompressHeader(); enum ReplyState { @@ -236,11 +221,12 @@ public: qint64 currentChunkRead; QPointer connection; QPointer connectionChannel; - bool initInflate; - bool streamEnd; + #ifndef QT_NO_COMPRESS z_stream inflateStrm; + qint64 uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out); #endif + bool autoDecompress; QByteDataBuffer responseData; // uncompressed body -- cgit v1.2.3