From 7b76379a89158f10780cbfc74965027246faec68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Nordheim?= Date: Thu, 7 May 2020 17:34:22 +0200 Subject: Make use of QDecompressHelper for HTTP downloads Changes are not too big for now. Just replaces use of the previous calls to the zlib decompression function. And initialize QDecompressHelper when we know the content-encoding. Task-number: QTBUG-83269 Change-Id: I41358feaef2e7ac5f48f14e3f95ec094e0c110b7 Reviewed-by: Timur Pocheptsov --- src/network/access/qdecompresshelper.cpp | 13 ++ src/network/access/qdecompresshelper_p.h | 1 + src/network/access/qhttp2protocolhandler.cpp | 22 ++- src/network/access/qhttpnetworkconnection.cpp | 3 +- src/network/access/qhttpnetworkreply.cpp | 147 ++++----------------- src/network/access/qhttpnetworkreply_p.h | 16 +-- .../qdecompresshelper/tst_qdecompresshelper.cpp | 8 +- 7 files changed, 69 insertions(+), 141 deletions(-) diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp index b478e220ea..5959a736d5 100644 --- a/src/network/access/qdecompresshelper.cpp +++ b/src/network/access/qdecompresshelper.cpp @@ -79,6 +79,19 @@ bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding) return encodingFromByteArray(encoding) != QDecompressHelper::None; } +QByteArrayList QDecompressHelper::acceptedEncoding() +{ + static QByteArrayList accepted = []() { + QByteArrayList list; + list.reserve(sizeof(contentEncodingMapping) / sizeof(contentEncodingMapping[0])); + for (const auto &mapping : contentEncodingMapping) { + list << QByteArray(mapping.name); + } + return list; + }(); + return accepted; +} + QDecompressHelper::~QDecompressHelper() { clear(); diff --git a/src/network/access/qdecompresshelper_p.h b/src/network/access/qdecompresshelper_p.h index 5252925862..9d8ca245e1 100644 --- a/src/network/access/qdecompresshelper_p.h +++ b/src/network/access/qdecompresshelper_p.h @@ -90,6 +90,7 @@ public: void clear(); static bool isSupportedEncoding(const QByteArray &encoding); + static QByteArrayList acceptedEncoding(); private: bool countInternal(); diff --git a/src/network/access/qhttp2protocolhandler.cpp b/src/network/access/qhttp2protocolhandler.cpp index f71a4e9bd4..44c397c882 100644 --- a/src/network/access/qhttp2protocolhandler.cpp +++ b/src/network/access/qhttp2protocolhandler.cpp @@ -1163,8 +1163,11 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const HPack::HttpHeader if (QHttpNetworkReply::isHttpRedirect(statusCode) && redirectUrl.isValid()) httpReply->setRedirectUrl(redirectUrl); - if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress) + if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress) { httpReplyPrivate->removeAutoDecompressHeader(); + httpReplyPrivate->decompressHelper.setEncoding( + httpReplyPrivate->headerField("content-encoding")); + } if (QHttpNetworkReply::isHttpRedirect(statusCode) || statusCode == 401 || statusCode == 407) { @@ -1207,12 +1210,17 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const Frame &frame, const QByteArray wrapped(data, length); if (httpRequest.d->autoDecompress && replyPrivate->isCompressed()) { - QByteDataBuffer inDataBuffer; - inDataBuffer.append(wrapped); - replyPrivate->uncompressBodyData(&inDataBuffer, &replyPrivate->responseData); - // Now, make sure replyPrivate's destructor will properly clean up - // buffers allocated (if any) by zlib. - replyPrivate->autoDecompress = true; + Q_ASSERT(replyPrivate->decompressHelper.isValid()); + + replyPrivate->decompressHelper.feed(wrapped); + while (replyPrivate->decompressHelper.hasData()) { + QByteArray output(4 * 1024, Qt::Uninitialized); + qint64 read = replyPrivate->decompressHelper.read(output.data(), output.size()); + if (read > 0) { + output.resize(read); + replyPrivate->responseData.append(std::move(output)); + } + } } else { replyPrivate->responseData.append(wrapped); } diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp index 468a530e7b..8c108689a5 100644 --- a/src/network/access/qhttpnetworkconnection.cpp +++ b/src/network/access/qhttpnetworkconnection.cpp @@ -297,7 +297,8 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair) value = request.headerField("accept-encoding"); if (value.isEmpty()) { #ifndef QT_NO_COMPRESS - request.setHeaderField("Accept-Encoding", "gzip, deflate"); + const QByteArrayList &acceptedEncoding = QDecompressHelper::acceptedEncoding(); + request.setHeaderField("Accept-Encoding", acceptedEncoding.join(", ")); request.d->autoDecompress = true; #else // if zlib is not available set this to false always diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp index 66a431c5f9..e11ea401d2 100644 --- a/src/network/access/qhttpnetworkreply.cpp +++ b/src/network/access/qhttpnetworkreply.cpp @@ -46,10 +46,6 @@ # include #endif -#ifndef QT_NO_COMPRESS -#include -#endif - QT_BEGIN_NAMESPACE QHttpNetworkReply::QHttpNetworkReply(const QUrl &url, QObject *parent) @@ -63,11 +59,6 @@ QHttpNetworkReply::~QHttpNetworkReply() if (d->connection) { d->connection->d_func()->removeReply(this); } - -#ifndef QT_NO_COMPRESS - if (d->autoDecompress && d->isCompressed() && d->inflateStrm) - inflateEnd(d->inflateStrm); -#endif } QUrl QHttpNetworkReply::url() const @@ -335,9 +326,6 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl) autoDecompress(false), responseData(), requestIsPrepared(false) ,pipeliningUsed(false), h2Used(false), downstreamLimited(false) ,userProvidedDownloadBuffer(nullptr) -#ifndef QT_NO_COMPRESS - ,inflateStrm(nullptr) -#endif { QString scheme = newUrl.scheme(); @@ -347,13 +335,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl) connectionCloseEnabled = false; } -QHttpNetworkReplyPrivate::~QHttpNetworkReplyPrivate() -{ -#ifndef QT_NO_COMPRESS - if (inflateStrm) - delete inflateStrm; -#endif -} +QHttpNetworkReplyPrivate::~QHttpNetworkReplyPrivate() = default; void QHttpNetworkReplyPrivate::clearHttpLayerInformation() { @@ -366,10 +348,7 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation() currentChunkRead = 0; lastChunkRead = false; connectionCloseEnabled = true; -#ifndef QT_NO_COMPRESS - if (autoDecompress && inflateStrm) - inflateEnd(inflateStrm); -#endif + decompressHelper.clear(); fields.clear(); } @@ -388,11 +367,15 @@ qint64 QHttpNetworkReplyPrivate::bytesAvailable() const return (state != ReadingDataState ? 0 : fragment.size()); } -bool QHttpNetworkReplyPrivate::isCompressed() +bool QHttpNetworkReplyPrivate::isCompressed() const +{ + return QDecompressHelper::isSupportedEncoding(headerField("content-encoding")); +} + +bool QHttpNetworkReply::isCompressed() const { - QByteArray encoding = headerField("content-encoding"); - return encoding.compare("gzip", Qt::CaseInsensitive) == 0 || - encoding.compare("deflate", Qt::CaseInsensitive) == 0; + Q_D(const QHttpNetworkReply); + return d->isCompressed(); } void QHttpNetworkReplyPrivate::removeAutoDecompressHeader() @@ -596,18 +579,10 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket) headerField("proxy-connection").toLower().contains("close")) || (majorVersion == 1 && minorVersion == 0 && (connectionHeaderField.isEmpty() && !headerField("proxy-connection").toLower().contains("keep-alive"))); - -#ifndef QT_NO_COMPRESS if (autoDecompress && isCompressed()) { - // allocate inflate state - if (!inflateStrm) - inflateStrm = new z_stream; - int ret = initializeInflateStream(); - if (ret != Z_OK) - return -1; + if (!decompressHelper.setEncoding(headerField("content-encoding"))) + return -1; // Either the encoding was unsupported or the decoder could not be set up } -#endif - } return bytes; } @@ -709,12 +684,8 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff { qint64 bytes = 0; -#ifndef QT_NO_COMPRESS - // for gzip we'll allocate a temporary one that we then decompress + // for compressed data we'll allocate a temporary one that we then decompress QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out); -#else - QByteDataBuffer *tempOutDataBuffer = out; -#endif if (isChunked()) { @@ -730,94 +701,28 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable()); } -#ifndef QT_NO_COMPRESS // This is true if there is compressed encoding and we're supposed to use it. if (autoDecompress) { - qint64 uncompressRet = uncompressBodyData(tempOutDataBuffer, out); - delete tempOutDataBuffer; - if (uncompressRet < 0) + QScopedPointer holder(tempOutDataBuffer); + if (!decompressHelper.isValid()) return -1; - } -#endif - contentRead += bytes; - return bytes; -} - -#ifndef QT_NO_COMPRESS -int QHttpNetworkReplyPrivate::initializeInflateStream() -{ - Q_ASSERT(inflateStrm); - - inflateStrm->zalloc = Z_NULL; - inflateStrm->zfree = Z_NULL; - inflateStrm->opaque = Z_NULL; - inflateStrm->avail_in = 0; - inflateStrm->next_in = Z_NULL; - // "windowBits can also be greater than 15 for optional gzip decoding. - // Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection" - // http://www.zlib.net/manual.html - int ret = inflateInit2(inflateStrm, MAX_WBITS+32); - Q_ASSERT(ret == Z_OK); - return ret; -} - -qint64 QHttpNetworkReplyPrivate::uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out) -{ - if (!inflateStrm) { // happens when called from the SPDY protocol handler - inflateStrm = new z_stream; - initializeInflateStream(); - } - - if (!inflateStrm) - return -1; - - bool triedRawDeflate = false; - for (int i = 0; i < in->bufferCount(); i++) { - QByteArray &bIn = (*in)[i]; - - inflateStrm->avail_in = bIn.size(); - inflateStrm->next_in = reinterpret_cast(bIn.data()); - - do { - QByteArray bOut; - // make a wild guess about the uncompressed size. - bOut.reserve(inflateStrm->avail_in * 3 + 512); - inflateStrm->avail_out = bOut.capacity(); - inflateStrm->next_out = reinterpret_cast(bOut.data()); - - int ret = inflate(inflateStrm, Z_NO_FLUSH); - //All negative return codes are errors, in the context of HTTP compression, Z_NEED_DICT is also an error. - // in the case where we get Z_DATA_ERROR this could be because we received raw deflate compressed data. - if (ret == Z_DATA_ERROR && !triedRawDeflate) { - inflateEnd(inflateStrm); - triedRawDeflate = true; - inflateStrm->zalloc = Z_NULL; - inflateStrm->zfree = Z_NULL; - inflateStrm->opaque = Z_NULL; - inflateStrm->avail_in = 0; - inflateStrm->next_in = Z_NULL; - int ret = inflateInit2(inflateStrm, -MAX_WBITS); - if (ret != Z_OK) { - return -1; - } else { - inflateStrm->avail_in = bIn.size(); - inflateStrm->next_in = reinterpret_cast(bIn.data()); - continue; - } - } else if (ret < 0 || ret == Z_NEED_DICT) { + decompressHelper.feed(std::move(*tempOutDataBuffer)); + while (decompressHelper.hasData()) { + QByteArray output(4 * 1024, Qt::Uninitialized); + qint64 read = decompressHelper.read(output.data(), output.size()); + if (read < 0) { return -1; + } else if (read > 0) { + output.resize(read); + out->append(std::move(output)); } - bOut.resize(bOut.capacity() - inflateStrm->avail_out); - out->append(bOut); - if (ret == Z_STREAM_END) - return out->byteAmount(); - } while (inflateStrm->avail_in > 0); + } } - return out->byteAmount(); + contentRead += bytes; + return bytes; } -#endif qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QAbstractSocket *socket, QByteDataBuffer *out, qint64 size) { diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h index f8b45a8adc..7a23d310cc 100644 --- a/src/network/access/qhttpnetworkreply_p.h +++ b/src/network/access/qhttpnetworkreply_p.h @@ -55,10 +55,6 @@ #include -#ifndef QT_NO_COMPRESS -struct z_stream_s; -#endif - #include // it's safe to include these even if SSL support is not enabled #include @@ -80,6 +76,8 @@ Q_MOC_INCLUDE() #endif Q_MOC_INCLUDE() +#include + QT_REQUIRE_CONFIG(http); QT_BEGIN_NAMESPACE @@ -157,6 +155,8 @@ public: static bool isHttpRedirect(int statusCode); + bool isCompressed() const; + #ifndef QT_NO_SSL QSslConfiguration sslConfiguration() const; void setSslConfiguration(const QSslConfiguration &config); @@ -224,7 +224,7 @@ public: bool isChunked(); bool isConnectionCloseEnabled(); - bool isCompressed(); + bool isCompressed() const; void removeAutoDecompressHeader(); enum ReplyState { @@ -276,11 +276,7 @@ public: char* userProvidedDownloadBuffer; QUrl redirectUrl; -#ifndef QT_NO_COMPRESS - z_stream_s *inflateStrm; - int initializeInflateStream(); - qint64 uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out); -#endif + QDecompressHelper decompressHelper; }; diff --git a/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp b/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp index 0b302982a6..4f54839ec0 100644 --- a/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp +++ b/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp @@ -82,11 +82,15 @@ void tst_QDecompressHelper::cleanupTestCase() void tst_QDecompressHelper::encodingSupported() { - QVERIFY(!QDecompressHelper::isSupportedEncoding("identity")); - QVERIFY(!QDecompressHelper::isSupportedEncoding("fake")); + const QByteArrayList &accepted = QDecompressHelper::acceptedEncoding(); QVERIFY(QDecompressHelper::isSupportedEncoding("deflate")); + QVERIFY(accepted.contains("deflate")); QVERIFY(QDecompressHelper::isSupportedEncoding("gzip")); + QVERIFY(accepted.contains("gzip")); + int expected = 2; + + QCOMPARE(expected, accepted.size()); } void tst_QDecompressHelper::sharedDecompress_data() -- cgit v1.2.3