From ad1a5bf63fa63532c3267d4f365286dc78288651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Nordheim?= Date: Wed, 6 May 2020 18:24:07 +0200 Subject: QDecompressHelper: Add brotli support Task-number: QTBUG-83269 Change-Id: If23b098ee76a4892e4c2c6ce5c635688d8d9138d Reviewed-by: Timur Pocheptsov --- src/network/.prev_CMakeLists.txt | 5 ++ src/network/CMakeLists.txt | 5 ++ src/network/access/access.pri | 4 ++ src/network/access/qdecompresshelper.cpp | 115 +++++++++++++++++++++++++++++++ src/network/access/qdecompresshelper_p.h | 6 ++ src/network/configure.cmake | 9 +++ src/network/configure.json | 25 ++++++- 7 files changed, 168 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/network/.prev_CMakeLists.txt b/src/network/.prev_CMakeLists.txt index 8f7e9c07be..b677b8020a 100644 --- a/src/network/.prev_CMakeLists.txt +++ b/src/network/.prev_CMakeLists.txt @@ -123,6 +123,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_http socket/qhttpsocketengine.cpp socket/qhttpsocketengine_p.h ) +qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http + LIBRARIES + WrapBrotli::WrapBrotliDec +) + qt_extend_target(Network CONDITION QT_FEATURE_system_zlib LIBRARIES ZLIB::ZLIB diff --git a/src/network/CMakeLists.txt b/src/network/CMakeLists.txt index c421622a44..157a31f1da 100644 --- a/src/network/CMakeLists.txt +++ b/src/network/CMakeLists.txt @@ -123,6 +123,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_http socket/qhttpsocketengine.cpp socket/qhttpsocketengine_p.h ) +qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http + LIBRARIES + WrapBrotli::WrapBrotliDec +) + qt_extend_target(Network CONDITION QT_FEATURE_system_zlib LIBRARIES ZLIB::ZLIB diff --git a/src/network/access/access.pri b/src/network/access/access.pri index 4add94a111..bb8f155d23 100644 --- a/src/network/access/access.pri +++ b/src/network/access/access.pri @@ -116,4 +116,8 @@ qtConfig(http) { access/qhttpthreaddelegate_p.h \ access/qnetworkreplyhttpimpl_p.h \ access/qhttp2configuration.h + + qtConfig(brotli) { + QMAKE_USE_PRIVATE += brotli + } } diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp index 5959a736d5..e56a411fc1 100644 --- a/src/network/access/qdecompresshelper.cpp +++ b/src/network/access/qdecompresshelper.cpp @@ -44,6 +44,10 @@ #include +#if QT_CONFIG(brotli) +# include +#endif + #include QT_BEGIN_NAMESPACE @@ -57,6 +61,9 @@ struct ContentEncodingMapping constexpr ContentEncodingMapping contentEncodingMapping[] { { "deflate", QDecompressHelper::Deflate }, { "gzip", QDecompressHelper::GZip }, +#if QT_CONFIG(brotli) + { "br", QDecompressHelper::Brotli }, +#endif }; QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept @@ -72,6 +79,13 @@ z_stream *toZlibPointer(void *ptr) { return static_cast(ptr); } + +#if QT_CONFIG(brotli) +BrotliDecoderState *toBrotliPointer(void *ptr) +{ + return static_cast(ptr); +} +#endif } bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding) @@ -134,6 +148,13 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce) decoderPointer = inflateStream; break; } + case Brotli: +#if QT_CONFIG(brotli) + decoderPointer = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); +#else + Q_UNREACHABLE(); +#endif + break; } if (!decoderPointer) { qWarning("Failed to initialize the decoder."); @@ -325,6 +346,9 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) case GZip: bytesRead = readZLib(data, maxSize); break; + case Brotli: + bytesRead = readBrotli(data, maxSize); + break; } if (bytesRead == -1) clear(); @@ -369,6 +393,14 @@ void QDecompressHelper::clear() delete inflateStream; break; } + case Brotli: { +#if QT_CONFIG(brotli) + BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer); + if (brotliDecoderState) + BrotliDecoderDestroyInstance(brotliDecoderState); +#endif + break; + } } decoderPointer = nullptr; contentEncoding = None; @@ -489,4 +521,87 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) return bytesDecoded; } +qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) +{ +#if !QT_CONFIG(brotli) + Q_UNUSED(data); + Q_UNUSED(maxSize); + Q_UNREACHABLE(); +#else + qint64 bytesDecoded = 0; + + BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer); + + while (decoderHasData && bytesDecoded < maxSize) { + Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState)); + if (brotliUnconsumedDataPtr) { + Q_ASSERT(brotliUnconsumedAmount); + size_t toRead = std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount); + memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead); + bytesDecoded += toRead; + brotliUnconsumedAmount -= toRead; + brotliUnconsumedDataPtr += toRead; + if (brotliUnconsumedAmount == 0) { + brotliUnconsumedDataPtr = nullptr; + decoderHasData = false; + } + } + if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) { + brotliUnconsumedDataPtr = + BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount); + decoderHasData = true; + } + } + if (bytesDecoded == maxSize) + return bytesDecoded; + Q_ASSERT(bytesDecoded < maxSize); + + QByteArray input; + if (!compressedDataBuffer.isEmpty()) + input = compressedDataBuffer.read(); + const uint8_t *encodedPtr = reinterpret_cast(input.constData()); + size_t encodedBytesRemaining = input.size(); + + uint8_t *decodedPtr = reinterpret_cast(data + bytesDecoded); + size_t unusedDecodedSize = size_t(maxSize - bytesDecoded); + while (unusedDecodedSize > 0) { + auto previousUnusedDecodedSize = unusedDecodedSize; + BrotliDecoderResult result = BrotliDecoderDecompressStream( + brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize, + &decodedPtr, nullptr); + bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize; + + switch (result) { + case BROTLI_DECODER_RESULT_ERROR: + qWarning("Brotli error: %s", + BrotliDecoderErrorString(BrotliDecoderGetErrorCode(brotliDecoderState))); + return -1; + case BROTLI_DECODER_RESULT_SUCCESS: + BrotliDecoderDestroyInstance(brotliDecoderState); + decoderPointer = nullptr; + return bytesDecoded; + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + if (!compressedDataBuffer.isEmpty()) { + input = compressedDataBuffer.read(); + encodedPtr = reinterpret_cast(input.constData()); + encodedBytesRemaining = input.size(); + break; + } + return bytesDecoded; + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + // Some data is leftover inside the brotli decoder, remember for next time + decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState); + Q_ASSERT(unusedDecodedSize == 0); + break; + } + } + if (encodedBytesRemaining) { + // Some input was left unused; move back to the buffer + input = input.right(QByteArray::size_type(encodedBytesRemaining)); + compressedDataBuffer.prepend(input); + } + return bytesDecoded; +#endif +} + QT_END_NAMESPACE diff --git a/src/network/access/qdecompresshelper_p.h b/src/network/access/qdecompresshelper_p.h index 9d8ca245e1..851a1946d9 100644 --- a/src/network/access/qdecompresshelper_p.h +++ b/src/network/access/qdecompresshelper_p.h @@ -66,6 +66,7 @@ public: None, Deflate, GZip, + Brotli, }; QDecompressHelper() = default; @@ -101,6 +102,7 @@ private: qint64 encodedBytesAvailable() const; qsizetype readZLib(char *data, qsizetype maxSize); + qsizetype readBrotli(char *data, qsizetype maxSize); QByteDataBuffer compressedDataBuffer; bool decoderHasData = false; @@ -112,6 +114,10 @@ private: ContentEncoding contentEncoding = None; void *decoderPointer = nullptr; +#if QT_CONFIG(brotli) + const uint8_t *brotliUnconsumedDataPtr = nullptr; + size_t brotliUnconsumedAmount = 0; +#endif }; QT_END_NAMESPACE diff --git a/src/network/configure.cmake b/src/network/configure.cmake index fcdf46f955..e54c7c4ee6 100644 --- a/src/network/configure.cmake +++ b/src/network/configure.cmake @@ -6,6 +6,7 @@ #### Libraries +qt_find_package(WrapBrotli PROVIDED_TARGETS WrapBrotli::WrapBrotliDec) qt_find_package(Libproxy PROVIDED_TARGETS PkgConfig::Libproxy MODULE_NAME network QMAKE_LIB libproxy) qt_find_package(WrapOpenSSLHeaders PROVIDED_TARGETS WrapOpenSSLHeaders::WrapOpenSSLHeaders MODULE_NAME network QMAKE_LIB openssl_headers) # openssl_headers @@ -366,6 +367,13 @@ qt_feature("networkdiskcache" PUBLIC CONDITION QT_FEATURE_temporaryfile ) qt_feature_definition("networkdiskcache" "QT_NO_NETWORKDISKCACHE" NEGATE VALUE "1") +qt_feature("brotli" PUBLIC + SECTION "Networking" + LABEL "Brotli Decompression Support" + PURPOSE "Support for downloading and decompressing resources compressed with Brotli through QNetworkAccessManager." + CONDITION WrapBrotli_FOUND +) +qt_feature_definition("brotli" "QT_NO_BROTLI" NEGATE VALUE "1") qt_feature("localserver" PUBLIC SECTION "Networking" LABEL "QLocalServer" @@ -432,6 +440,7 @@ qt_configure_add_summary_entry(ARGS "ftp") qt_configure_add_summary_entry(ARGS "sctp") qt_configure_add_summary_entry(ARGS "system-proxies") qt_configure_add_summary_entry(ARGS "gssapi") +qt_configure_add_summary_entry(ARGS "brotli") qt_configure_end_summary_section() # end of "Qt Network" section qt_configure_add_report_entry( TYPE NOTE diff --git a/src/network/configure.json b/src/network/configure.json index 6425103a33..a8166340dc 100644 --- a/src/network/configure.json +++ b/src/network/configure.json @@ -25,6 +25,21 @@ }, "libraries": { + "brotli": { + "label": "Brotli Decompression", + "test": { + "main": [ + "BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);" + ] + }, + "headers": [ + "brotli/decode.h" + ], + "sources": [ + { "type": "pkgConfig", "args": "libbrotlidec" }, + "-lbrotlidec" + ] + }, "corewlan": { "label": "CoreWLan", "export": "", @@ -387,6 +402,13 @@ "condition": "features.temporaryfile", "output": [ "publicFeature", "feature" ] }, + "brotli": { + "label": "Brotli Decompression Support", + "purpose": "Support for downloading and decompressing resources compressed with Brotli through QNetworkAccessManager.", + "section": "Networking", + "condition": "libs.brotli", + "output": [ "publicFeature", "feature" ] + }, "localserver": { "label": "QLocalServer", "purpose": "Provides a local socket based server.", @@ -479,7 +501,8 @@ For example: "ftp", "sctp", "system-proxies", - "gssapi" + "gssapi", + "brotli" ] } ] -- cgit v1.2.3