From d40f88e8d23f9d5fdb08ff9a394d9697a0d86fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A5rten=20Nordheim?= Date: Wed, 6 May 2020 18:30:43 +0200 Subject: QDecompressHelper: Introduce zstd support Also take this opportunity to reshuffle the content-encodings in the intended ordering since the ordering is used to signify priority. Task-number: QTBUG-83269 Change-Id: I022eecf1ba03b54dbd9c98a9d63d05fb05fd2124 Reviewed-by: Thiago Macieira --- src/network/.prev_CMakeLists.txt | 5 ++ src/network/CMakeLists.txt | 5 ++ src/network/access/access.pri | 4 ++ src/network/access/qdecompresshelper.cpp | 82 +++++++++++++++++++++++++++++++- src/network/access/qdecompresshelper_p.h | 2 + 5 files changed, 96 insertions(+), 2 deletions(-) (limited to 'src/network') diff --git a/src/network/.prev_CMakeLists.txt b/src/network/.prev_CMakeLists.txt index b677b8020a..d7ef0fd712 100644 --- a/src/network/.prev_CMakeLists.txt +++ b/src/network/.prev_CMakeLists.txt @@ -128,6 +128,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http WrapBrotli::WrapBrotliDec ) +qt_extend_target(Network CONDITION QT_FEATURE_http AND QT_FEATURE_zstd + LIBRARIES + ZSTD::ZSTD +) + qt_extend_target(Network CONDITION QT_FEATURE_system_zlib LIBRARIES ZLIB::ZLIB diff --git a/src/network/CMakeLists.txt b/src/network/CMakeLists.txt index 157a31f1da..309122fa73 100644 --- a/src/network/CMakeLists.txt +++ b/src/network/CMakeLists.txt @@ -128,6 +128,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http WrapBrotli::WrapBrotliDec ) +qt_extend_target(Network CONDITION QT_FEATURE_http AND QT_FEATURE_zstd + LIBRARIES + ZSTD::ZSTD +) + qt_extend_target(Network CONDITION QT_FEATURE_system_zlib LIBRARIES ZLIB::ZLIB diff --git a/src/network/access/access.pri b/src/network/access/access.pri index bb8f155d23..af49fe2bb4 100644 --- a/src/network/access/access.pri +++ b/src/network/access/access.pri @@ -120,4 +120,8 @@ qtConfig(http) { qtConfig(brotli) { QMAKE_USE_PRIVATE += brotli } + + qtConfig(zstd) { + QMAKE_USE_PRIVATE += zstd + } } diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp index 4fbb2d8889..25e87bbc43 100644 --- a/src/network/access/qdecompresshelper.cpp +++ b/src/network/access/qdecompresshelper.cpp @@ -48,6 +48,10 @@ # include #endif +#if QT_CONFIG(zstd) +# include +#endif + #include QT_BEGIN_NAMESPACE @@ -59,11 +63,14 @@ struct ContentEncodingMapping }; constexpr ContentEncodingMapping contentEncodingMapping[] { - { "gzip", QDecompressHelper::GZip }, - { "deflate", QDecompressHelper::Deflate }, +#if QT_CONFIG(zstd) + { "zstd", QDecompressHelper::Zstandard }, +#endif #if QT_CONFIG(brotli) { "br", QDecompressHelper::Brotli }, #endif + { "gzip", QDecompressHelper::GZip }, + { "deflate", QDecompressHelper::Deflate }, }; QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept @@ -86,6 +93,13 @@ BrotliDecoderState *toBrotliPointer(void *ptr) return static_cast(ptr); } #endif + +#if QT_CONFIG(zstd) +ZSTD_DStream *toZstandardPointer(void *ptr) +{ + return static_cast(ptr); +} +#endif } bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding) @@ -153,6 +167,13 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce) decoderPointer = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); #else Q_UNREACHABLE(); +#endif + break; + case Zstandard: +#if QT_CONFIG(zstd) + decoderPointer = ZSTD_createDStream(); +#else + Q_UNREACHABLE(); #endif break; } @@ -349,6 +370,9 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) case Brotli: bytesRead = readBrotli(data, maxSize); break; + case Zstandard: + bytesRead = readZstandard(data, maxSize); + break; } if (bytesRead == -1) clear(); @@ -398,6 +422,14 @@ void QDecompressHelper::clear() BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer); if (brotliDecoderState) BrotliDecoderDestroyInstance(brotliDecoderState); +#endif + break; + } + case Zstandard: { +#if QT_CONFIG(zstd) + ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer); + if (zstdStream) + ZSTD_freeDStream(zstdStream); #endif break; } @@ -604,4 +636,50 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) #endif } +qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize) +{ +#if !QT_CONFIG(zstd) + Q_UNUSED(data); + Q_UNUSED(maxSize); + Q_UNREACHABLE(); +#else + ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer); + + QByteArray input; + if (!compressedDataBuffer.isEmpty()) + input = compressedDataBuffer.read(); + ZSTD_inBuffer inBuf { input.constData(), size_t(input.size()), 0 }; + + ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 }; + + bool dataLeftover = false; + qsizetype bytesDecoded = 0; + while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) { + + dataLeftover = false; + size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf); + if (ZSTD_isError(retValue)) { + qWarning("ZStandard error: %s", ZSTD_getErrorName(retValue)); + return -1; + } else if (retValue >= 0) { + decoderHasData = false; + bytesDecoded = outBuf.pos; + // if pos == size then there may be data left over in internal buffers + if (outBuf.pos == outBuf.size) { + decoderHasData = true; + } else if (inBuf.pos == inBuf.size && !compressedDataBuffer.isEmpty()) { + input = compressedDataBuffer.read(); + inBuf = { input.constData(), size_t(input.size()), 0 }; + } + } + } + if (inBuf.pos < inBuf.size) { + // Some input was left unused; move back to the buffer + input = input.mid(QByteArray::size_type(inBuf.pos)); + compressedDataBuffer.prepend(std::move(input)); + } + return bytesDecoded; +#endif +} + QT_END_NAMESPACE diff --git a/src/network/access/qdecompresshelper_p.h b/src/network/access/qdecompresshelper_p.h index 851a1946d9..6514e7d418 100644 --- a/src/network/access/qdecompresshelper_p.h +++ b/src/network/access/qdecompresshelper_p.h @@ -67,6 +67,7 @@ public: Deflate, GZip, Brotli, + Zstandard, }; QDecompressHelper() = default; @@ -103,6 +104,7 @@ private: qsizetype readZLib(char *data, qsizetype maxSize); qsizetype readBrotli(char *data, qsizetype maxSize); + qsizetype readZstandard(char *data, qsizetype maxSize); QByteDataBuffer compressedDataBuffer; bool decoderHasData = false; -- cgit v1.2.3