QDecompressHelper: Introduce zstd support

Also take this opportunity to reshuffle the content-encodings in the
intended ordering since the ordering is used to signify priority.

Task-number: QTBUG-83269
Change-Id: I022eecf1ba03b54dbd9c98a9d63d05fb05fd2124
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

13 files changed, 252 insertions, 2 deletions

diff --git a/src/network/.prev_CMakeLists.txt b/src/network/.prev_CMakeLists.txt
index b677b8020a..d7ef0fd712 100644
--- a/src/network/.prev_CMakeLists.txt
+++ b/src/network/.prev_CMakeLists.txt
@@ -128,6 +128,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
         WrapBrotli::WrapBrotliDec
 )
 
+qt_extend_target(Network CONDITION QT_FEATURE_http AND QT_FEATURE_zstd
+    LIBRARIES
+        ZSTD::ZSTD
+)
+
 qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
     LIBRARIES
         ZLIB::ZLIB
diff --git a/src/network/CMakeLists.txt b/src/network/CMakeLists.txt
index 157a31f1da..309122fa73 100644
--- a/src/network/CMakeLists.txt
+++ b/src/network/CMakeLists.txt
@@ -128,6 +128,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
         WrapBrotli::WrapBrotliDec
 )
 
+qt_extend_target(Network CONDITION QT_FEATURE_http AND QT_FEATURE_zstd
+    LIBRARIES
+        ZSTD::ZSTD
+)
+
 qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
     LIBRARIES
         ZLIB::ZLIB
diff --git a/src/network/access/access.pri b/src/network/access/access.pri
index bb8f155d23..af49fe2bb4 100644
--- a/src/network/access/access.pri
+++ b/src/network/access/access.pri
@@ -120,4 +120,8 @@ qtConfig(http) {
     qtConfig(brotli) {
         QMAKE_USE_PRIVATE += brotli
     }
+
+    qtConfig(zstd) {
+        QMAKE_USE_PRIVATE += zstd
+    }
 }
diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp
index 4fbb2d8889..25e87bbc43 100644
--- a/src/network/access/qdecompresshelper.cpp
+++ b/src/network/access/qdecompresshelper.cpp
@@ -48,6 +48,10 @@
 #    include <brotli/decode.h>
 #endif
 
+#if QT_CONFIG(zstd)
+#    include <zstd.h>
+#endif
+
 #include <array>
 
 QT_BEGIN_NAMESPACE
@@ -59,11 +63,14 @@ struct ContentEncodingMapping
 };
 
 constexpr ContentEncodingMapping contentEncodingMapping[] {
-    { "gzip", QDecompressHelper::GZip },
-    { "deflate", QDecompressHelper::Deflate },
+#if QT_CONFIG(zstd)
+    { "zstd", QDecompressHelper::Zstandard },
+#endif
 #if QT_CONFIG(brotli)
     { "br", QDecompressHelper::Brotli },
 #endif
+    { "gzip", QDecompressHelper::GZip },
+    { "deflate", QDecompressHelper::Deflate },
 };
 
 QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept
@@ -86,6 +93,13 @@ BrotliDecoderState *toBrotliPointer(void *ptr)
     return static_cast<BrotliDecoderState *>(ptr);
 }
 #endif
+
+#if QT_CONFIG(zstd)
+ZSTD_DStream *toZstandardPointer(void *ptr)
+{
+    return static_cast<ZSTD_DStream *>(ptr);
+}
+#endif
 }
 
 bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding)
@@ -155,6 +169,13 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce)
         Q_UNREACHABLE();
 #endif
         break;
+    case Zstandard:
+#if QT_CONFIG(zstd)
+        decoderPointer = ZSTD_createDStream();
+#else
+        Q_UNREACHABLE();
+#endif
+        break;
     }
     if (!decoderPointer) {
         qWarning("Failed to initialize the decoder.");
@@ -349,6 +370,9 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize)
     case Brotli:
         bytesRead = readBrotli(data, maxSize);
         break;
+    case Zstandard:
+        bytesRead = readZstandard(data, maxSize);
+        break;
     }
     if (bytesRead == -1)
         clear();
@@ -401,6 +425,14 @@ void QDecompressHelper::clear()
 #endif
         break;
     }
+    case Zstandard: {
+#if QT_CONFIG(zstd)
+        ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
+        if (zstdStream)
+            ZSTD_freeDStream(zstdStream);
+#endif
+        break;
+    }
     }
     decoderPointer = nullptr;
     contentEncoding = None;
@@ -604,4 +636,50 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize)
 #endif
 }
 
+qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize)
+{
+#if !QT_CONFIG(zstd)
+    Q_UNUSED(data);
+    Q_UNUSED(maxSize);
+    Q_UNREACHABLE();
+#else
+    ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
+
+    QByteArray input;
+    if (!compressedDataBuffer.isEmpty())
+        input = compressedDataBuffer.read();
+    ZSTD_inBuffer inBuf { input.constData(), size_t(input.size()), 0 };
+
+    ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 };
+
+    bool dataLeftover = false;
+    qsizetype bytesDecoded = 0;
+    while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) {
+
+        dataLeftover = false;
+        size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf);
+        if (ZSTD_isError(retValue)) {
+            qWarning("ZStandard error: %s", ZSTD_getErrorName(retValue));
+            return -1;
+        } else if (retValue >= 0) {
+            decoderHasData = false;
+            bytesDecoded = outBuf.pos;
+            // if pos == size then there may be data left over in internal buffers
+            if (outBuf.pos == outBuf.size) {
+                decoderHasData = true;
+            } else if (inBuf.pos == inBuf.size && !compressedDataBuffer.isEmpty()) {
+                input = compressedDataBuffer.read();
+                inBuf = { input.constData(), size_t(input.size()), 0 };
+            }
+        }
+    }
+    if (inBuf.pos < inBuf.size) {
+        // Some input was left unused; move back to the buffer
+        input = input.mid(QByteArray::size_type(inBuf.pos));
+        compressedDataBuffer.prepend(std::move(input));
+    }
+    return bytesDecoded;
+#endif
+}
+
 QT_END_NAMESPACE
diff --git a/src/network/access/qdecompresshelper_p.h b/src/network/access/qdecompresshelper_p.h
index 851a1946d9..6514e7d418 100644
--- a/src/network/access/qdecompresshelper_p.h
+++ b/src/network/access/qdecompresshelper_p.h
@@ -67,6 +67,7 @@ public:
         Deflate,
         GZip,
         Brotli,
+        Zstandard,
     };
 
     QDecompressHelper() = default;
@@ -103,6 +104,7 @@ private:
 
     qsizetype readZLib(char *data, qsizetype maxSize);
     qsizetype readBrotli(char *data, qsizetype maxSize);
+    qsizetype readZstandard(char *data, qsizetype maxSize);
 
     QByteDataBuffer compressedDataBuffer;
     bool decoderHasData = false;
diff --git a/tests/auto/network/access/http2/tst_http2.cpp b/tests/auto/network/access/http2/tst_http2.cpp
index bc685e5ca9..0282942225 100644
--- a/tests/auto/network/access/http2/tst_http2.cpp
+++ b/tests/auto/network/access/http2/tst_http2.cpp
@@ -801,6 +801,11 @@ void tst_Http2::contentEncoding_data()
                                      "hello world");
 #endif
 
+#if QT_CONFIG(zstd)
+    contentEncodingData.emplace_back(
+            "zstd", QByteArray::fromBase64("KLUv/QRYWQAAaGVsbG8gd29ybGRoaR6y"), "hello world");
+#endif
+
     // Loop through and add the data...
     for (const auto &data : contentEncodingData) {
         const char *name = data.contentEncoding.data();
diff --git a/tests/auto/network/access/qdecompresshelper/CMakeLists.txt b/tests/auto/network/access/qdecompresshelper/CMakeLists.txt
index 5038f36b94..3b2f1b5e6f 100644
--- a/tests/auto/network/access/qdecompresshelper/CMakeLists.txt
+++ b/tests/auto/network/access/qdecompresshelper/CMakeLists.txt
@@ -9,6 +9,7 @@ qt_add_test(tst_qdecompresshelper
         gzip.rcc.cpp
         inflate.rcc.cpp
         tst_qdecompresshelper.cpp
+        zstandard.rcc.cpp
     DEFINES
         SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} # special case
     PUBLIC_LIBRARIES
diff --git a/tests/auto/network/access/qdecompresshelper/qdecompresshelper.pro b/tests/auto/network/access/qdecompresshelper/qdecompresshelper.pro
index d7bb6bcd62..254f04a707 100644
--- a/tests/auto/network/access/qdecompresshelper/qdecompresshelper.pro
+++ b/tests/auto/network/access/qdecompresshelper/qdecompresshelper.pro
@@ -7,5 +7,6 @@ SOURCES += \
     tst_qdecompresshelper.cpp \
     gzip.rcc.cpp \
     inflate.rcc.cpp \
+    zstandard.rcc.cpp \
 
 DEFINES += SRC_DIR="$$PWD"
diff --git a/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp b/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
index 2641c65497..7a3aa37a47 100644
--- a/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
+++ b/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
@@ -73,9 +73,15 @@ void tst_QDecompressHelper::initTestCase()
 {
     Q_INIT_RESOURCE(gzip);
     Q_INIT_RESOURCE(inflate);
+#if QT_CONFIG(zstd)
+    Q_INIT_RESOURCE(zstandard);
+#endif
 }
 void tst_QDecompressHelper::cleanupTestCase()
 {
+#if QT_CONFIG(zstd)
+    Q_CLEANUP_RESOURCE(zstandard);
+#endif
     Q_CLEANUP_RESOURCE(inflate);
     Q_CLEANUP_RESOURCE(gzip);
 }
@@ -98,6 +104,11 @@ void tst_QDecompressHelper::encodingSupported()
     ++expected;
 #endif
 
+#if QT_CONFIG(zstd)
+    QVERIFY(QDecompressHelper::isSupportedEncoding("zstd"));
+    QVERIFY(accepted.contains("zstd"));
+    ++expected;
+#endif
     QCOMPARE(expected, accepted.size());
 }
 
@@ -129,6 +140,12 @@ void tst_QDecompressHelper::sharedDecompress_data()
             << QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
             << QByteArray("hello world");
 #endif
+
+#if QT_CONFIG(zstd)
+    QTest::newRow("zstandard-hello-world")
+            << QByteArray("zstd") << QByteArray::fromBase64("KLUv/QRYWQAAaGVsbG8gd29ybGRoaR6y")
+            << QByteArray("hello world");
+#endif
 }
 
 void tst_QDecompressHelper::decompress_data()
@@ -340,6 +357,10 @@ void tst_QDecompressHelper::decompressBigData_data()
 #if QT_CONFIG(brotli)
     QTest::newRow("brotli-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB;
 #endif
+
+#if QT_CONFIG(zstd)
+    QTest::newRow("zstandard-4G") << QByteArray("zstd") << (":/4G.zst") << fourGiB;
+#endif
 }
 
 void tst_QDecompressHelper::decompressBigData()
diff --git a/tests/auto/network/access/qdecompresshelper/zstandard.rcc.cpp b/tests/auto/network/access/qdecompresshelper/zstandard.rcc.cpp
new file mode 100644
index 0000000000..c8a435192f
--- /dev/null
+++ b/tests/auto/network/access/qdecompresshelper/zstandard.rcc.cpp
@@ -0,0 +1,116 @@
+/****************************************************************************
+** Resource object code
+**
+** Created by: The Resource Compiler for Qt version 6.0.0
+**
+** WARNING! All changes made in this file will be lost!
+*****************************************************************************/
+
+static const unsigned char qt_resource_data[] = {
+  // D:/projects/qt/dev/src/qtbase/tests/auto/network/access/decompresshelper/4G.zst
+  0x0,0x0,0x1,0x75,
+  0x0,
+  0x2,0x3,0x93,0x78,0xda,0xed,0xd4,0x21,0xe,0x83,0x40,0x10,0x86,0xd1,0x29,0x98,
+  0x26,0x98,0x3d,0x46,0x1d,0x1a,0x8f,0xec,0x29,0x50,0xdc,0x84,0x13,0xe1,0x2b,0x90,
+  0x1c,0x89,0xcd,0x32,0xe9,0x25,0x2a,0xfa,0x26,0x79,0xc9,0xe8,0x5f,0x7c,0xaf,0x7d,
+  0xac,0xc7,0x1a,0x79,0x8f,0xf4,0x8e,0x78,0xe6,0x73,0xb5,0xa9,0x74,0x5d,0x94,0x0,
+  0xfe,0xcf,0xfc,0xed,0x41,0x6d,0xe7,0x50,0xcc,0x1,0x32,0x60,0xe,0x90,0x1,0x73,
+  0x80,0xc,0x98,0x4,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,
+  0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,
+  0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,
+  0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,
+  0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,
+  0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,
+  0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,
+  0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,
+  0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,
+  0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,
+  0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,
+  0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,
+  0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,
+  0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,
+  0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,
+  0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,
+  0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,
+  0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x80,0x5f,0xe8,0xd3,0xf2,0x69,0xdb,0xd,
+  0xcd,0x15,0x90,0xe9,
+  
+};
+
+static const unsigned char qt_resource_name[] = {
+  // 4G.zst
+  0x0,0x6,
+  0x3,0x8a,0x61,0xa4,
+  0x0,0x34,
+  0x0,0x47,0x0,0x2e,0x0,0x7a,0x0,0x73,0x0,0x74,
+  
+};
+
+static const unsigned char qt_resource_struct[] = {
+  // :
+  0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x1,
+0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,
+  // :/4G.zst
+  0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x0,
+0x0,0x0,0x1,0x72,0x1c,0x8d,0x7,0xac,
+
+};
+
+#ifdef QT_NAMESPACE
+#  define QT_RCC_PREPEND_NAMESPACE(name) ::QT_NAMESPACE::name
+#  define QT_RCC_MANGLE_NAMESPACE0(x) x
+#  define QT_RCC_MANGLE_NAMESPACE1(a, b) a##_##b
+#  define QT_RCC_MANGLE_NAMESPACE2(a, b) QT_RCC_MANGLE_NAMESPACE1(a,b)
+#  define QT_RCC_MANGLE_NAMESPACE(name) QT_RCC_MANGLE_NAMESPACE2( \
+        QT_RCC_MANGLE_NAMESPACE0(name), QT_RCC_MANGLE_NAMESPACE0(QT_NAMESPACE))
+#else
+#   define QT_RCC_PREPEND_NAMESPACE(name) name
+#   define QT_RCC_MANGLE_NAMESPACE(name) name
+#endif
+
+#ifdef QT_NAMESPACE
+namespace QT_NAMESPACE {
+#endif
+
+bool qRegisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);
+bool qUnregisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);
+
+#if defined(__ELF__) || defined(__APPLE__)
+static inline unsigned char qResourceFeatureZlib()
+{
+    extern const unsigned char qt_resourceFeatureZlib;
+    return qt_resourceFeatureZlib;
+}
+#else
+unsigned char qResourceFeatureZlib();
+#endif
+
+#ifdef QT_NAMESPACE
+}
+#endif
+
+int QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)();
+int QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)()
+{
+    int version = 3;
+    QT_RCC_PREPEND_NAMESPACE(qRegisterResourceData)
+        (version, qt_resource_struct, qt_resource_name, qt_resource_data);
+    return 1;
+}
+
+int QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)();
+int QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)()
+{
+    int version = 3;
+    version += QT_RCC_PREPEND_NAMESPACE(qResourceFeatureZlib());
+    QT_RCC_PREPEND_NAMESPACE(qUnregisterResourceData)
+       (version, qt_resource_struct, qt_resource_name, qt_resource_data);
+    return 1;
+}
+
+namespace {
+   struct initializer {
+       initializer() { QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)(); }
+       ~initializer() { QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)(); }
+   } dummy;
+}
diff --git a/tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp b/tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp
index 6bec0c8ee1..e4dafb58d0 100644
--- a/tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp
+++ b/tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp
@@ -9270,6 +9270,14 @@ void tst_QNetworkReply::contentEncoding_data()
             << QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
             << QByteArray("hello world");
 #endif
+
+#if defined(QT_BUILD_INTERNAL) && QT_CONFIG(zstd)
+    QTest::newRow("zstandard-hello-world")
+            << QByteArray("zstd") << QByteArray::fromBase64("KLUv/QRYWQAAaGVsbG8gd29ybGRoaR6y")
+            << QByteArray("hello world");
+#else
+    qDebug("Note: ZStandard testdata is only available for developer builds.");
+#endif
 }
 
 void tst_QNetworkReply::contentEncoding()
diff --git a/tests/benchmarks/network/access/qdecompresshelper/50mb.txt.zst b/tests/benchmarks/network/access/qdecompresshelper/50mb.txt.zst
new file mode 100644
index 0000000000..e9546e230e
--- /dev/null
+++ b/tests/benchmarks/network/access/qdecompresshelper/50mb.txt.zst
diff --git a/tests/benchmarks/network/access/qdecompresshelper/main.cpp b/tests/benchmarks/network/access/qdecompresshelper/main.cpp
index 5296c9681c..3c651b16dd 100644
--- a/tests/benchmarks/network/access/qdecompresshelper/main.cpp
+++ b/tests/benchmarks/network/access/qdecompresshelper/main.cpp
@@ -57,6 +57,10 @@ void tst_QDecompressHelper::decompress_data()
     QTest::addRow("brotli") << QByteArray("br") << srcDir + QString("50mb.txt.br");
     dataAdded = true;
 #endif
+#if QT_CONFIG(zstd)
+    QTest::addRow("zstandard") << QByteArray("zstd") << srcDir + QString("50mb.txt.zst");
+    dataAdded = true;
+#endif
     if (!dataAdded)
         QSKIP("There's no decompression support");
 }