QDecompressHelper: Skip double-decompression if download is small

To retain backwards compatibility with some QNetworkReply usage,
namely connecting solely to finished-signal and allocating a buffer to
read into, but without storing the entire decompressed file in memory
until read, we may decompress the file twice.

With this patch users can now avoid this double decompression if the
amount of buffered data stays below 10 MiB. This means any file smaller
than 10 MiB will never need to be decompressed twice to know the size of
it. On top of that, if the data is handled as it arrives (e.g. in
readyRead) and the buffer is kept below 10 MiB it won't need to
decompress twice either.

This is active as long as "countDecompressed" is true, though it
currently always is in QNetworkAccessManger, with a future goal to make
it possible to control with public API. Since it requires the user to
potentially adapt their usage of QNetworkReply.

In this patch we also stop tracking the amount of unhandled uncompressed
bytes (uncompressedBytes) in favor of tracking the total amount of bytes
which has been read() by the user of QDecompressHelper (totalBytesRead),
since we can more intuitively work out the total amount of unread bytes
using this value.

Change-Id: Ie3d8d6e39a18343fcf9b610f45c7fe7e4cd4e474
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

1 files changed, 17 insertions, 9 deletions

diff --git a/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp b/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
index c9849fdcdf..321d373a78 100644
--- a/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
+++ b/tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
@@ -347,20 +347,24 @@ void tst_QDecompressHelper::decompressBigData_data()
     QTest::addColumn<QByteArray>("encoding");
     QTest::addColumn<QString>("path");
     QTest::addColumn<qint64>("size");
+    QTest::addColumn<bool>("countAhead");
 
     qint64 fourGiB = 4ll * 1024ll * 1024ll * 1024ll;
     qint64 fiveGiB = 5ll * 1024ll * 1024ll * 1024ll;
 
-    QTest::newRow("gzip-4G") << QByteArray("gzip") << QString(":/4G.gz") << fourGiB;
+    // Only use countAhead on one of these since they share codepath anyway
+    QTest::newRow("gzip-counted-4G") << QByteArray("gzip") << QString(":/4G.gz") << fourGiB << true;
     QTest::newRow("deflate-5G") << QByteArray("deflate") << QString(":/5GiB.txt.inflate")
-                                << fiveGiB;
+                                << fiveGiB << false;
 
 #if QT_CONFIG(brotli)
-    QTest::newRow("brotli-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB;
+    QTest::newRow("brotli-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB << false;
+    QTest::newRow("brotli-counted-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB << true;
 #endif
 
 #if QT_CONFIG(zstd)
-    QTest::newRow("zstandard-4G") << QByteArray("zstd") << (":/4G.zst") << fourGiB;
+    QTest::newRow("zstandard-4G") << QByteArray("zstd") << (":/4G.zst") << fourGiB << false;
+    QTest::newRow("zstandard-counted-4G") << QByteArray("zstd") << (":/4G.zst") << fourGiB << true;
 #endif
 }
 
@@ -373,16 +377,20 @@ void tst_QDecompressHelper::decompressBigData()
     const qint64 third = file.bytesAvailable() / 3;
 
     QDecompressHelper helper;
+    QFETCH(bool, countAhead);
+    helper.setCountingBytesEnabled(countAhead);
     helper.setDecompressedSafetyCheckThreshold(-1);
     QFETCH(QByteArray, encoding);
     helper.setEncoding(encoding);
 
-    QByteArray output(32 * 1024, Qt::Uninitialized);
+    // The size of 'output' should be at least QDecompressHelper::MaxDecompressedDataBufferSize + 1
+    QByteArray output(10 * 1024 * 1024 + 1, Qt::Uninitialized);
     qint64 totalSize = 0;
     while (!file.atEnd()) {
         helper.feed(file.read(third));
         while (helper.hasData()) {
             qsizetype bytesRead = helper.read(output.data(), output.size());
+            QVERIFY(bytesRead >= 0);
             QVERIFY(bytesRead <= output.size());
             totalSize += bytesRead;
             const auto isZero = [](char c) { return c == '\0'; };
@@ -449,10 +457,10 @@ void tst_QDecompressHelper::bigZlib()
     helper.feed(compressedData.mid(firstHalf.size()));
 
     // We need the whole thing in one go... which is why this test is not available for 32-bit
-    qint64 expected = 5ll * 1024ll * 1024ll * 1024ll;
-    // This can be replaced with QByteArray after the qsizetype change is merged
-    std::unique_ptr<char[]> output(new char[expected]);
-    qsizetype size = helper.read(output.get(), expected);
+    const qint64 expected = 5ll * 1024ll * 1024ll * 1024ll;
+    // Request a few more byte than what is available, to verify exact size
+    QByteArray output(expected + 42, Qt::Uninitialized);
+    const qsizetype size = helper.read(output.data(), output.size());
     QCOMPARE(size, expected);
 #endif
 }