Add support for containers > 4 Gi elements in QDataStream

The format is changed from 6.7 to support more than UINT32_MAX - 1
elements. The format used to have a quint32 size. Now if the size is
larger or equal to 0xfffffffe (2^32 -2) the old size is an extend
value 0xfffffffe followed by one quint64 with the actual value. The
32 bit size with all bits set is still used as null value.

Fixes: QTBUG-105034
Change-Id: I62188be170fe779022ad58ab84a54b1eaf46e5d9
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

4 files changed, 193 insertions, 0 deletions

diff --git a/tests/manual/corelib/CMakeLists.txt b/tests/manual/corelib/CMakeLists.txt
index 0115fc76f9..8ed7441e77 100644
--- a/tests/manual/corelib/CMakeLists.txt
+++ b/tests/manual/corelib/CMakeLists.txt
@@ -1,5 +1,6 @@
 # Copyright (C) 2023 The Qt Company Ltd.
 # SPDX-License-Identifier: BSD-3-Clause
 
+add_subdirectory(qdatastream)
 add_subdirectory(time)
 add_subdirectory(tools)
diff --git a/tests/manual/corelib/qdatastream/CMakeLists.txt b/tests/manual/corelib/qdatastream/CMakeLists.txt
new file mode 100644
index 0000000000..5d26a862e5
--- /dev/null
+++ b/tests/manual/corelib/qdatastream/CMakeLists.txt
@@ -0,0 +1,15 @@
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+#####################################################################
+## tst_manual_qdatastream Test:
+#####################################################################
+
+qt_internal_add_manual_test(tst_manual_qdatastream
+    SOURCES
+        tst_manualqdatastream.cpp
+    LIBRARIES
+        Qt::Core
+    ENABLE_AUTOGEN_TOOLS
+        uic
+)
diff --git a/tests/manual/corelib/qdatastream/qdatastream.pro b/tests/manual/corelib/qdatastream/qdatastream.pro
new file mode 100644
index 0000000000..9143f62851
--- /dev/null
+++ b/tests/manual/corelib/qdatastream/qdatastream.pro
@@ -0,0 +1,6 @@
+CONFIG += testcase
+
+SOURCES += tst_manualqdatastream.cpp
+QT = core testlib
+
+TARGET = tst_manual_qdatastream
diff --git a/tests/manual/corelib/qdatastream/tst_manualqdatastream.cpp b/tests/manual/corelib/qdatastream/tst_manualqdatastream.cpp
new file mode 100644
index 0000000000..59bff74e13
--- /dev/null
+++ b/tests/manual/corelib/qdatastream/tst_manualqdatastream.cpp
@@ -0,0 +1,171 @@
+// Copyright (C) 2023 BlackBerry Limited. All rights reserved.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QDataStream>
+#include <QHash>
+#include <QList>
+#include <QMap>
+#include <QSet>
+#include <QTest>
+
+// These tests are way too slow to be part of automatic unit tests
+class tst_QDataStream : public QObject
+{
+    Q_OBJECT
+
+    template <class T>
+    void fill(T &input);
+    template <>
+    void fill(QSet<qsizetype> &input);
+    template <>
+    void fill(QMap<qsizetype, qsizetype> &input);
+    template <>
+    void fill(QHash<qsizetype, qsizetype> &input);
+
+    template <class T>
+    void stream_big();
+
+public slots:
+    void initTestCase();
+
+private slots:
+    void stream_bigQString();
+    void stream_bigQList();
+    void stream_bigQSet();
+    void stream_bigQMap();
+    void stream_bigQHash();
+};
+
+void tst_QDataStream::initTestCase()
+{
+    qputenv("QTEST_FUNCTION_TIMEOUT", "9000000");
+}
+
+template <class T>
+void tst_QDataStream::fill(T &input)
+{
+    constexpr qsizetype GiB = 1024 * 1024 * 1024;
+    constexpr qsizetype BaseSize = 4 * GiB + 1;
+    qDebug("Filling container with %lld entries", qint64(BaseSize));
+    QElapsedTimer timer;
+    timer.start();
+    try {
+        input.reserve(BaseSize);
+        input.resize(BaseSize, 'a');
+    } catch (const std::bad_alloc &) {
+        QSKIP("Could not allocate 4 GiB of RAM.");
+    }
+    qDebug("Created dataset in %lld ms", timer.elapsed());
+}
+
+template <>
+void tst_QDataStream::fill(QSet<qsizetype> &input)
+{
+    constexpr qsizetype GiB = 1024 * 1024 * 1024;
+    constexpr qsizetype BaseSize = 4 * GiB + 1;
+    qDebug("Filling container with %lld entries", qint64(BaseSize));
+    QElapsedTimer timer;
+    timer.start();
+    try {
+        input.reserve(BaseSize);
+        for (qsizetype i = 0; i < BaseSize; ++i)
+            input.insert(i);
+    } catch (const std::bad_alloc &) {
+        QSKIP("Could not allocate 4 Gi entries.");
+    }
+    qDebug("Created dataset in %lld ms", timer.elapsed());
+}
+
+template <>
+void tst_QDataStream::fill(QMap<qsizetype, qsizetype> &input)
+{
+    constexpr qsizetype GiB = 1024 * 1024 * 1024;
+    constexpr qsizetype BaseSize = 4 * GiB + 1;
+    qDebug("Filling container with %lld entries", qint64(BaseSize));
+    QElapsedTimer timer;
+    timer.start();
+    try {
+        for (qsizetype i = 0; i < BaseSize; ++i)
+            input.insert(i, i);
+    } catch (const std::bad_alloc &) {
+        QSKIP("Could not allocate 4 Gi entries.");
+    }
+    qDebug("Created dataset in %lld ms", timer.elapsed());
+}
+
+template <>
+void tst_QDataStream::fill(QHash<qsizetype, qsizetype> &input)
+{
+    constexpr qsizetype GiB = 1024 * 1024 * 1024;
+    constexpr qsizetype BaseSize = 4 * GiB + 1;
+    qDebug("Filling container with %lld entries", qint64(BaseSize));
+    QElapsedTimer timer;
+    timer.start();
+    try {
+        input.reserve(BaseSize);
+        for (qsizetype i = 0; i < BaseSize; ++i)
+            input.emplace(i, i);
+    } catch (const std::bad_alloc &) {
+        QSKIP("Could not allocate 4 Gi entries.");
+    }
+    qDebug("Created dataset in %lld ms", timer.elapsed());
+}
+
+template <class T>
+void tst_QDataStream::stream_big()
+{
+#if QT_POINTER_SIZE > 4
+    QElapsedTimer timer;
+    T input;
+    fill(input);
+    QByteArray ba;
+    QDataStream inputstream(&ba, QIODevice::WriteOnly);
+    timer.start();
+    try {
+        inputstream << input;
+    } catch (const std::bad_alloc &) {
+        QSKIP("Not enough memory to copy into QDataStream.");
+    }
+    qDebug("Streamed into QDataStream in %lld ms", timer.elapsed());
+    T output;
+    QDataStream outputstream(ba);
+    timer.start();
+    try {
+        outputstream >> output;
+    } catch (const std::bad_alloc &) {
+        QSKIP("Not enough memory to copy out of QDataStream.");
+    }
+    qDebug("Streamed out of QDataStream in %lld ms", timer.elapsed());
+    QCOMPARE(input.size(), output.size());
+    QCOMPARE(input, output);
+#else
+    QSKIP("This test is 64-bit only.");
+#endif
+}
+
+void tst_QDataStream::stream_bigQString()
+{
+    stream_big<QString>();
+}
+void tst_QDataStream::stream_bigQList()
+{
+    stream_big<QList<char>>();
+}
+
+void tst_QDataStream::stream_bigQSet()
+{
+    stream_big<QSet<qsizetype>>();
+}
+
+void tst_QDataStream::stream_bigQMap()
+{
+    stream_big<QMap<qsizetype, qsizetype>>();
+}
+void tst_QDataStream::stream_bigQHash()
+{
+    stream_big<QHash<qsizetype, qsizetype>>();
+}
+
+QTEST_MAIN(tst_QDataStream)
+
+#include "tst_manualqdatastream.moc"