Make sure we don't try to ask QByteArray to allocate too much

QFile::readAll could be asked to read a file that is over 1 GB in size
and thus cause an assertion:

ASSERT failure in qAllocMore: "Requested size is too large!", ...

The idea behind the existing code was correct, but the value was wrong.
It prevented overflow of the integer size request, but didn't prevent
overflowing the storage size.

Change-Id: I072e6e419f47b639454f3fd96deb0f88d03e960c
Reviewed-by: Martin Smith <martin.smith@digia.com>

3 files changed, 14 insertions, 2 deletions

diff --git a/src/corelib/io/qiodevice.cpp b/src/corelib/io/qiodevice.cpp
index 0709a93bad..72ec6ff403 100644
--- a/src/corelib/io/qiodevice.cpp
+++ b/src/corelib/io/qiodevice.cpp
@@ -1025,7 +1025,7 @@ QByteArray QIODevice::readAll()
     } else {
         // Read it all in one go.
         // If resize fails, don't read anything.
-        if (readBytes + theSize - d->pos > INT_MAX)
+        if (quint64(readBytes + theSize - d->pos) > QByteArray::MaxSize)
             return QByteArray();
         result.resize(int(readBytes + theSize - d->pos));
         readBytes += read(result.data() + readBytes, result.size() - readBytes);
diff --git a/src/corelib/tools/qbytearray.cpp b/src/corelib/tools/qbytearray.cpp
index 6ac442d27b..d8b2efbef3 100644
--- a/src/corelib/tools/qbytearray.cpp
+++ b/src/corelib/tools/qbytearray.cpp
@@ -63,7 +63,7 @@ int qFindByteArray(
 int qAllocMore(int alloc, int extra) Q_DECL_NOTHROW
 {
     Q_ASSERT(alloc >= 0 && extra >= 0);
-    Q_ASSERT_X(alloc < (1 << 30) - extra, "qAllocMore", "Requested size is too large!");
+    Q_ASSERT_X(uint(alloc) < QByteArray::MaxSize, "qAllocMore", "Requested size is too large!");
 
     unsigned nalloc = qNextPowerOfTwo(alloc + extra);
 
@@ -777,6 +777,15 @@ static inline char qToLower(char c)
 */
 
 /*!
+    \variable QByteArray::MaxSize
+    \internal
+    \since 5.4
+
+    The maximum size of a QByteArray, in bytes. Also applies to a the maximum
+    storage size of QString and QVector, though not the number of elements.
+*/
+
+/*!
     \enum QByteArray::Base64Option
     \since 5.2
 
diff --git a/src/corelib/tools/qbytearray.h b/src/corelib/tools/qbytearray.h
index f13b1c16cd..3bcc7b1f2a 100644
--- a/src/corelib/tools/qbytearray.h
+++ b/src/corelib/tools/qbytearray.h
@@ -173,6 +173,9 @@ private:
     typedef QTypedArrayData<char> Data;
 
 public:
+    // undocumented:
+    static const quint64 MaxSize = (1 << 30) - sizeof(Data);
+
     enum Base64Option {
         Base64Encoding = 0,
         Base64UrlEncoding = 1,