Work around Linux libc overflow in mmap64

The mmap64 functions in all Linux libc fail to properly check that the
value fits in the system call parameter. I guess the developers just
said "16 PB are enough for everyone"...

Change-Id: Ic39b2c4fd9c84522a8fafffd14ac91567ce09c09
Reviewed-by: Sami Nurmenniemi <sami.nurmenniemi@qt.io>
Reviewed-by: Olivier Goffart (Woboq GmbH) <ogoffart@woboq.com>

1 files changed, 14 insertions, 1 deletions

diff --git a/src/corelib/io/qfsfileengine_unix.cpp b/src/corelib/io/qfsfileengine_unix.cpp
index bf2f47d399..f8e31ed92b 100644
--- a/src/corelib/io/qfsfileengine_unix.cpp
+++ b/src/corelib/io/qfsfileengine_unix.cpp
@@ -688,6 +688,19 @@ QDateTime QFSFileEngine::fileTime(FileTime time) const
 
 uchar *QFSFileEnginePrivate::map(qint64 offset, qint64 size, QFile::MemoryMapFlags flags)
 {
+#if (defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)) && Q_PROCESSOR_WORDSIZE == 4
+    // The Linux mmap2 system call on 32-bit takes a page-shifted 32-bit
+    // integer so the maximum offset is 1 << (32+12) (the shift is always 12,
+    // regardless of the actual page size). Unfortunately, the mmap64()
+    // function is known to be broken in all Linux libcs (glibc, uclibc, musl
+    // and Bionic): all of them do the right shift, but don't confirm that the
+    // result fits into the 32-bit parameter to the kernel.
+
+    static qint64 MaxFileOffset = (Q_INT64_C(1) << (32+12)) - 1;
+#else
+    static qint64 MaxFileOffset = std::numeric_limits<QT_OFF_T>::max();
+#endif
+
     Q_Q(QFSFileEngine);
     Q_UNUSED(flags);
     if (openMode == QIODevice::NotOpen) {
@@ -695,7 +708,7 @@ uchar *QFSFileEnginePrivate::map(qint64 offset, qint64 size, QFile::MemoryMapFla
         return 0;
     }
 
-    if (offset < 0 || offset != qint64(QT_OFF_T(offset))
+    if (offset < 0 || offset > MaxFileOffset
             || size < 0 || quint64(size) > quint64(size_t(-1))) {
         q->setError(QFile::UnspecifiedError, qt_error_string(int(EINVAL)));
         return 0;