Fix 32/64-bit issues with QFile::map/unmap() on *nix systems

We would previously silently ignore overflows in 32-bit systems and not
properly support 64-bit offsets in systems that support it because of
integer overflow.

There was also a problem that could prevent unmap from succeeding,
because we were passing the wrong length argument.

Task-number: QT-1594
Reviewed-by: Thiago Macieira

2 files changed, 22 insertions, 11 deletions

diff --git a/src/corelib/io/qfsfileengine_p.h b/src/corelib/io/qfsfileengine_p.h
index 66e0219889..ee127c1313 100644
--- a/src/corelib/io/qfsfileengine_p.h
+++ b/src/corelib/io/qfsfileengine_p.h
@@ -114,7 +114,7 @@ public:
     mutable int cachedFd;
     mutable DWORD fileAttrib;
 #else
-    QHash<uchar *, QPair<int /*offset*/, int /*handle|len*/> > maps;
+    QHash<uchar *, QPair<int /*offset % PageSize*/, size_t /*length + offset % PageSize*/> > maps;
     mutable QT_STATBUF st;
 #endif
     int fd;
diff --git a/src/corelib/io/qfsfileengine_unix.cpp b/src/corelib/io/qfsfileengine_unix.cpp
index b0cddaaae1..d3466852f9 100644
--- a/src/corelib/io/qfsfileengine_unix.cpp
+++ b/src/corelib/io/qfsfileengine_unix.cpp
@@ -1243,34 +1243,45 @@ uchar *QFSFileEnginePrivate::map(qint64 offset, qint64 size, QFile::MemoryMapFla
         q->setError(QFile::PermissionsError, qt_error_string(int(EACCES)));
         return 0;
     }
-    if (offset < 0) {
+
+    if (offset < 0 || offset != qint64(QT_OFF_T(offset))
+            || size < 0 || size > (size_t)-1) {
         q->setError(QFile::UnspecifiedError, qt_error_string(int(EINVAL)));
         return 0;
     }
+
     int access = 0;
     if (openMode & QIODevice::ReadOnly) access |= PROT_READ;
     if (openMode & QIODevice::WriteOnly) access |= PROT_WRITE;
 
-    int pagesSize = getpagesize();
-    int realOffset = offset / pagesSize;
-    int extra = offset % pagesSize;
+    int pageSize = getpagesize();
+    int extra = offset % pageSize;
+
+    if (size + extra > (size_t)-1) {
+        q->setError(QFile::UnspecifiedError, qt_error_string(int(EINVAL)));
+        return 0;
+    }
+
+    size_t realSize = (size_t)size + extra;
+    QT_OFF_T realOffset = QT_OFF_T(offset);
+    realOffset &= ~(QT_OFF_T(pageSize));
 
 #ifdef Q_OS_SYMBIAN
     void *mapAddress;
-    TRAPD(err,     mapAddress = mmap((void*)0, (size_t)size + extra,
-                   access, MAP_SHARED, nativeHandle(), realOffset * pagesSize));
+    TRAPD(err,     mapAddress = QT_MMAP((void*)0, realSize,
+                   access, MAP_SHARED, nativeHandle(), realOffset));
     if (err != KErrNone) {
         qWarning("OpenC bug: leave from mmap %d", err);
         mapAddress = MAP_FAILED;
         errno = EINVAL;
     }
 #else
-     void *mapAddress = mmap((void*)0, (size_t)size + extra,
-                    access, MAP_SHARED, nativeHandle(), realOffset * pagesSize);
+    void *mapAddress = QT_MMAP((void*)0, realSize,
+                   access, MAP_SHARED, nativeHandle(), realOffset);
 #endif
     if (MAP_FAILED != mapAddress) {
         uchar *address = extra + static_cast<uchar*>(mapAddress);
-        maps[address] = QPair<int,int>(extra, size);
+        maps[address] = QPair<int,size_t>(extra, realSize);
         return address;
     }
 
@@ -1300,7 +1311,7 @@ bool QFSFileEnginePrivate::unmap(uchar *ptr)
     }
 
     uchar *start = ptr - maps[ptr].first;
-    int len = maps[ptr].second;
+    size_t len = maps[ptr].second;
     if (-1 == munmap(start, len)) {
         q->setError(QFile::UnspecifiedError, qt_error_string(errno));
         return false;