From e211ab76d766878b4dbe88901b9a7a4a70ce7332 Mon Sep 17 00:00:00 2001
From: Andre de la Rocha <andre.rocha@qt.io>
Date: Thu, 18 Jan 2018 18:43:25 +0100
Subject: Handle OOM condition in the validation of plugin metadata
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A large plugin dll (e.g., one with many MB of debug info) could cause
a 32-bit application to crash when the entire dll file could not fit
within the address space of the process. The code for validating a
plugin library and retrieving metadata from it first tried to map
the entire file in memory, which failed for large files, returning
NULL. In this case, the code tried then to read the entire file via
QFile::readAll(), which deep below caused a bad_alloc exception in
malloc, resulting in the termination of the application. This change
handles the case where the library could not be mapped into memory,
in spite of memory mapping being supported, by reporting the error
and returning false, making the plugin unavailable.

[ChangeLog][QtCore][QPluginLoader] Fixed a bug that would cause the
Qt plugin scanning system to allocate too much memory and possibly
crash the process.

Task-number: QTBUG-65197
Change-Id: I8c7235d86175c9fcd2b87fcb1151570da9b9ebe3
Reviewed-by: Jan Arve Sæther <jan-arve.saether@qt.io>
---
 src/corelib/plugin/qlibrary.cpp | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

(limited to 'src/corelib/plugin/qlibrary.cpp')

diff --git a/src/corelib/plugin/qlibrary.cpp b/src/corelib/plugin/qlibrary.cpp
index ebad7f1751..9e6e70756f 100644
--- a/src/corelib/plugin/qlibrary.cpp
+++ b/src/corelib/plugin/qlibrary.cpp
@@ -237,21 +237,34 @@ static bool findPatternUnloaded(const QString &library, QLibraryPrivate *lib)
         if (lib)
             lib->errorString = file.errorString();
         if (qt_debug_component()) {
-            qWarning("%s: %s", (const char*) QFile::encodeName(library),
+            qWarning("%s: %s", QFile::encodeName(library).constData(),
                 qPrintable(QSystemError::stdString()));
         }
         return false;
     }
 
     QByteArray data;
-    const char *filedata = 0;
     ulong fdlen = file.size();
-    filedata = (char *) file.map(0, fdlen);
+    const char *filedata = reinterpret_cast<char *>(file.map(0, fdlen));
+
     if (filedata == 0) {
-        // try reading the data into memory instead
-        data = file.readAll();
-        filedata = data.constData();
-        fdlen = data.size();
+        if (uchar *mapdata = file.map(0, 1)) {
+            file.unmap(mapdata);
+            // Mapping is supported, but failed for the entire file, likely due to OOM.
+            // Return false, as readAll() would cause a bad_alloc and terminate the process.
+            if (lib)
+                lib->errorString = QLibrary::tr("Out of memory while loading plugin '%1'.").arg(library);
+            if (qt_debug_component()) {
+                qWarning("%s: %s", QFile::encodeName(library).constData(),
+                    qPrintable(QSystemError::stdString(ENOMEM)));
+            }
+            return false;
+        } else {
+            // Try reading the data into memory instead.
+            data = file.readAll();
+            filedata = data.constData();
+            fdlen = data.size();
+        }
     }
 
     /*
@@ -745,7 +758,7 @@ void QLibraryPrivate::updatePluginState()
         if (qt_debug_component()) {
             qWarning("In %s:\n"
                  "  Plugin uses incompatible Qt library (%d.%d.%d) [%s]",
-                 (const char*) QFile::encodeName(fileName),
+                 QFile::encodeName(fileName).constData(),
                  (qt_version&0xff0000) >> 16, (qt_version&0xff00) >> 8, qt_version&0xff,
                  debug ? "debug" : "release");
         }
-- 
cgit v1.2.3