QML: Check for stack overflows when creating objects

Fixes: QTBUG-106875
Change-Id: I3b0abda6948b79a9e3cf263f27885037fff1804c
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>
Reviewed-by: Sami Shalayel <sami.shalayel@qt.io>
(cherry picked from commit edc01fbfa430d6f0ce66f1871ab28e0f691ee252)

4 files changed, 41 insertions, 3 deletions

diff --git a/src/qml/jsruntime/qv4engine_p.h b/src/qml/jsruntime/qv4engine_p.h
index e033c5b067..0f42b411e8 100644
--- a/src/qml/jsruntime/qv4engine_p.h
+++ b/src/qml/jsruntime/qv4engine_p.h
@@ -723,6 +723,9 @@ public:
     QQmlRefPointer<ExecutableCompilationUnit> loadModule(const QUrl &_url, const ExecutableCompilationUnit *referrer = nullptr);
 
 private:
+    template<int Frames>
+    friend struct ExecutionEngineCallDepthRecorder;
+
 #if QT_CONFIG(qml_debug)
     QScopedPointer<QV4::Debugging::Debugger> m_debugger;
     QScopedPointer<QV4::Profiling::Profiler> m_profiler;
@@ -753,14 +756,17 @@ private:
 };
 
 #define CHECK_STACK_LIMITS(v4) if ((v4)->checkStackLimits()) return Encode::undefined(); \
-    ExecutionEngineCallDepthRecorder _executionEngineCallDepthRecorder(v4);
+    ExecutionEngineCallDepthRecorder<1> _executionEngineCallDepthRecorder(v4);
 
+template<int Frames = 1>
 struct ExecutionEngineCallDepthRecorder
 {
     ExecutionEngine *ee;
 
-    ExecutionEngineCallDepthRecorder(ExecutionEngine *e): ee(e) { ++ee->callDepth; }
-    ~ExecutionEngineCallDepthRecorder() { --ee->callDepth; }
+    ExecutionEngineCallDepthRecorder(ExecutionEngine *e): ee(e) { ee->callDepth += Frames; }
+    ~ExecutionEngineCallDepthRecorder() { ee->callDepth -= Frames; }
+
+    bool hasOverflow() const { return ee->callDepth >= ExecutionEngine::maxCallDepth; }
 };
 
 inline bool ExecutionEngine::checkStackLimits()
diff --git a/src/qml/qml/qqmlincubator.cpp b/src/qml/qml/qqmlincubator.cpp
index caf419b778..b8227c2131 100644
--- a/src/qml/qml/qqmlincubator.cpp
+++ b/src/qml/qml/qqmlincubator.cpp
@@ -279,6 +279,20 @@ void QQmlIncubatorPrivate::incubate(QQmlInstantiationInterrupt &i)
     // get a copy of the engine pointer as it might get reset;
     QQmlEnginePrivate *enginePriv = this->enginePriv;
 
+    // Incubating objects takes quite a bit more stack space than our usual V4 function
+    enum { EstimatedSizeInV4Frames = 2 };
+    QV4::ExecutionEngineCallDepthRecorder<EstimatedSizeInV4Frames> callDepthRecorder(
+                compilationUnit->engine);
+    if (callDepthRecorder.hasOverflow()) {
+        QQmlError error;
+        error.setMessageType(QtCriticalMsg);
+        error.setUrl(compilationUnit->url());
+        error.setDescription(QQmlComponent::tr("Maximum call stack size exceeded."));
+        errors << error;
+        progress = QQmlIncubatorPrivate::Completed;
+        goto finishIncubate;
+    }
+
     if (!vmeGuard.isOK()) {
         QQmlError error;
         error.setMessageType(QtInfoMsg);
diff --git a/tests/auto/quick/qquickloader/data/overflow.qml b/tests/auto/quick/qquickloader/data/overflow.qml
new file mode 100644
index 0000000000..8ca196e2ed
--- /dev/null
+++ b/tests/auto/quick/qquickloader/data/overflow.qml
@@ -0,0 +1,5 @@
+import QtQuick 2.15
+
+Loader {
+    source: "overflow.qml"
+}
diff --git a/tests/auto/quick/qquickloader/tst_qquickloader.cpp b/tests/auto/quick/qquickloader/tst_qquickloader.cpp
index db678ae5a1..4539a89cb3 100644
--- a/tests/auto/quick/qquickloader/tst_qquickloader.cpp
+++ b/tests/auto/quick/qquickloader/tst_qquickloader.cpp
@@ -134,6 +134,8 @@ private slots:
     void setSourceAndCheckStatus();
     void asyncLoaderRace();
     void noEngine();
+
+    void stackOverflow();
 };
 
 Q_DECLARE_METATYPE(QList<QQmlError>)
@@ -1530,6 +1532,17 @@ void tst_QQuickLoader::noEngine()
     QTRY_COMPARE(o->property("changes").toInt(), 1);
 }
 
+void tst_QQuickLoader::stackOverflow()
+{
+    QQmlEngine engine;
+    const QUrl url = testFileUrl("overflow.qml");
+    QQmlComponent component(&engine, url);
+    QVERIFY2(component.isReady(), qPrintable(component.errorString()));
+    const QString message = url.toString() + QStringLiteral(": Maximum call stack size exceeded.");
+    QTest::ignoreMessage(QtCriticalMsg, qPrintable(message));
+    QScopedPointer<QObject> o(component.create());
+}
+
 QTEST_MAIN(tst_QQuickLoader)
 
 #include "tst_qquickloader.moc"