V4: Fix mark stack overruns

Instead of applying a heuristic on when to call drain() in unrelated
code, we check the stack limit on each push(). If the soft limit is
reached we try to drain. As drain() itself can push again, we try to
limit the stack size by allowing at most 65 recursions of drain(). If
none of that helps, we crash with a meaningful error message.

This allows us to remove all the hacky drain() calls in other parts of
the code.

Change-Id: Ib979339470da0e85981de8131e7997755b757c71
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

2 files changed, 39 insertions, 24 deletions

diff --git a/src/qml/memory/qv4mm.cpp b/src/qml/memory/qv4mm.cpp
index 35b2ff2749..3ed1bc48fd 100644
--- a/src/qml/memory/qv4mm.cpp
+++ b/src/qml/memory/qv4mm.cpp
@@ -848,16 +848,18 @@ Heap::Object *MemoryManager::allocObjectWithMemberData(const QV4::VTable *vtable
 static uint markStackSize = 0;
 
 MarkStack::MarkStack(ExecutionEngine *engine)
-    : engine(engine)
+    : m_engine(engine)
 {
-    base = (Heap::Base **)engine->gcStack->base();
-    top = base;
-    limit = base + engine->maxGCStackSize()/sizeof(Heap::Base)*3/4;
+    m_base = (Heap::Base **)engine->gcStack->base();
+    m_top = m_base;
+    const size_t size = engine->maxGCStackSize() / sizeof(Heap::Base);
+    m_hardLimit = m_base + size;
+    m_softLimit = m_base + size * 3 / 4;
 }
 
 void MarkStack::drain()
 {
-    while (top > base) {
+    while (m_top > m_base) {
         Heap::Base *h = pop();
         ++markStackSize;
         Q_ASSERT(h); // at this point we should only have Heap::Base objects in this area on the stack. If not, weird things might happen.
@@ -904,20 +906,15 @@ void MemoryManager::collectRoots(MarkStack *markStack)
 
         if (keepAlive)
             qobjectWrapper->mark(markStack);
-
-        if (markStack->top >= markStack->limit)
-            markStack->drain();
     }
 }
 
 void MemoryManager::mark()
 {
     markStackSize = 0;
-
     MarkStack markStack(engine);
     collectRoots(&markStack);
-
-    markStack.drain();
+    // dtor of MarkStack drains
 }
 
 void MemoryManager::sweep(bool lastSweep, ClassDestroyStatsCallback classCountPtr)
@@ -1220,8 +1217,6 @@ void MemoryManager::collectFromJSStack(MarkStack *markStack) const
             Q_ASSERT(m->inUse());
             // Skip pointers to already freed objects, they are bogus as well
             m->mark(markStack);
-            if (markStack->top >= markStack->limit)
-                markStack->drain();
         }
         ++v;
     }
diff --git a/src/qml/memory/qv4mmdefs_p.h b/src/qml/memory/qv4mmdefs_p.h
index 8a53492822..117e8f4360 100644
--- a/src/qml/memory/qv4mmdefs_p.h
+++ b/src/qml/memory/qv4mmdefs_p.h
@@ -53,6 +53,7 @@
 #include <private/qv4global_p.h>
 #include <private/qv4runtimeapi_p.h>
 #include <QtCore/qalgorithms.h>
+#include <QtCore/qmath.h>
 #include <qdebug.h>
 
 QT_BEGIN_NAMESPACE
@@ -270,22 +271,41 @@ Q_STATIC_ASSERT(sizeof(HeapItem) == Chunk::SlotSize);
 Q_STATIC_ASSERT(QT_POINTER_SIZE*8 == Chunk::Bits);
 Q_STATIC_ASSERT((1 << Chunk::BitShift) == Chunk::Bits);
 
-struct MarkStack {
+struct Q_QML_PRIVATE_EXPORT MarkStack {
     MarkStack(ExecutionEngine *engine);
-    Heap::Base **top = nullptr;
-    Heap::Base **base = nullptr;
-    Heap::Base **limit = nullptr;
-    ExecutionEngine *engine;
+    ~MarkStack() { drain(); }
+
     void push(Heap::Base *m) {
-        *top = m;
-        ++top;
-    }
-    Heap::Base *pop() {
-        --top;
-        return *top;
+        *(m_top++) = m;
+
+        if (m_top < m_softLimit)
+            return;
+
+        // If at or above soft limit, partition the remaining space into at most 64 segments and
+        // allow one C++ recursion of drain() per segment, plus one for the fence post.
+        const quintptr segmentSize = qNextPowerOfTwo(quintptr(m_hardLimit - m_softLimit) / 64u);
+        if (m_drainRecursion * segmentSize <= quintptr(m_top - m_softLimit)) {
+            ++m_drainRecursion;
+            drain();
+            --m_drainRecursion;
+        } else if (m_top == m_hardLimit) {
+            qFatal("GC mark stack overrun. Either simplify your application or"
+                   "increase QV4_GC_MAX_STACK_SIZE");
+        }
     }
+
+    ExecutionEngine *engine() const { return m_engine; }
+
+private:
+    Heap::Base *pop() { return *(--m_top); }
     void drain();
 
+    Heap::Base **m_top = nullptr;
+    Heap::Base **m_base = nullptr;
+    Heap::Base **m_softLimit = nullptr;
+    Heap::Base **m_hardLimit = nullptr;
+    ExecutionEngine *m_engine = nullptr;
+    quintptr m_drainRecursion = 0;
 };
 
 // Some helper to automate the generation of our