Prevent huge arrays to overflow the JS stack during GC

The JS stack is used as a worklist while marking in order to prevent
recursion overflowing the C stack. Now if all contents of an array are
pushed onto the stack, it can easily cause an overflow. To prevent this,
drain the stack periodically.

This is fix that should not go into 5.11, as it's already fixed there by
using a ValueArray that will have this exact behavior.

Change-Id: Id5bd28879f6ef0265344d9a70c25f6c66b067309
Task-number: QTBUG-62087
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

1 files changed, 23 insertions, 2 deletions

diff --git a/src/qml/jsruntime/qv4arraydata.cpp b/src/qml/jsruntime/qv4arraydata.cpp
index 6d1fb62e0a..dac899e23f 100644
--- a/src/qml/jsruntime/qv4arraydata.cpp
+++ b/src/qml/jsruntime/qv4arraydata.cpp
@@ -235,18 +235,39 @@ void ArrayData::ensureAttributes(Object *o)
     ArrayData::realloc(o, Heap::ArrayData::Simple, 0, true);
 }
 
+// Note: This function, and the calls from SimpleArrayData::markObjects, is a partial backport
+// of the functionality of ValueArray in 5.11. It prevents huge arrays to overflow the JS stack
+// during the mark phase by draining it while marking the array contents.
+static void drainMarkStack(QV4::ExecutionEngine *engine, Value *markBase)
+{
+    while (engine->jsStackTop > markBase) {
+        Heap::Base *h = engine->popForGC();
+        Q_ASSERT(h); // at this point we should only have Heap::Base objects in this area on the stack. If not, weird things might happen.
+        Q_ASSERT (h->vtable()->markObjects);
+        h->vtable()->markObjects(h, engine);
+    }
+}
 
 void SimpleArrayData::markObjects(Heap::Base *d, ExecutionEngine *e)
 {
+    Value *markBase = e->jsStackTop;
+    const auto *maxMarkStack = markBase + 32 * 1024;
+
     Heap::SimpleArrayData *dd = static_cast<Heap::SimpleArrayData *>(d);
     uint end = dd->offset + dd->len;
     if (end > dd->alloc) {
-        for (uint i = 0; i < end - dd->alloc; ++i)
+        for (uint i = 0; i < end - dd->alloc; ++i) {
+            if (e->jsStackTop > maxMarkStack)
+                drainMarkStack(e, markBase);
             dd->arrayData[i].mark(e);
+        }
         end = dd->alloc;
     }
-    for (uint i = dd->offset; i < end; ++i)
+    for (uint i = dd->offset; i < end; ++i) {
+        if (e->jsStackTop > maxMarkStack)
+            drainMarkStack(e, markBase);
         dd->arrayData[i].mark(e);
+    }
 }
 
 ReturnedValue SimpleArrayData::get(const Heap::ArrayData *d, uint index)