QV4DebugService: Reduce unnecessary recursion and redundancy

Large parts of the protocol are unnecessary. There is no reason to send
a separate chunk of "handles" with almost every reply. The refs are
given as part of the regular data and if the client wants to find out
more, it can do further lookups. Also, it makes no sense to encode the
function and script names as objects, as they are in fact not JavaScript
objects.

Unfortunately these cleanups require some cooperation from the client.
Older clients will misbehave if we just drop the redundancy. Therefore,
we introduce parameters which the client can explicitly set with
the "connect" message. redundantRefs tells the service if redundant
references are required, namesAsObjects tells it if script and function
names have to be sent as objects/

Once we can require clients that support these options, we can drop the
code that generates redundant data.

Also, fix tst_qv4debugger::evaluateExpression() to actually check all
the expressions evaluated, not only the first and second one.

Task-number: QTBUG-42435
Change-Id: If93d2a2b9d0b8035f85dbef871bc1b03f199171d
Reviewed-by: hjk <hjk@qt.io>
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

6 files changed, 134 insertions, 53 deletions

diff --git a/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.cpp b/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.cpp
index aed2759383..5d2e754057 100644
--- a/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.cpp
+++ b/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.cpp
@@ -118,15 +118,18 @@ int QV4DataCollector::encodeScopeType(QV4::Heap::ExecutionContext::ContextType s
     }
 }
 
-QV4DataCollector::QV4DataCollector(QV4::ExecutionEngine *engine) : m_engine(engine)
+QV4DataCollector::QV4DataCollector(QV4::ExecutionEngine *engine)
+    : m_engine(engine), m_namesAsObjects(true), m_redundantRefs(true)
 {
     m_values.set(engine, engine->newArrayObject());
 }
 
+// TODO: Directly call addRef() once we don't need to support redundantRefs anymore
 QV4DataCollector::Ref QV4DataCollector::collect(const QV4::ScopedValue &value)
 {
     Ref ref = addRef(value);
-    m_collectedRefs.append(ref);
+    if (m_redundantRefs)
+        m_collectedRefs.append(ref);
     return ref;
 }
 
@@ -190,24 +193,33 @@ const QV4::Object *collectProperty(const QV4::ScopedValue &value, QV4::Execution
     }
 }
 
-QJsonObject QV4DataCollector::lookupRef(Ref ref)
+QJsonObject QV4DataCollector::lookupRef(Ref ref, bool deep)
 {
     QJsonObject dict;
-    if (lookupSpecialRef(ref, &dict))
-        return dict;
+
+    if (m_namesAsObjects) {
+        if (lookupSpecialRef(ref, &dict))
+            return dict;
+    }
+
+    if (m_redundantRefs)
+        deep = true;
 
     dict.insert(QStringLiteral("handle"), qint64(ref));
     QV4::Scope scope(engine());
     QV4::ScopedValue value(scope, getValue(ref));
 
-    if (const QV4::Object *o = collectProperty(value, engine(), dict))
-        dict.insert(QStringLiteral("properties"), collectProperties(o));
+    const QV4::Object *object = collectProperty(value, engine(), dict);
+    if (deep && object)
+        dict.insert(QStringLiteral("properties"), collectProperties(object));
 
     return dict;
 }
 
+// TODO: Drop this method once we don't need to support namesAsObjects anymore
 QV4DataCollector::Ref QV4DataCollector::addFunctionRef(const QString &functionName)
 {
+    Q_ASSERT(m_namesAsObjects);
     Ref ref = addRef(QV4::Primitive::emptyValue(), false);
 
     QJsonObject dict;
@@ -220,8 +232,10 @@ QV4DataCollector::Ref QV4DataCollector::addFunctionRef(const QString &functionNa
     return ref;
 }
 
+// TODO: Drop this method once we don't need to support namesAsObjects anymore
 QV4DataCollector::Ref QV4DataCollector::addScriptRef(const QString &scriptName)
 {
+    Q_ASSERT(m_namesAsObjects);
     Ref ref = addRef(QV4::Primitive::emptyValue(), false);
 
     QJsonObject dict;
@@ -250,6 +264,7 @@ bool QV4DataCollector::collectScope(QJsonObject *dict, int frameNr, int scopeNr)
     if (!ctxt)
         return false;
 
+    Refs collectedRefs;
     QV4::ScopedValue v(scope);
     int nFormals = ctxt->formalCount();
     for (unsigned i = 0, ei = nFormals; i != ei; ++i) {
@@ -258,7 +273,7 @@ bool QV4DataCollector::collectScope(QJsonObject *dict, int frameNr, int scopeNr)
             qName = name->string;
         names.append(qName);
         v = ctxt->argument(i);
-        collect(v);
+        collectedRefs.append(collect(v));
     }
 
     for (unsigned i = 0, ei = ctxt->variableCount(); i != ei; ++i) {
@@ -267,19 +282,24 @@ bool QV4DataCollector::collectScope(QJsonObject *dict, int frameNr, int scopeNr)
             qName = name->string;
         names.append(qName);
         v = ctxt->d()->locals[i];
-        collect(v);
+        collectedRefs.append(collect(v));
     }
 
     QV4::ScopedObject scopeObject(scope, engine()->newObject());
 
-    Q_ASSERT(names.size() == m_collectedRefs.size());
-    for (int i = 0, ei = m_collectedRefs.size(); i != ei; ++i)
+    Q_ASSERT(names.size() == collectedRefs.size());
+    for (int i = 0, ei = collectedRefs.size(); i != ei; ++i)
         scopeObject->put(engine(), names.at(i),
-                         QV4::Value::fromReturnedValue(getValue(m_collectedRefs.at(i))));
+                         QV4::Value::fromReturnedValue(getValue(collectedRefs.at(i))));
 
     Ref scopeObjectRef = addRef(scopeObject);
-    dict->insert(QStringLiteral("ref"), qint64(scopeObjectRef));
-    m_collectedRefs.append(scopeObjectRef);
+    if (m_redundantRefs) {
+        dict->insert(QStringLiteral("ref"), qint64(scopeObjectRef));
+        m_collectedRefs.append(scopeObjectRef);
+    } else {
+        *dict = lookupRef(scopeObjectRef, true);
+    }
+
     return true;
 }
 
@@ -291,15 +311,16 @@ QJsonObject toRef(QV4DataCollector::Ref ref) {
 
 QJsonObject QV4DataCollector::buildFrame(const QV4::StackFrame &stackFrame, int frameNr)
 {
-    QV4DataCollector::Ref ref;
-
     QJsonObject frame;
     frame[QLatin1String("index")] = frameNr;
     frame[QLatin1String("debuggerFrame")] = false;
-    ref = addFunctionRef(stackFrame.function);
-    frame[QLatin1String("func")] = toRef(ref);
-    ref = addScriptRef(stackFrame.source);
-    frame[QLatin1String("script")] = toRef(ref);
+    if (m_namesAsObjects) {
+        frame[QLatin1String("func")] = toRef(addFunctionRef(stackFrame.function));
+        frame[QLatin1String("script")] = toRef(addScriptRef(stackFrame.source));
+    } else {
+        frame[QLatin1String("func")] = stackFrame.function;
+        frame[QLatin1String("script")] = stackFrame.source;
+    }
     frame[QLatin1String("line")] = stackFrame.line - 1;
     if (stackFrame.column >= 0)
         frame[QLatin1String("column")] = stackFrame.column;
@@ -338,15 +359,17 @@ QJsonObject QV4DataCollector::buildFrame(const QV4::StackFrame &stackFrame, int
     return frame;
 }
 
+// TODO: Drop this method once we don't need to support redundantRefs anymore
 QJsonArray QV4DataCollector::flushCollectedRefs()
 {
+    Q_ASSERT(m_redundantRefs);
     QJsonArray refs;
     std::sort(m_collectedRefs.begin(), m_collectedRefs.end());
     for (int i = 0, ei = m_collectedRefs.size(); i != ei; ++i) {
         QV4DataCollector::Ref ref = m_collectedRefs.at(i);
         if (i > 0 && ref == m_collectedRefs.at(i - 1))
             continue;
-        refs.append(lookupRef(ref));
+        refs.append(lookupRef(ref, true));
     }
 
     m_collectedRefs.clear();
@@ -358,6 +381,8 @@ void QV4DataCollector::clear()
     m_values.set(engine(), engine()->newArrayObject());
     m_collectedRefs.clear();
     m_specialRefs.clear();
+    m_namesAsObjects = true;
+    m_redundantRefs = true;
 }
 
 QV4DataCollector::Ref QV4DataCollector::addRef(QV4::Value value, bool deduplicate)
@@ -401,8 +426,10 @@ QV4::ReturnedValue QV4DataCollector::getValue(Ref ref)
     return array->getIndexed(ref, Q_NULLPTR);
 }
 
+// TODO: Drop this method once we don't need to support namesAsObjects anymore
 bool QV4DataCollector::lookupSpecialRef(Ref ref, QJsonObject *dict)
 {
+    Q_ASSERT(m_namesAsObjects);
     SpecialRefs::const_iterator it = m_specialRefs.constFind(ref);
     if (it == m_specialRefs.cend())
         return false;
@@ -440,7 +467,8 @@ QJsonObject QV4DataCollector::collectAsJson(const QString &name, const QV4::Scop
     if (value->isManaged() && !value->isString()) {
         Ref ref = addRef(value);
         dict.insert(QStringLiteral("ref"), qint64(ref));
-        m_collectedRefs.append(ref);
+        if (m_redundantRefs)
+            m_collectedRefs.append(ref);
     }
 
     collectProperty(value, engine(), dict);
diff --git a/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.h b/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.h
index fd6356f22e..2c2514a1b3 100644
--- a/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.h
+++ b/src/plugins/qmltooling/qmldbg_debugger/qv4datacollector.h
@@ -66,34 +66,42 @@ public:
 
     QV4DataCollector(QV4::ExecutionEngine *engine);
 
-    Ref collect(const QV4::ScopedValue &value);
-    Ref addFunctionRef(const QString &functionName);
-    Ref addScriptRef(const QString &scriptName);
+    Ref collect(const QV4::ScopedValue &value);      // only for redundantRefs
+    Ref addFunctionRef(const QString &functionName); // only for namesAsObjects
+    Ref addScriptRef(const QString &scriptName);     // only for namesAsObjects
+
+    void setNamesAsObjects(bool namesAsObjects) { m_namesAsObjects = namesAsObjects; }
+    bool namesAsObjects() const { return m_namesAsObjects; }
+
+    void setRedundantRefs(bool redundantRefs) { m_redundantRefs = redundantRefs; }
+    bool redundantRefs() const { return m_redundantRefs; }
 
     bool isValidRef(Ref ref) const;
-    QJsonObject lookupRef(Ref ref);
+    QJsonObject lookupRef(Ref ref, bool deep);
 
     bool collectScope(QJsonObject *dict, int frameNr, int scopeNr);
     QJsonObject buildFrame(const QV4::StackFrame &stackFrame, int frameNr);
 
     QV4::ExecutionEngine *engine() const { return m_engine; }
-    QJsonArray flushCollectedRefs();
+    QJsonArray flushCollectedRefs(); // only for redundantRefs
     void clear();
 
 private:
     Ref addRef(QV4::Value value, bool deduplicate = true);
     QV4::ReturnedValue getValue(Ref ref);
-    bool lookupSpecialRef(Ref ref, QJsonObject *dict);
+    bool lookupSpecialRef(Ref ref, QJsonObject *dict); // only for namesAsObjects
 
     QJsonArray collectProperties(const QV4::Object *object);
     QJsonObject collectAsJson(const QString &name, const QV4::ScopedValue &value);
     void collectArgumentsInContext();
 
     QV4::ExecutionEngine *m_engine;
-    Refs m_collectedRefs;
+    Refs m_collectedRefs;                        // only for redundantRefs
     QV4::PersistentValue m_values;
-    typedef QHash<Ref, QJsonObject> SpecialRefs;
-    SpecialRefs m_specialRefs;
+    typedef QHash<Ref, QJsonObject> SpecialRefs; // only for namesAsObjects
+    SpecialRefs m_specialRefs;                   // only for namesAsObjects
+    bool m_namesAsObjects;
+    bool m_redundantRefs;
 };
 
 QT_END_NAMESPACE
diff --git a/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.cpp b/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.cpp
index df316c1ae6..a624cf22a4 100644
--- a/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.cpp
+++ b/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.cpp
@@ -148,7 +148,7 @@ void BacktraceJob::run()
         result.insert(QStringLiteral("toFrame"), fromFrame + frameArray.size());
         result.insert(QStringLiteral("frames"), frameArray);
     }
-    collectedRefs = collector->flushCollectedRefs();
+    flushRedundantRefs();
 }
 
 FrameJob::FrameJob(QV4DataCollector *collector, int frameNr) :
@@ -163,7 +163,7 @@ void FrameJob::run()
         success = false;
     } else {
         result = collector->buildFrame(frames[frameNr], frameNr);
-        collectedRefs = collector->flushCollectedRefs();
+        flushRedundantRefs();
         success = true;
     }
 }
@@ -193,7 +193,7 @@ void ScopeJob::run()
     result[QLatin1String("index")] = scopeNr;
     result[QLatin1String("frameIndex")] = frameNr;
     result[QLatin1String("object")] = object;
-    collectedRefs = collector->flushCollectedRefs();
+    flushRedundantRefs();
 }
 
 bool ScopeJob::wasSuccessful() const
@@ -223,9 +223,9 @@ void ValueLookupJob::run()
             exception = QString::fromLatin1("Invalid Ref: %1").arg(ref);
             break;
         }
-        result[QString::number(ref)] = collector->lookupRef(ref);
+        result[QString::number(ref)] = collector->lookupRef(ref, true);
     }
-    collectedRefs = collector->flushCollectedRefs();
+    flushRedundantRefs();
     if (scopeObject)
         engine->popContext();
 }
@@ -246,8 +246,9 @@ void ExpressionEvalJob::handleResult(QV4::ScopedValue &value)
 {
     if (hasExeption())
         exception = value->toQStringNoThrow();
-    result = collector->lookupRef(collector->collect(value));
-    collectedRefs = collector->flushCollectedRefs();
+    result = collector->lookupRef(collector->collect(value), true);
+    if (collector->redundantRefs())
+        collectedRefs = collector->flushCollectedRefs();
 }
 
 const QString &ExpressionEvalJob::exceptionMessage() const
@@ -260,8 +261,10 @@ const QJsonObject &ExpressionEvalJob::returnValue() const
     return result;
 }
 
+// TODO: Drop this method once we don't need to support redundantRefs anymore
 const QJsonArray &ExpressionEvalJob::refs() const
 {
+    Q_ASSERT(collector->redundantRefs());
     return collectedRefs;
 }
 
diff --git a/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.h b/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.h
index 00d3e6206a..eca8710e15 100644
--- a/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.h
+++ b/src/plugins/qmltooling/qmldbg_debugger/qv4debugjob.h
@@ -78,11 +78,24 @@ class CollectJob : public QV4DebugJob
 protected:
     QV4DataCollector *collector;
     QJsonObject result;
-    QJsonArray collectedRefs;
+    QJsonArray collectedRefs; // only for redundantRefs
+
+    void flushRedundantRefs()
+    {
+        if (collector->redundantRefs())
+            collectedRefs = collector->flushCollectedRefs();
+    }
+
 public:
     CollectJob(QV4DataCollector *collector) : collector(collector) {}
     const QJsonObject &returnValue() const { return result; }
-    const QJsonArray &refs() const { return collectedRefs; }
+
+    // TODO: Drop this method once we don't need to support redundantRefs anymore
+    const QJsonArray &refs() const
+    {
+        Q_ASSERT(collector->redundantRefs());
+        return collectedRefs;
+    }
 };
 
 class BacktraceJob: public CollectJob
@@ -133,7 +146,7 @@ class ExpressionEvalJob: public JavaScriptJob
     QV4DataCollector *collector;
     QString exception;
     QJsonObject result;
-    QJsonArray collectedRefs;
+    QJsonArray collectedRefs; // only for redundantRefs
 
 public:
     ExpressionEvalJob(QV4::ExecutionEngine *engine, int frameNr, int context,
@@ -141,7 +154,7 @@ public:
     void handleResult(QV4::ScopedValue &value) override;
     const QString &exceptionMessage() const;
     const QJsonObject &returnValue() const;
-    const QJsonArray &refs() const;
+    const QJsonArray &refs() const; // only for redundantRefs
 };
 
 class GatherSourcesJob: public QV4DebugJob
diff --git a/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.cpp b/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.cpp
index 1d2cc092dc..fbf6397a18 100644
--- a/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.cpp
+++ b/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.cpp
@@ -121,8 +121,18 @@ protected:
         response.insert(QStringLiteral("running"), debugService->debuggerAgent.isRunning());
     }
 
+    QV4DataCollector *saneCollector(QV4Debugger *debugger)
+    {
+        QV4DataCollector *collector = debugger->collector();
+        collector->setNamesAsObjects(debugService->clientRequiresNamesAsObjects());
+        collector->setRedundantRefs(debugService->clientRequiresRedundantRefs());
+        return collector;
+    }
+
+    // TODO: drop this method once we don't need to support redundantRefs anymore.
     void addRefs(const QJsonArray &refs)
     {
+        Q_ASSERT(debugService->clientRequiresRedundantRefs());
         response.insert(QStringLiteral("refs"), refs);
     }
 
@@ -286,7 +296,7 @@ public:
             return;
         }
 
-        BacktraceJob job(debugger->collector(), fromFrame, toFrame);
+        BacktraceJob job(saneCollector(debugger), fromFrame, toFrame);
         debugger->runInEngine(&job);
 
         // response:
@@ -295,7 +305,8 @@ public:
         addSuccess(true);
         addRunning();
         addBody(job.returnValue());
-        addRefs(job.refs());
+        if (debugService->clientRequiresRedundantRefs())
+            addRefs(job.refs());
     }
 };
 
@@ -322,7 +333,7 @@ public:
             return;
         }
 
-        FrameJob job(debugger->collector(), frameNr);
+        FrameJob job(saneCollector(debugger), frameNr);
         debugger->runInEngine(&job);
         if (!job.wasSuccessful()) {
             createErrorResponse(QStringLiteral("frame retrieval failed"));
@@ -337,7 +348,8 @@ public:
         addSuccess(true);
         addRunning();
         addBody(job.returnValue());
-        addRefs(job.refs());
+        if (debugService->clientRequiresRedundantRefs())
+            addRefs(job.refs());
     }
 };
 
@@ -369,7 +381,7 @@ public:
             return;
         }
 
-        ScopeJob job(debugger->collector(), frameNr, scopeNr);
+        ScopeJob job(saneCollector(debugger), frameNr, scopeNr);
         debugger->runInEngine(&job);
         if (!job.wasSuccessful()) {
             createErrorResponse(QStringLiteral("scope retrieval failed"));
@@ -382,7 +394,8 @@ public:
         addSuccess(true);
         addRunning();
         addBody(job.returnValue());
-        addRefs(job.refs());
+        if (debugService->clientRequiresRedundantRefs())
+            addRefs(job.refs());
     }
 };
 
@@ -410,7 +423,7 @@ public:
             debugger = debuggers.first();
         }
 
-        ValueLookupJob job(handles, debugger->collector());
+        ValueLookupJob job(handles, saneCollector(debugger));
         debugger->runInEngine(&job);
         if (!job.exceptionMessage().isEmpty()) {
             createErrorResponse(job.exceptionMessage());
@@ -421,7 +434,8 @@ public:
             addSuccess(true);
             addRunning();
             addBody(job.returnValue());
-            addRefs(job.refs());
+            if (debugService->clientRequiresRedundantRefs())
+                addRefs(job.refs());
         }
     }
 };
@@ -630,7 +644,7 @@ public:
         }
 
         ExpressionEvalJob job(debugger->engine(), frame, context, expression,
-                              debugger->collector());
+                              saneCollector(debugger));
         debugger->runInEngine(&job);
         if (job.hasExeption()) {
             createErrorResponse(job.exceptionMessage());
@@ -640,7 +654,8 @@ public:
             addSuccess(true);
             addRunning();
             addBody(job.returnValue());
-            addRefs(job.refs());
+            if (debugService->clientRequiresRedundantRefs())
+                addRefs(job.refs());
         }
     }
 };
@@ -662,7 +677,7 @@ V8CommandHandler *QV4DebugServiceImpl::v8CommandHandler(const QString &command)
 
 QV4DebugServiceImpl::QV4DebugServiceImpl(QObject *parent) :
     QQmlConfigurableDebugService<QV4DebugService>(1, parent),
-    debuggerAgent(this), theSelectedFrame(0),
+    debuggerAgent(this), theSelectedFrame(0), redundantRefs(true), namesAsObjects(true),
     unknownV8CommandHandler(new UnknownV8CommandHandler)
 {
     addHandler(new V8VersionRequest);
@@ -766,6 +781,14 @@ void QV4DebugServiceImpl::messageReceived(const QByteArray &message)
         TRACE_PROTOCOL(qDebug() << "... type:" << type);
 
         if (type == V4_CONNECT) {
+            QJsonObject parameters = QJsonDocument::fromJson(payload).object();
+            namesAsObjects = true;
+            redundantRefs = true;
+            if (parameters.contains("namesAsObjects"))
+                namesAsObjects = parameters.value("namesAsObjects").toBool();
+            if (parameters.contains("redundantRefs"))
+                namesAsObjects = parameters.value("redundantRefs").toBool();
+
             emit messageToClient(name(), packMessage(type));
             stopWaiting();
         } else if (type == V4_PAUSE) {
diff --git a/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.h b/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.h
index 69e32189b8..bb13890ae4 100644
--- a/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.h
+++ b/src/plugins/qmltooling/qmldbg_debugger/qv4debugservice.h
@@ -86,6 +86,9 @@ public:
     int selectedFrame() const;
     void selectFrame(int frameNr);
 
+    bool clientRequiresRedundantRefs() const { return redundantRefs; }
+    bool clientRequiresNamesAsObjects() const { return namesAsObjects; }
+
     QV4DebuggerAgent debuggerAgent;
 
 protected:
@@ -105,6 +108,9 @@ private:
     static int sequence;
     int theSelectedFrame;
 
+    bool redundantRefs;
+    bool namesAsObjects;
+
     void addHandler(V8CommandHandler* handler);
     QHash<QString, V8CommandHandler*> handlers;
     QScopedPointer<UnknownV8CommandHandler> unknownV8CommandHandler;