Re-do QML type compilation of ahead-of-time generated QML cache files

As we currently lack the ability to do type resolution at AOT cache
generation time, we need to re-do the work after loading the cache file,
making us essentially only re-use the code and avoiding the step of
parsing.

Change-Id: I12844692d4766345d8a313b59d21abf1f868e2d1
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

7 files changed, 270 insertions, 3 deletions

diff --git a/src/qml/compiler/qqmlirbuilder.cpp b/src/qml/compiler/qqmlirbuilder.cpp
index 54d0cb4f46..8a7507c92e 100644
--- a/src/qml/compiler/qqmlirbuilder.cpp
+++ b/src/qml/compiler/qqmlirbuilder.cpp
@@ -2061,4 +2061,156 @@ QQmlPropertyData *PropertyResolver::signal(const QString &name, bool *notInRevis
     return 0;
 }
 
+IRLoader::IRLoader(const QV4::CompiledData::Unit *qmlData, QmlIR::Document *output)
+    : unit(qmlData)
+    , output(output)
+{
+    pool = output->jsParserEngine.pool();
+}
+
+void IRLoader::load()
+{
+    output->jsGenerator.stringTable.clear();
+    for (uint i = 0; i < unit->stringTableSize; ++i)
+        output->jsGenerator.stringTable.registerString(unit->stringAt(i));
+
+    for (quint32 i = 0; i < unit->nImports; ++i)
+        output->imports << unit->importAt(i);
+
+    if (unit->flags & QV4::CompiledData::Unit::IsSingleton) {
+        QmlIR::Pragma *p = New<QmlIR::Pragma>();
+        p->location = QV4::CompiledData::Location();
+        p->type = QmlIR::Pragma::PragmaSingleton;
+        output->pragmas << p;
+    }
+
+    output->indexOfRootObject = unit->indexOfRootObject;
+
+    for (uint i = 0; i < unit->nObjects; ++i) {
+        const QV4::CompiledData::Object *serializedObject = unit->objectAt(i);
+        QmlIR::Object *object = loadObject(serializedObject);
+        output->objects.append(object);
+    }
+}
+
+struct FakeExpression : public QQmlJS::AST::NullExpression
+{
+    FakeExpression(int start, int length)
+        : location(start, length)
+    {}
+
+    virtual QQmlJS::AST::SourceLocation firstSourceLocation() const
+    { return location; }
+
+    virtual QQmlJS::AST::SourceLocation lastSourceLocation() const
+    { return location; }
+
+private:
+    QQmlJS::AST::SourceLocation location;
+};
+
+QmlIR::Object *IRLoader::loadObject(const QV4::CompiledData::Object *serializedObject)
+{
+    QmlIR::Object *object = pool->New<QmlIR::Object>();
+    object->init(pool, serializedObject->inheritedTypeNameIndex, serializedObject->idNameIndex);
+
+    object->indexOfDefaultPropertyOrAlias = serializedObject->indexOfDefaultPropertyOrAlias;
+    object->defaultPropertyIsAlias = serializedObject->defaultPropertyIsAlias;
+
+    object->location = serializedObject->location;
+    object->locationOfIdProperty = serializedObject->locationOfIdProperty;
+
+    QVector<int> functionIndices;
+    functionIndices.reserve(serializedObject->nFunctions + serializedObject->nBindings / 2);
+
+    for (uint i = 0; i < serializedObject->nBindings; ++i) {
+        QmlIR::Binding *b = pool->New<QmlIR::Binding>();
+        *static_cast<QV4::CompiledData::Binding*>(b) = serializedObject->bindingTable()[i];
+        object->bindings->append(b);
+        if (b->type == QV4::CompiledData::Binding::Type_Script) {
+            functionIndices.append(b->value.compiledScriptIndex);
+            b->value.compiledScriptIndex = functionIndices.count() - 1;
+
+            QmlIR::CompiledFunctionOrExpression *foe = pool->New<QmlIR::CompiledFunctionOrExpression>();
+            foe->disableAcceleratedLookups = true;
+            foe->nameIndex = 0;
+
+            QQmlJS::AST::ExpressionNode *expr;
+
+            if (b->stringIndex != quint32(0)) {
+                const int start = output->code.length();
+                const QString script = output->stringAt(b->stringIndex);
+                const int length = script.length();
+                output->code.append(script);
+                expr = new (pool) FakeExpression(start, length);
+            } else
+                expr = new (pool) QQmlJS::AST::NullExpression();
+            foe->node = new (pool) QQmlJS::AST::ExpressionStatement(expr); // dummy
+            object->functionsAndExpressions->append(foe);
+        }
+    }
+
+    Q_ASSERT(object->functionsAndExpressions->count == functionIndices.count());
+
+    for (uint i = 0; i < serializedObject->nSignals; ++i) {
+        const QV4::CompiledData::Signal *serializedSignal = serializedObject->signalAt(i);
+        QmlIR::Signal *s = pool->New<QmlIR::Signal>();
+        s->nameIndex = serializedSignal->nameIndex;
+        s->location = serializedSignal->location;
+        s->parameters = pool->New<QmlIR::PoolList<QmlIR::SignalParameter> >();
+
+        for (uint i = 0; i < serializedSignal->nParameters; ++i) {
+            QmlIR::SignalParameter *p = pool->New<QmlIR::SignalParameter>();
+            *static_cast<QV4::CompiledData::Parameter*>(p) = *serializedSignal->parameterAt(i);
+            s->parameters->append(p);
+        }
+
+        object->qmlSignals->append(s);
+    }
+
+    const QV4::CompiledData::Property *serializedProperty = serializedObject->propertyTable();
+    for (uint i = 0; i < serializedObject->nProperties; ++i, ++serializedProperty) {
+        QmlIR::Property *p = pool->New<QmlIR::Property>();
+        *static_cast<QV4::CompiledData::Property*>(p) = *serializedProperty;
+        object->properties->append(p);
+    }
+
+    QQmlJS::Engine *jsParserEngine = &output->jsParserEngine;
+
+    const QV4::CompiledData::LEUInt32 *functionIdx = serializedObject->functionOffsetTable();
+    for (uint i = 0; i < serializedObject->nFunctions; ++i, ++functionIdx) {
+        QmlIR::Function *f = pool->New<QmlIR::Function>();
+        const QV4::CompiledData::Function *compiledFunction = unit->functionAt(*functionIdx);
+
+        functionIndices.append(*functionIdx);
+        f->index = functionIndices.count() - 1;
+        f->location = compiledFunction->location;
+        f->nameIndex = compiledFunction->nameIndex;
+
+        QQmlJS::AST::FormalParameterList *paramList = 0;
+        const QV4::CompiledData::LEUInt32 *formalNameIdx = compiledFunction->formalsTable();
+        for (uint i = 0; i < compiledFunction->nFormals; ++i, ++formalNameIdx) {
+            const QString formal = unit->stringAt(*formalNameIdx);
+            QStringRef paramNameRef = jsParserEngine->newStringRef(formal);
+
+            if (paramList)
+                paramList = new (pool) QQmlJS::AST::FormalParameterList(paramList, paramNameRef);
+            else
+                paramList = new (pool) QQmlJS::AST::FormalParameterList(paramNameRef);
+        }
+
+        if (paramList)
+            paramList = paramList->finish();
+
+        const QString name = unit->stringAt(compiledFunction->nameIndex);
+        f->functionDeclaration = new(pool) QQmlJS::AST::FunctionDeclaration(jsParserEngine->newStringRef(name), paramList, /*body*/0);
+
+        object->functions->append(f);
+    }
+
+    object->runtimeFunctionIndices.allocate(pool, functionIndices);
+
+    return object;
+}
+
 #endif // V4_BOOTSTRAP
diff --git a/src/qml/compiler/qqmlirbuilder_p.h b/src/qml/compiler/qqmlirbuilder_p.h
index 95756845c3..2022112e07 100644
--- a/src/qml/compiler/qqmlirbuilder_p.h
+++ b/src/qml/compiler/qqmlirbuilder_p.h
@@ -622,6 +622,21 @@ private:
     int _importedScriptsTemp;
 };
 
+struct IRLoader {
+    IRLoader(const QV4::CompiledData::Unit *unit, QmlIR::Document *output);
+
+    void load();
+
+private:
+    QmlIR::Object *loadObject(const QV4::CompiledData::Object *serializedObject);
+
+    template <typename _Tp> _Tp *New() { return pool->New<_Tp>(); }
+
+    const QV4::CompiledData::Unit *unit;
+    QmlIR::Document *output;
+    QQmlJS::MemoryPool *pool;
+};
+
 } // namespace QmlIR
 
 QT_END_NAMESPACE
diff --git a/src/qml/compiler/qv4compileddata.cpp b/src/qml/compiler/qv4compileddata.cpp
index 8f5ea44599..8f8d374e24 100644
--- a/src/qml/compiler/qv4compileddata.cpp
+++ b/src/qml/compiler/qv4compileddata.cpp
@@ -476,7 +476,90 @@ bool CompilationUnit::saveCodeToDisk(QIODevice *device, const Unit *unit, QStrin
 
 Unit *CompilationUnit::createUnitData(QmlIR::Document *irDocument)
 {
-    return irDocument->jsGenerator.generateUnit(QV4::Compiler::JSUnitGenerator::GenerateWithoutStringTable);
+    if (!irDocument->javaScriptCompilationUnit->data)
+        return irDocument->jsGenerator.generateUnit(QV4::Compiler::JSUnitGenerator::GenerateWithoutStringTable);
+
+    QQmlRefPointer<QV4::CompiledData::CompilationUnit> compilationUnit = irDocument->javaScriptCompilationUnit;
+    QV4::CompiledData::Unit *jsUnit = const_cast<QV4::CompiledData::Unit*>(irDocument->javaScriptCompilationUnit->data);
+
+    QV4::Compiler::StringTableGenerator &stringTable = irDocument->jsGenerator.stringTable;
+
+    // Collect signals that have had a change in signature (from onClicked to onClicked(mouse) for example)
+    // and now need fixing in the QV4::CompiledData. Also register strings at the same time, to finalize
+    // the string table.
+    QVector<quint32> changedSignals;
+    QVector<QQmlJS::AST::FormalParameterList*> changedSignalParameters;
+    for (QmlIR::Object *o: qAsConst(irDocument->objects)) {
+        for (QmlIR::Binding *binding = o->firstBinding(); binding; binding = binding->next) {
+            if (!(binding->flags & QV4::CompiledData::Binding::IsSignalHandlerExpression))
+                continue;
+
+            quint32 functionIndex = binding->value.compiledScriptIndex;
+            QmlIR::CompiledFunctionOrExpression *foe = o->functionsAndExpressions->slowAt(functionIndex);
+            if (!foe)
+                continue;
+
+            // save absolute index
+            changedSignals << o->runtimeFunctionIndices.at(functionIndex);
+
+            Q_ASSERT(foe->node);
+            Q_ASSERT(QQmlJS::AST::cast<QQmlJS::AST::FunctionDeclaration*>(foe->node));
+
+            QQmlJS::AST::FormalParameterList *parameters = QQmlJS::AST::cast<QQmlJS::AST::FunctionDeclaration*>(foe->node)->formals;
+            changedSignalParameters << parameters;
+
+            for (; parameters; parameters = parameters->next)
+                stringTable.registerString(parameters->name.toString());
+        }
+    }
+
+    QVector<quint32> signalParameterNameTable;
+    quint32 signalParameterNameTableOffset = jsUnit->unitSize;
+
+    // Update signal signatures
+    if (!changedSignals.isEmpty()) {
+        if (jsUnit == compilationUnit->data) {
+            char *unitCopy = (char*)malloc(jsUnit->unitSize);
+            memcpy(unitCopy, jsUnit, jsUnit->unitSize);
+            jsUnit = reinterpret_cast<QV4::CompiledData::Unit*>(unitCopy);
+        }
+
+        for (int i = 0; i < changedSignals.count(); ++i) {
+            const uint functionIndex = changedSignals.at(i);
+            // The data is now read-write due to the copy above, so the const_cast is ok.
+            QV4::CompiledData::Function *function = const_cast<QV4::CompiledData::Function *>(jsUnit->functionAt(functionIndex));
+            Q_ASSERT(function->nFormals == quint32(0));
+
+            function->formalsOffset = signalParameterNameTableOffset - jsUnit->functionOffsetTable()[functionIndex];
+
+            for (QQmlJS::AST::FormalParameterList *parameters = changedSignalParameters.at(i);
+                 parameters; parameters = parameters->next) {
+                signalParameterNameTable.append(stringTable.getStringId(parameters->name.toString()));
+                function->nFormals = function->nFormals + 1;
+            }
+
+            // Hack to ensure an activation is created.
+            function->flags |= QV4::CompiledData::Function::HasCatchOrWith | QV4::CompiledData::Function::HasDirectEval;
+
+            signalParameterNameTableOffset += function->nFormals * sizeof(quint32);
+        }
+    }
+
+    if (!signalParameterNameTable.isEmpty()) {
+        Q_ASSERT(jsUnit != compilationUnit->data);
+        const uint signalParameterTableSize = signalParameterNameTable.count() * sizeof(quint32);
+        uint newSize = jsUnit->unitSize + signalParameterTableSize;
+        const uint oldSize = jsUnit->unitSize;
+        char *unitWithSignalParameters = (char*)realloc(jsUnit, newSize);
+        memcpy(unitWithSignalParameters + oldSize, signalParameterNameTable.constData(), signalParameterTableSize);
+        jsUnit = reinterpret_cast<QV4::CompiledData::Unit*>(unitWithSignalParameters);
+        jsUnit->unitSize = newSize;
+    }
+
+    if (jsUnit != compilationUnit->data)
+        jsUnit->flags &= ~QV4::CompiledData::Unit::StaticData;
+
+    return jsUnit;
 }
 
 QString Binding::valueAsString(const Unit *unit) const
diff --git a/src/qml/compiler/qv4compileddata_p.h b/src/qml/compiler/qv4compileddata_p.h
index f5acfda631..13a0c4b075 100644
--- a/src/qml/compiler/qv4compileddata_p.h
+++ b/src/qml/compiler/qv4compileddata_p.h
@@ -71,7 +71,7 @@
 QT_BEGIN_NAMESPACE
 
 // Bump this whenever the compiler data structures change in an incompatible way.
-#define QV4_DATA_STRUCTURE_VERSION 0x08
+#define QV4_DATA_STRUCTURE_VERSION 0x09
 
 class QIODevice;
 class QQmlPropertyCache;
@@ -625,7 +625,8 @@ struct Unit
         StaticData = 0x4, // Unit data persistent in memory?
         IsSingleton = 0x8,
         IsSharedLibrary = 0x10, // .pragma shared?
-        ContainsMachineCode = 0x20 // used to determine if we need to mmap with execute permissions
+        ContainsMachineCode = 0x20, // used to determine if we need to mmap with execute permissions
+        PendingTypeCompilation = 0x40 // the QML data structures present are incomplete and require type compilation
     };
     LEUInt32 flags;
     LEUInt32 stringTableSize;
diff --git a/src/qml/qml/qqmltypeloader.cpp b/src/qml/qml/qqmltypeloader.cpp
index 8dea866c9a..f4f04e12c0 100644
--- a/src/qml/qml/qqmltypeloader.cpp
+++ b/src/qml/qml/qqmltypeloader.cpp
@@ -2075,6 +2075,11 @@ bool QQmlTypeData::tryLoadFromDiskCache()
         }
     }
 
+    if (unit->data->flags & QV4::CompiledData::Unit::PendingTypeCompilation) {
+        restoreIR(unit);
+        return true;
+    }
+
     m_compiledData = unit;
 
     for (int i = 0, count = m_compiledData->objectCount(); i < count; ++i)
@@ -2397,6 +2402,15 @@ bool QQmlTypeData::loadFromSource()
     return true;
 }
 
+void QQmlTypeData::restoreIR(QQmlRefPointer<QV4::CompiledData::CompilationUnit> unit)
+{
+    m_document.reset(new QmlIR::Document(isDebugging()));
+    QmlIR::IRLoader loader(unit->data, m_document.data());
+    loader.load();
+    m_document->javaScriptCompilationUnit = unit;
+    continueLoadFromIR();
+}
+
 void QQmlTypeData::continueLoadFromIR()
 {
     m_typeReferences.collectFromObjects(m_document->objects.constBegin(), m_document->objects.constEnd());
diff --git a/src/qml/qml/qqmltypeloader_p.h b/src/qml/qml/qqmltypeloader_p.h
index 789330297c..915b1bcc4c 100644
--- a/src/qml/qml/qqmltypeloader_p.h
+++ b/src/qml/qml/qqmltypeloader_p.h
@@ -446,6 +446,7 @@ protected:
 private:
     bool tryLoadFromDiskCache();
     bool loadFromSource();
+    void restoreIR(QQmlRefPointer<QV4::CompiledData::CompilationUnit> unit);
     void continueLoadFromIR();
     void resolveTypes();
     QQmlCompileError buildTypeResolutionCaches(
diff --git a/tools/qmlcachegen/qmlcachegen.cpp b/tools/qmlcachegen/qmlcachegen.cpp
index 84ad1c7778..21ff882c2f 100644
--- a/tools/qmlcachegen/qmlcachegen.cpp
+++ b/tools/qmlcachegen/qmlcachegen.cpp
@@ -144,6 +144,7 @@ static bool compileQmlFile(const QString &inputFileName, QV4::EvalISelFactory *i
         QV4::CompiledData::ResolvedTypeReferenceMap dummyDependencies;
         QV4::CompiledData::Unit *unit = generator.generate(irDocument, /*engine*/nullptr, dummyDependencies);
         unit->flags |= QV4::CompiledData::Unit::StaticData;
+        unit->flags |= QV4::CompiledData::Unit::PendingTypeCompilation;
         irDocument.javaScriptCompilationUnit->data = unit;
 
         if (!irDocument.javaScriptCompilationUnit->saveToDisk(inputFileName, &error->message))