Add support for compiling ES modules ahead of time

This is also pretty straight-forward by adding .mjs as supported
extension in the qmake and cmake support.

This also tweaks qv4engine.cpp to share the same module compilation
function across all code paths.

Change-Id: Ia0e23c78a794f2330ecf8f991ee6ea948f4ac89d
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

5 files changed, 86 insertions, 77 deletions

diff --git a/tools/qmlcachegen/Qt5QuickCompilerConfig.cmake.in b/tools/qmlcachegen/Qt5QuickCompilerConfig.cmake.in
index e4963c3a33..a5c531fb0d 100644
--- a/tools/qmlcachegen/Qt5QuickCompilerConfig.cmake.in
+++ b/tools/qmlcachegen/Qt5QuickCompilerConfig.cmake.in
@@ -4,6 +4,7 @@ function(QTQUICK_COMPILER_DETERMINE_OUTPUT_FILENAME outvariable filename)
     file(RELATIVE_PATH relpath ${CMAKE_CURRENT_SOURCE_DIR} ${filename})
     string(REPLACE \".qml\" \"_qml\" relpath ${relpath})
     string(REPLACE \".js\" \"_js\" relpath ${relpath})
+    string(REPLACE \".mjs\" \"_mjs\" relpath ${relpath})
     string(REPLACE \"/\" \"_\" relpath ${relpath})
     set(${outvariable} ${CMAKE_CURRENT_BINARY_DIR}/${relpath}.cpp PARENT_SCOPE)
 endfunction()
@@ -63,7 +64,7 @@ but not all the files it references.
         string(REGEX REPLACE \"[\r\n]+\" \";\" rcc_contents ${rcc_contents})
         foreach(it ${rcc_contents})
             get_filename_component(extension ${it} EXT)
-            if(extension STREQUAL \".qml\" OR extension STREQUAL \".js\" OR extension STREQUAL \".ui.qml\")
+            if(extension STREQUAL \".qml\" OR extension STREQUAL \".js\" OR extension STREQUAL \".ui.qml\" OR extension STREQUAL \".mjs\")
                 qtquick_compiler_determine_output_filename(output_file ${it})
                 add_custom_command(OUTPUT ${output_file} COMMAND ${compiler_path} ARGS --resource=${input_resource} ${it} -o ${output_file} DEPENDS ${it})
                 list(APPEND compiler_output ${output_file})
diff --git a/tools/qmlcachegen/qmlcachegen.cpp b/tools/qmlcachegen/qmlcachegen.cpp
index 4dddab18b2..762f6e7221 100644
--- a/tools/qmlcachegen/qmlcachegen.cpp
+++ b/tools/qmlcachegen/qmlcachegen.cpp
@@ -34,6 +34,8 @@
 #include <QDateTime>
 #include <QHashFunctions>
 #include <QSaveFile>
+#include <QScopedPointer>
+#include <QScopeGuard>
 
 #include <private/qqmlirbuilder_p.h>
 #include <private/qqmljsparser_p.h>
@@ -67,6 +69,7 @@ struct Error
     QString message;
     void print();
     Error augment(const QString &contextErrorMessage) const;
+    void appendDiagnostics(const QString &inputFileName, const QList<QQmlJS::DiagnosticMessage> &diagnostics);
 };
 
 void Error::print()
@@ -96,6 +99,15 @@ QString diagnosticErrorMessage(const QString &fileName, const QQmlJS::Diagnostic
     return message;
 }
 
+void Error::appendDiagnostics(const QString &inputFileName, const QList<DiagnosticMessage> &diagnostics)
+{
+    for (const QQmlJS::DiagnosticMessage &parseError: diagnostics) {
+        if (!message.isEmpty())
+            message += QLatin1Char('\n');
+        message += diagnosticErrorMessage(inputFileName, parseError);
+    }
+}
+
 // Ensure that ListElement objects keep all property assignments in their string form
 static void annotateListElements(QmlIR::Document *document)
 {
@@ -185,11 +197,7 @@ static bool compileQmlFile(const QString &inputFileName, SaveFunction saveFuncti
     {
         QmlIR::IRBuilder irBuilder(illegalNames);
         if (!irBuilder.generateFromQml(sourceCode, inputFileName, &irDocument)) {
-            for (const QQmlJS::DiagnosticMessage &parseError: qAsConst(irBuilder.errors)) {
-                if (!error->message.isEmpty())
-                    error->message += QLatin1Char('\n');
-                error->message += diagnosticErrorMessage(inputFileName, parseError);
-            }
+            error->appendDiagnostics(inputFileName, irBuilder.errors);
             return false;
         }
     }
@@ -213,11 +221,7 @@ static bool compileQmlFile(const QString &inputFileName, SaveFunction saveFuncti
             const QVector<int> runtimeFunctionIndices = v4CodeGen.generateJSCodeForFunctionsAndBindings(functionsToCompile);
             QList<QQmlJS::DiagnosticMessage> jsErrors = v4CodeGen.errors();
             if (!jsErrors.isEmpty()) {
-                for (const QQmlJS::DiagnosticMessage &e: qAsConst(jsErrors)) {
-                    if (!error->message.isEmpty())
-                        error->message += QLatin1Char('\n');
-                    error->message += diagnosticErrorMessage(inputFileName, e);
-                }
+                error->appendDiagnostics(inputFileName, jsErrors);
                 return false;
             }
 
@@ -247,7 +251,8 @@ static bool compileQmlFile(const QString &inputFileName, SaveFunction saveFuncti
 
 static bool compileJSFile(const QString &inputFileName, const QString &inputFileUrl, SaveFunction saveFunction, Error *error)
 {
-    QmlIR::Document irDocument(/*debugMode*/false);
+    QQmlRefPointer<QV4::CompiledData::CompilationUnit> unit;
+    QScopedPointer<QV4::CompiledData::Unit, QScopedPointerPodDeleter> unitDataToFree;
 
     QString sourceCode;
     {
@@ -263,79 +268,79 @@ static bool compileJSFile(const QString &inputFileName, const QString &inputFile
         }
     }
 
-    QQmlJS::Engine *engine = &irDocument.jsParserEngine;
-    QmlIR::ScriptDirectivesCollector directivesCollector(&irDocument);
-    QQmlJS::Directives *oldDirs = engine->directives();
-    engine->setDirectives(&directivesCollector);
+    const bool isModule = inputFileName.endsWith(QLatin1String(".mjs"));
+    if (isModule) {
+        QList<QQmlJS::DiagnosticMessage> diagnostics;
+        // Precompiled files are relocatable and the final location will be set when loading.
+        QString url;
+        unit = QV4::ExecutionEngine::compileModule(/*debugMode*/false, url, sourceCode, QDateTime(), &diagnostics);
+        error->appendDiagnostics(inputFileName, diagnostics);
+        if (!unit)
+            return false;
+    } else {
+        QmlIR::Document irDocument(/*debugMode*/false);
 
-    QQmlJS::AST::Program *program = nullptr;
+        QQmlJS::Engine *engine = &irDocument.jsParserEngine;
+        QmlIR::ScriptDirectivesCollector directivesCollector(&irDocument);
+        QQmlJS::Directives *oldDirs = engine->directives();
+        engine->setDirectives(&directivesCollector);
+        auto directivesGuard = qScopeGuard([engine, oldDirs]{
+            engine->setDirectives(oldDirs);
+        });
 
-    {
-        QQmlJS::Lexer lexer(engine);
-         lexer.setCode(sourceCode, /*line*/1, /*parseAsBinding*/false);
-         QQmlJS::Parser parser(engine);
+        QQmlJS::AST::Program *program = nullptr;
 
-         bool parsed = parser.parseProgram();
+        {
+            QQmlJS::Lexer lexer(engine);
+            lexer.setCode(sourceCode, /*line*/1, /*parseAsBinding*/false);
+            QQmlJS::Parser parser(engine);
 
-         for (const QQmlJS::DiagnosticMessage &parseError: parser.diagnosticMessages()) {
-             if (!error->message.isEmpty())
-                 error->message += QLatin1Char('\n');
-             error->message += diagnosticErrorMessage(inputFileName, parseError);
-         }
+            bool parsed = parser.parseProgram();
 
-         if (!parsed) {
-             engine->setDirectives(oldDirs);
-             return false;
-         }
+            error->appendDiagnostics(inputFileName, parser.diagnosticMessages());
 
-         program = QQmlJS::AST::cast<QQmlJS::AST::Program*>(parser.rootNode());
-         if (!program) {
-             lexer.setCode(QStringLiteral("undefined;"), 1, false);
-             parsed = parser.parseProgram();
-             Q_ASSERT(parsed);
-             program = QQmlJS::AST::cast<QQmlJS::AST::Program*>(parser.rootNode());
-             Q_ASSERT(program);
-         }
-    }
+            if (!parsed)
+                return false;
 
-    {
-        QmlIR::JSCodeGen v4CodeGen(irDocument.code, &irDocument.jsGenerator,
-                                   &irDocument.jsModule, &irDocument.jsParserEngine,
-                                   irDocument.program, /*import cache*/nullptr,
-                                   &irDocument.jsGenerator.stringTable, illegalNames);
-        v4CodeGen.setUseFastLookups(false); // Disable lookups in non-standalone (aka QML) mode
-        v4CodeGen.generateFromProgram(inputFileName, inputFileUrl, sourceCode, program,
-                                      &irDocument.jsModule, QV4::Compiler::ContextType::Global);
-        QList<QQmlJS::DiagnosticMessage> jsErrors = v4CodeGen.errors();
-        if (!jsErrors.isEmpty()) {
-            for (const QQmlJS::DiagnosticMessage &e: qAsConst(jsErrors)) {
-                if (!error->message.isEmpty())
-                    error->message += QLatin1Char('\n');
-                error->message += diagnosticErrorMessage(inputFileName, e);
+            program = QQmlJS::AST::cast<QQmlJS::AST::Program*>(parser.rootNode());
+            if (!program) {
+                lexer.setCode(QStringLiteral("undefined;"), 1, false);
+                parsed = parser.parseProgram();
+                Q_ASSERT(parsed);
+                program = QQmlJS::AST::cast<QQmlJS::AST::Program*>(parser.rootNode());
+                Q_ASSERT(program);
             }
-            engine->setDirectives(oldDirs);
-            return false;
         }
 
-        // Precompiled files are relocatable and the final location will be set when loading.
-        irDocument.jsModule.fileName.clear();
-        irDocument.jsModule.finalUrl.clear();
-
-        irDocument.javaScriptCompilationUnit = v4CodeGen.generateCompilationUnit(/*generate unit*/false);
-        QmlIR::QmlUnitGenerator generator;
-        generator.generate(irDocument);
-        QV4::CompiledData::Unit *unit = const_cast<QV4::CompiledData::Unit*>(irDocument.javaScriptCompilationUnit->data);
-        unit->flags |= QV4::CompiledData::Unit::StaticData;
+        {
+            QmlIR::JSCodeGen v4CodeGen(irDocument.code, &irDocument.jsGenerator,
+                                       &irDocument.jsModule, &irDocument.jsParserEngine,
+                                       irDocument.program, /*import cache*/nullptr,
+                                       &irDocument.jsGenerator.stringTable, illegalNames);
+            v4CodeGen.setUseFastLookups(false); // Disable lookups in non-standalone (aka QML) mode
+            v4CodeGen.generateFromProgram(inputFileName, inputFileUrl, sourceCode, program,
+                                          &irDocument.jsModule, QV4::Compiler::ContextType::Global);
+            QList<QQmlJS::DiagnosticMessage> jsErrors = v4CodeGen.errors();
+            if (!jsErrors.isEmpty()) {
+                error->appendDiagnostics(inputFileName, jsErrors);
+                return false;
+            }
 
-        if (!saveFunction(irDocument.javaScriptCompilationUnit, &error->message)) {
-            engine->setDirectives(oldDirs);
-            return false;
+            // Precompiled files are relocatable and the final location will be set when loading.
+            irDocument.jsModule.fileName.clear();
+            irDocument.jsModule.finalUrl.clear();
+
+            irDocument.javaScriptCompilationUnit = v4CodeGen.generateCompilationUnit(/*generate unit*/false);
+            QmlIR::QmlUnitGenerator generator;
+            generator.generate(irDocument);
+            QV4::CompiledData::Unit *unitData = const_cast<QV4::CompiledData::Unit*>(irDocument.javaScriptCompilationUnit->data);
+            unitData->flags |= QV4::CompiledData::Unit::StaticData;
+            unitDataToFree.reset(unitData);
+            unit = irDocument.javaScriptCompilationUnit;
         }
-
-        free(unit);
     }
-    engine->setDirectives(oldDirs);
-    return true;
+
+    return saveFunction(unit, &error->message);
 }
 
 static bool saveUnitAsCpp(const QString &inputFileName, const QString &outputFileName,
@@ -546,7 +551,7 @@ int main(int argc, char **argv)
             error.augment(QLatin1String("Error compiling qml file: ")).print();
             return EXIT_FAILURE;
         }
-    } else if (inputFile.endsWith(QLatin1String(".js"))) {
+    } else if (inputFile.endsWith(QLatin1String(".js")) || inputFile.endsWith(QLatin1String(".mjs"))) {
         Error error;
         if (!compileJSFile(inputFile, inputFileUrl, saveFunction, &error)) {
             error.augment(QLatin1String("Error compiling js file: ")).print();
diff --git a/tools/qmlcachegen/qtquickcompiler.prf b/tools/qmlcachegen/qtquickcompiler.prf
index d05908560d..7d8a857847 100644
--- a/tools/qmlcachegen/qtquickcompiler.prf
+++ b/tools/qmlcachegen/qtquickcompiler.prf
@@ -35,7 +35,7 @@ for(res, RESOURCES) {
 
     absRes = $$absolute_path($$res, $$_PRO_FILE_PWD_)
     rccContents = $$system($$QMAKE_RCC_DEP -list $$system_quote($$absRes),lines)
-    contains(rccContents,.*\\.js$)|contains(rccContents,.*\\.qml$) {
+    contains(rccContents,.*\\.js$)|contains(rccContents,.*\\.qml$)|contains(rccContents,.*\\.mjs$) {
         new_resource = $$qmlCacheResourceFileOutputName($$res)
         mkpath($$dirname(new_resource))
         remaining_files = $$system($$QML_CACHEGEN_FILTER -filter-resource-file -o $$system_quote($$new_resource) $$system_quote($$absRes),lines)
@@ -49,7 +49,7 @@ for(res, RESOURCES) {
         QMLCACHE_RESOURCE_FILES += $$absRes
 
         for(candidate, rccContents) {
-            contains(candidate,.*\\.js$)|contains(candidate,.*\\.qml$) {
+            contains(candidate,.*\\.js$)|contains(candidate,.*\\.qml$)|contains(candidate,.*\\.mjs$) {
                 QMLCACHE_FILES += $$candidate
             }
         }
@@ -70,6 +70,7 @@ defineReplace(qmlCacheOutputName) {
     name = $$relative_path($$name, $$_PRO_FILE_PWD_)
     name = $$replace(name, \\.qml$, _qml)
     name = $$replace(name, \\.js$, _js)
+    name = $$replace(name, \\.mjs$, _mjs)
     name = $$replace(name,/,_)
     name = $$QMLCACHE_DIR/$${name}
     return($${name})
diff --git a/tools/qmlcachegen/resourcefilemapper.cpp b/tools/qmlcachegen/resourcefilemapper.cpp
index c2fd057541..6a00b39f2e 100644
--- a/tools/qmlcachegen/resourcefilemapper.cpp
+++ b/tools/qmlcachegen/resourcefilemapper.cpp
@@ -67,7 +67,7 @@ QStringList ResourceFileMapper::qmlCompilerFiles() const
          it != end; ++it) {
         const QString &qrcPath = it.key();
         const QString suffix = QFileInfo(qrcPath).suffix();
-        if (suffix != QStringLiteral("qml") && suffix != QStringLiteral("js"))
+        if (suffix != QStringLiteral("qml") && suffix != QStringLiteral("js") && suffix != QStringLiteral("mjs"))
             continue;
         files << qrcPath;
     }
diff --git a/tools/qmlcachegen/resourcefilter.cpp b/tools/qmlcachegen/resourcefilter.cpp
index 196dbd4a75..3ad6e9ca0d 100644
--- a/tools/qmlcachegen/resourcefilter.cpp
+++ b/tools/qmlcachegen/resourcefilter.cpp
@@ -139,7 +139,9 @@ int filterResourceFile(const QString &input, const QString &output)
             if (currentFileName.isEmpty())
                 continue;
 
-            if (!currentFileName.endsWith(QStringLiteral(".qml")) && !currentFileName.endsWith(QStringLiteral(".js"))) {
+            if (!currentFileName.endsWith(QStringLiteral(".qml"))
+                && !currentFileName.endsWith(QStringLiteral(".js"))
+                && !currentFileName.endsWith(QStringLiteral(".mjs"))) {
                 writer.writeStartElement(QStringLiteral("file"));
 
                 if (!fileAttributes.hasAttribute(QStringLiteral("alias")))