Add support for disk caching of ES modules

Two minor fixes needed for this otherwise straight-forward change:

    (1) When compiling modules, use the full url for the source file of
    the compilation unit, as that's what we use for the relocation check
    when loading the cache file.

    (2) Record the proper source time stamp for cache invalidation.

As a bonus, when importing scripts from .qml files, we now also attempt
to use the cached version that we created on the fly in an effort to
replace heap memory with mmap backed memory - just like we do for .qml
files.

Change-Id: I5b03a18e3c44d537c3242cb1d969636df32fe42a
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

9 files changed, 79 insertions, 14 deletions

diff --git a/src/qml/jsruntime/qv4engine.cpp b/src/qml/jsruntime/qv4engine.cpp
index 14bc5f084e..c5eec33d18 100644
--- a/src/qml/jsruntime/qv4engine.cpp
+++ b/src/qml/jsruntime/qv4engine.cpp
@@ -1633,17 +1633,19 @@ QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(con
         return nullptr;
     }
 
+    const QDateTime timeStamp = QFileInfo(f).lastModified();
+
     const QString sourceCode = QString::fromUtf8(f.readAll());
     f.close();
 
-    return compileModule(url, sourceCode);
+    return compileModule(url, sourceCode, timeStamp);
 }
 
 
-QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(const QUrl &url, const QString &sourceCode)
+QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(const QUrl &url, const QString &sourceCode, const QDateTime &sourceTimeStamp)
 {
     QList<QQmlJS::DiagnosticMessage> diagnostics;
-    auto unit = compileModule(/*debugMode*/debugger() != nullptr, url, sourceCode, &diagnostics);
+    auto unit = compileModule(/*debugMode*/debugger() != nullptr, url, sourceCode, sourceTimeStamp, &diagnostics);
     for (const QQmlJS::DiagnosticMessage &m : diagnostics) {
         if (m.isError()) {
             throwSyntaxError(m.message, url.toString(), m.loc.startLine, m.loc.startColumn);
@@ -1656,7 +1658,8 @@ QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(con
     return unit;
 }
 
-QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(bool debugMode, const QUrl &url, const QString &sourceCode, QList<QQmlJS::DiagnosticMessage> *diagnostics)
+QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(bool debugMode, const QUrl &url, const QString &sourceCode,
+                                                                             const QDateTime &sourceTimeStamp, QList<QQmlJS::DiagnosticMessage> *diagnostics)
 {
     QQmlJS::Engine ee;
     QQmlJS::Lexer lexer(&ee);
@@ -1683,9 +1686,10 @@ QQmlRefPointer<CompiledData::CompilationUnit> ExecutionEngine::compileModule(boo
     using namespace QV4::Compiler;
     Compiler::Module compilerModule(debugMode);
     compilerModule.unitFlags |= CompiledData::Unit::IsESModule;
+    compilerModule.sourceTimeStamp = sourceTimeStamp;
     JSUnitGenerator jsGenerator(&compilerModule);
     Codegen cg(&jsGenerator, /*strictMode*/true);
-    cg.generateFromModule(url.fileName(), url.toString(), sourceCode, moduleNode, &compilerModule);
+    cg.generateFromModule(url.toString(), url.toString(), sourceCode, moduleNode, &compilerModule);
     auto errors = cg.errors();
     if (diagnostics)
         *diagnostics << errors;
diff --git a/src/qml/jsruntime/qv4engine_p.h b/src/qml/jsruntime/qv4engine_p.h
index 8312adee48..c8f8efaebf 100644
--- a/src/qml/jsruntime/qv4engine_p.h
+++ b/src/qml/jsruntime/qv4engine_p.h
@@ -577,8 +577,8 @@ public:
 
 #ifndef V4_BOOTSTRAP
     QQmlRefPointer<CompiledData::CompilationUnit> compileModule(const QUrl &url);
-    QQmlRefPointer<CompiledData::CompilationUnit> compileModule(const QUrl &url, const QString &sourceCode);
-    static QQmlRefPointer<CompiledData::CompilationUnit> compileModule(bool debugMode, const QUrl &url, const QString &sourceCode, QList<QQmlJS::DiagnosticMessage> *diagnostics);
+    QQmlRefPointer<CompiledData::CompilationUnit> compileModule(const QUrl &url, const QString &sourceCode, const QDateTime &sourceTimeStamp);
+    static QQmlRefPointer<CompiledData::CompilationUnit> compileModule(bool debugMode, const QUrl &url, const QString &sourceCode, const QDateTime &sourceTimeStamp, QList<QQmlJS::DiagnosticMessage> *diagnostics);
 
     mutable QMutex moduleMutex;
     QHash<QUrl, QQmlRefPointer<CompiledData::CompilationUnit>> modules;
diff --git a/src/qml/qml/qqmltypeloader.cpp b/src/qml/qml/qqmltypeloader.cpp
index daa4604dca..209a32b8cc 100644
--- a/src/qml/qml/qqmltypeloader.cpp
+++ b/src/qml/qml/qqmltypeloader.cpp
@@ -3003,7 +3003,7 @@ void QQmlScriptBlob::dataReceived(const SourceCodeData &data)
 
     if (m_isModule) {
         QList<QQmlJS::DiagnosticMessage> diagnostics;
-        unit = QV4::ExecutionEngine::compileModule(isDebugging(), url(), source, &diagnostics);
+        unit = QV4::ExecutionEngine::compileModule(isDebugging(), url(), source, data.sourceTimeStamp(), &diagnostics);
         QList<QQmlError> errors = QQmlEnginePrivate::qmlErrorFromDiagnostics(urlString(), diagnostics);
         if (!errors.isEmpty()) {
             setError(errors);
@@ -3034,12 +3034,17 @@ void QQmlScriptBlob::dataReceived(const SourceCodeData &data)
 
         QmlIR::QmlUnitGenerator qmlGenerator;
         qmlGenerator.generate(irUnit);
+    }
 
-        if ((!disableDiskCache() || forceDiskCache()) && !isDebugging()) {
-            QString errorString;
-            if (!unit->saveToDisk(url(), &errorString)) {
-                qCDebug(DBG_DISK_CACHE()) << "Error saving cached version of" << unit->fileName() << "to disk:" << errorString;
+    if ((!disableDiskCache() || forceDiskCache()) && !isDebugging()) {
+        QString errorString;
+        if (unit->saveToDisk(url(), &errorString)) {
+            QString error;
+            if (!unit->loadFromDisk(url(), data.sourceTimeStamp(), &error)) {
+                // ignore error, keep using the in-memory compilation unit.
             }
+        } else {
+            qCDebug(DBG_DISK_CACHE()) << "Error saving cached version of" << unit->fileName() << "to disk:" << errorString;
         }
     }
 
diff --git a/src/qml/qml/qqmltypeloader_p.h b/src/qml/qml/qqmltypeloader_p.h
index 26090c6af7..5d85773be3 100644
--- a/src/qml/qml/qqmltypeloader_p.h
+++ b/src/qml/qml/qqmltypeloader_p.h
@@ -542,6 +542,8 @@ public:
 
     QV4::ReturnedValue scriptValueForContext(QQmlContextData *parentCtxt);
 
+    QQmlRefPointer<QV4::CompiledData::CompilationUnit> compilationUnit() const { return m_precompiledScript; }
+
 protected:
     void clear() override; // From QQmlCleanup
 
diff --git a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
index a48b70668d..98a99e6673 100644
--- a/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
+++ b/tests/auto/qml/ecmascripttests/qjstest/test262runner.cpp
@@ -462,7 +462,7 @@ static bool executeTest(const QByteArray &data, bool runAsModule = false, const
             QFile f(url.toLocalFile());
             if (f.open(QIODevice::ReadOnly)) {
                 QByteArray content = harnessForModules + f.readAll();
-                module = vm.compileModule(url, QString::fromUtf8(content));
+                module = vm.compileModule(url, QString::fromUtf8(content), QFileInfo(f).lastModified());
                 if (vm.hasException)
                     break;
                 vm.injectModule(module);
diff --git a/tests/auto/qml/qmldiskcache/importmodule.qml b/tests/auto/qml/qmldiskcache/importmodule.qml
new file mode 100644
index 0000000000..f890d4cb5c
--- /dev/null
+++ b/tests/auto/qml/qmldiskcache/importmodule.qml
@@ -0,0 +1,5 @@
+import QtQml 2.0
+import "module.mjs" as Module
+QtObject {
+    property bool ok: Module.ok()
+}
diff --git a/tests/auto/qml/qmldiskcache/module.mjs b/tests/auto/qml/qmldiskcache/module.mjs
new file mode 100644
index 0000000000..32e0651b8b
--- /dev/null
+++ b/tests/auto/qml/qmldiskcache/module.mjs
@@ -0,0 +1,2 @@
+
+export function ok() { return true; }
diff --git a/tests/auto/qml/qmldiskcache/qmldiskcache.pro b/tests/auto/qml/qmldiskcache/qmldiskcache.pro
index f98a157b6a..74aefa6944 100644
--- a/tests/auto/qml/qmldiskcache/qmldiskcache.pro
+++ b/tests/auto/qml/qmldiskcache/qmldiskcache.pro
@@ -4,6 +4,6 @@ osx:CONFIG -= app_bundle
 
 SOURCES += tst_qmldiskcache.cpp
 
-RESOURCES += test.qml
+RESOURCES += test.qml importmodule.qml module.mjs
 
 QT += core-private qml-private testlib
diff --git a/tests/auto/qml/qmldiskcache/tst_qmldiskcache.cpp b/tests/auto/qml/qmldiskcache/tst_qmldiskcache.cpp
index 84d14c82ab..5c20e389f6 100644
--- a/tests/auto/qml/qmldiskcache/tst_qmldiskcache.cpp
+++ b/tests/auto/qml/qmldiskcache/tst_qmldiskcache.cpp
@@ -33,6 +33,7 @@
 #include <private/qv8engine_p.h>
 #include <private/qv4engine_p.h>
 #include <private/qv4codegen_p.h>
+#include <private/qqmlcomponent_p.h>
 #include <QQmlComponent>
 #include <QQmlEngine>
 #include <QQmlFileSelector>
@@ -60,6 +61,7 @@ private slots:
     void stableOrderOfDependentCompositeTypes();
     void singletonDependency();
     void cppRegisteredSingletonDependency();
+    void cacheModuleScripts();
 };
 
 // A wrapper around QQmlComponent to ensure the temporary reference counts
@@ -886,6 +888,51 @@ void tst_qmldiskcache::cppRegisteredSingletonDependency()
     }
 }
 
+void tst_qmldiskcache::cacheModuleScripts()
+{
+    const QString cacheDirectory = QStandardPaths::writableLocation(QStandardPaths::CacheLocation);
+    QVERIFY(QDir::root().mkpath(cacheDirectory));
+
+    const QString qmlCacheDirectory = cacheDirectory + QLatin1String("/qmlcache/");
+    QVERIFY(QDir(qmlCacheDirectory).removeRecursively());
+    QVERIFY(QDir::root().mkpath(qmlCacheDirectory));
+    QVERIFY(QDir(qmlCacheDirectory).entryList(QDir::NoDotAndDotDot).isEmpty());
+
+
+    QQmlEngine engine;
+
+    {
+        CleanlyLoadingComponent component(&engine, QUrl("qrc:/importmodule.qml"));
+        QScopedPointer<QObject> obj(component.create());
+        QVERIFY(!obj.isNull());
+        QVERIFY(obj->property("ok").toBool());
+
+        auto componentPrivate = QQmlComponentPrivate::get(&component);
+        QVERIFY(componentPrivate);
+        auto compilationUnit = componentPrivate->compilationUnit->dependentScripts.first()->compilationUnit();
+        QVERIFY(compilationUnit);
+        auto unitData = compilationUnit->unitData();
+        QVERIFY(unitData);
+        QVERIFY(unitData->flags & QV4::CompiledData::Unit::StaticData);
+        QVERIFY(unitData->flags & QV4::CompiledData::Unit::IsESModule);
+        QVERIFY(!compilationUnit->backingFile.isNull());
+    }
+
+    const QStringList entries = QDir(qmlCacheDirectory).entryList(QStringList("*.mjsc"), QDir::NoDotAndDotDot | QDir::Files);
+    QCOMPARE(entries.count(), 1);
+
+    QDateTime cacheFileTimeStamp;
+
+    {
+        QFile cacheFile(qmlCacheDirectory + QLatin1Char('/') + entries.constFirst());
+        QVERIFY2(cacheFile.open(QIODevice::ReadOnly), qPrintable(cacheFile.errorString()));
+        QV4::CompiledData::Unit unit;
+        QVERIFY(cacheFile.read(reinterpret_cast<char *>(&unit), sizeof(unit)) == sizeof(unit));
+
+        QVERIFY(unit.flags & QV4::CompiledData::Unit::IsESModule);
+    }
+}
+
 QTEST_MAIN(tst_qmldiskcache)
 
 #include "tst_qmldiskcache.moc"