Add support for compiling QML/JS files ahead of time in resources

This is bringing over the loading infrastructure from the Qt Quick
Compiler that allows embedding qml/js files in resources and compiling
them ahead of time.

At the moment, the approach of generating one cpp file per qml/js file
and the loader stub is needed because the build system does not support
dynamic resource generation. In addition, as per QTBUG-60961, we must
ensure that the generated data structures are aligned.

To retain compatibility this is enabled via CONFIG += qtquickcompiler,
but we may need to find a new name (but should keep the old one in any
case).

Task-number: QTBUG-60961
Change-Id: Ia9839bf98d3af4c50636b6e06815364a9fc7ee57
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

7 files changed, 1074 insertions, 10 deletions

diff --git a/tools/qmlcachegen/generateloader.cpp b/tools/qmlcachegen/generateloader.cpp
new file mode 100644
index 0000000000..dd2e244b56
--- /dev/null
+++ b/tools/qmlcachegen/generateloader.cpp
@@ -0,0 +1,398 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtQml module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#include <QByteArray>
+#include <QString>
+#include <QStringList>
+#include <QTextStream>
+#include <QVector>
+#include <QtEndian>
+#include <QStack>
+#include <QFileInfo>
+#include <QSaveFile>
+
+QString symbolNamespaceForPath(const QString &relativePath)
+{
+    QFileInfo fi(relativePath);
+    QString symbol = fi.path();
+    if (symbol.length() == 1 && symbol.startsWith(QLatin1Char('.'))) {
+        symbol.clear();
+    } else {
+        symbol.replace(QLatin1Char('/'), QLatin1Char('_'));
+        symbol += QLatin1Char('_');
+    }
+    symbol += fi.baseName();
+    symbol += QLatin1Char('_');
+    symbol += fi.suffix();
+    symbol.replace(QLatin1Char('.'), QLatin1Char('_'));
+    symbol.replace(QLatin1Char('+'), QLatin1Char('_'));
+    symbol.replace(QLatin1Char('-'), QLatin1Char('_'));
+    return symbol;
+}
+
+struct VirtualDirectoryEntry
+{
+    QString name;
+    QVector<VirtualDirectoryEntry*> dirEntries;
+    int firstChildIndex; // node index inside generated data
+    bool isDirectory;
+
+    VirtualDirectoryEntry()
+        : firstChildIndex(-1)
+        , isDirectory(true)
+    {}
+
+    ~VirtualDirectoryEntry()
+    {
+        qDeleteAll(dirEntries);
+    }
+
+    VirtualDirectoryEntry *append(const QString &name)
+    {
+        for (QVector<VirtualDirectoryEntry*>::Iterator it = dirEntries.begin(), end = dirEntries.end();
+             it != end; ++it) {
+            if ((*it)->name == name)
+                return *it;
+        }
+
+        VirtualDirectoryEntry *subEntry = new VirtualDirectoryEntry;
+        subEntry->name = name;
+        dirEntries.append(subEntry);
+        return subEntry;
+    }
+
+    void appendEmptyFile(const QString &name)
+    {
+        VirtualDirectoryEntry *subEntry = new VirtualDirectoryEntry;
+        subEntry->name = name;
+        subEntry->isDirectory = false;
+        dirEntries.append(subEntry);
+    }
+
+    bool isEmpty() const { return dirEntries.isEmpty(); }
+};
+
+struct DataStream
+{
+    DataStream(QVector<unsigned char > *data = 0)
+        : data(data)
+    {}
+
+    qint64 currentOffset() const { return data->size(); }
+
+    DataStream &operator<<(quint16 value)
+    {
+        unsigned char d[2];
+        qToBigEndian(value, d);
+        data->append(d[0]);
+        data->append(d[1]);
+        return *this;
+    }
+    DataStream &operator<<(quint32 value)
+    {
+        unsigned char d[4];
+        qToBigEndian(value, d);
+        data->append(d[0]);
+        data->append(d[1]);
+        data->append(d[2]);
+        data->append(d[3]);
+        return *this;
+    }
+private:
+    QVector<unsigned char> *data;
+};
+
+static bool resource_sort_order(const VirtualDirectoryEntry *lhs, const VirtualDirectoryEntry *rhs)
+{
+    return qt_hash(lhs->name) < qt_hash(rhs->name);
+}
+
+struct ResourceTree
+{
+    ResourceTree()
+    {}
+
+    void serialize(VirtualDirectoryEntry &root, QVector<unsigned char> *treeData, QVector<unsigned char> *stringData)
+    {
+        treeStream = DataStream(treeData);
+        stringStream = DataStream(stringData);
+
+        QStack<VirtualDirectoryEntry *> directories;
+
+        {
+            directories.push(&root);
+            while (!directories.isEmpty()) {
+                VirtualDirectoryEntry *entry = directories.pop();
+                registerString(entry->name);
+                if (entry->isDirectory)
+                    directories << entry->dirEntries;
+            }
+        }
+
+        {
+            quint32 currentDirectoryIndex = 1;
+            directories.push(&root);
+            while (!directories.isEmpty()) {
+                VirtualDirectoryEntry *entry = directories.pop();
+                entry->firstChildIndex = currentDirectoryIndex;
+                currentDirectoryIndex += entry->dirEntries.count();
+                std::sort(entry->dirEntries.begin(), entry->dirEntries.end(), resource_sort_order);
+
+                for (QVector<VirtualDirectoryEntry*>::ConstIterator child = entry->dirEntries.constBegin(), end = entry->dirEntries.constEnd();
+                     child != end; ++child) {
+                    if ((*child)->isDirectory)
+                        directories << *child;
+                }
+            }
+        }
+
+        {
+            writeTreeEntry(&root);
+            directories.push(&root);
+            while (!directories.isEmpty()) {
+                VirtualDirectoryEntry *entry = directories.pop();
+
+                for (QVector<VirtualDirectoryEntry*>::ConstIterator child = entry->dirEntries.constBegin(), end = entry->dirEntries.constEnd();
+                     child != end; ++child) {
+                    writeTreeEntry(*child);
+                    if ((*child)->isDirectory)
+                        directories << (*child);
+                }
+            }
+        }
+    }
+
+private:
+    DataStream treeStream;
+    DataStream stringStream;
+    QHash<QString, qint64> stringOffsets;
+
+    void registerString(const QString &name)
+    {
+        if (stringOffsets.contains(name))
+            return;
+        const qint64 offset = stringStream.currentOffset();
+        stringOffsets.insert(name, offset);
+
+        stringStream << quint16(name.length())
+                     << quint32(qt_hash(name));
+        for (int i = 0; i < name.length(); ++i)
+            stringStream << quint16(name.at(i).unicode());
+    }
+
+    void writeTreeEntry(VirtualDirectoryEntry *entry)
+    {
+        treeStream << quint32(stringOffsets.value(entry->name))
+                   << quint16(entry->isDirectory ? 0x2 : 0x0); // Flags: File or Directory
+
+        if (entry->isDirectory) {
+            treeStream << quint32(entry->dirEntries.count())
+                       << quint32(entry->firstChildIndex);
+        } else {
+            treeStream << quint16(QLocale::AnyCountry) << quint16(QLocale::C)
+                       << quint32(0x0);
+        }
+    }
+};
+
+static QByteArray generateResourceDirectoryTree(QTextStream &code, const QStringList &qrcFiles)
+{
+    QByteArray call;
+    if (qrcFiles.isEmpty())
+        return call;
+
+    VirtualDirectoryEntry resourceDirs;
+    resourceDirs.name = QStringLiteral("/");
+
+    foreach (const QString &entry, qrcFiles) {
+        const QStringList segments = entry.split(QLatin1Char('/'), QString::SkipEmptyParts);
+
+        VirtualDirectoryEntry *dirEntry = &resourceDirs;
+
+        for (int i = 0; i < segments.count() - 1; ++i)
+            dirEntry = dirEntry->append(segments.at(i));
+        dirEntry->appendEmptyFile(segments.last());
+    }
+
+    if (resourceDirs.isEmpty())
+        return call;
+
+    QVector<unsigned char> names;
+    QVector<unsigned char> tree;
+    ResourceTree().serialize(resourceDirs, &tree, &names);
+
+    code << "static const unsigned char qt_resource_tree[] = {\n";
+    for (int i = 0; i < tree.count(); ++i) {
+        code << uint(tree.at(i));
+        if (i < tree.count() - 1)
+            code << ',';
+        if (i % 16 == 0)
+            code << '\n';
+    }
+    code << "};\n";
+
+    code << "static const unsigned char qt_resource_names[] = {\n";
+    for (int i = 0; i < names.count(); ++i) {
+        code << uint(names.at(i));
+        if (i < names.count() - 1)
+            code << ',';
+        if (i % 16 == 0)
+            code << '\n';
+    }
+    code << "};\n";
+
+    code << "static const unsigned char qt_resource_empty_payout[] = { 0, 0, 0, 0, 0 };\n";
+
+    code << "QT_BEGIN_NAMESPACE\n";
+    code << "extern Q_CORE_EXPORT bool qRegisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);\n";
+    code << "QT_END_NAMESPACE\n";
+
+    call = "QT_PREPEND_NAMESPACE(qRegisterResourceData)(/*version*/0x01, qt_resource_tree, qt_resource_names, qt_resource_empty_payout);\n";
+
+    return call;
+}
+
+static QString qtResourceNameForFile(const QString &fileName)
+{
+    QFileInfo fi(fileName);
+    QString name = fi.completeBaseName();
+    if (name.isEmpty())
+        name = fi.fileName();
+    name.replace(QRegExp(QLatin1String("[^a-zA-Z0-9_]")), QLatin1String("_"));
+    return name;
+}
+
+bool generateLoader(const QStringList &compiledFiles, const QString &outputFileName, const QStringList &resourceFileMappings, QString *errorString)
+{
+    QByteArray generatedLoaderCode;
+
+    {
+        QTextStream stream(&generatedLoaderCode);
+        stream << "#include <QtQml/qqmlprivate.h>\n";
+        stream << "#include <private/qv4compileddata_p.h>\n";
+        stream << "#include <QtCore/qdir.h>\n";
+        stream << "#include <QtCore/qurl.h>\n";
+        stream << "\n";
+
+        QByteArray resourceRegisterCall = generateResourceDirectoryTree(stream, compiledFiles);
+
+        stream << "namespace QmlCacheGeneratedCode {\n";
+        for (int i = 0; i < compiledFiles.count(); ++i) {
+            const QString compiledFile = compiledFiles.at(i);
+            const QString ns = symbolNamespaceForPath(compiledFile);
+            stream << "namespace " << symbolNamespaceForPath(compiledFile) << " { \n";
+            stream << "    extern const unsigned char qmlData[];\n";
+            stream << "    QV4::CompiledData::CompilationUnit *createCompilationUnit() {\n";
+            stream << "        QV4::CompiledData::CompilationUnit *unit = new QV4::CompiledData::CompilationUnit;\n";
+            stream << "        unit->data = reinterpret_cast<const QV4::CompiledData::Unit*>(&qmlData);\n";
+            stream << "        return unit;\n";
+            stream << "    }\n";
+            stream << "    const QQmlPrivate::CachedQmlUnit unit = {\n";
+            stream << "        reinterpret_cast<const QV4::CompiledData::Unit*>(&qmlData), &createCompilationUnit, nullptr\n";
+            stream << "    };\n";
+            stream << "}\n";
+        }
+
+        stream << "\n}\n";
+        stream << "namespace {\n";
+
+        stream << "struct Registry {\n";
+        stream << "    Registry();\n";
+        stream << "    QHash<QString, const QQmlPrivate::CachedQmlUnit*> resourcePathToCachedUnit;\n";
+        stream << "    static const QQmlPrivate::CachedQmlUnit *lookupCachedUnit(const QUrl &url);\n";
+        stream << "};\n\n";
+        stream << "Q_GLOBAL_STATIC(Registry, unitRegistry);\n";
+        stream << "\n\n";
+
+        stream << "Registry::Registry() {\n";
+
+        for (int i = 0; i < compiledFiles.count(); ++i) {
+            const QString qrcFile = compiledFiles.at(i);
+            const QString ns = symbolNamespaceForPath(qrcFile);
+            stream << "        resourcePathToCachedUnit.insert(QStringLiteral(\"" << qrcFile << "\"), &QmlCacheGeneratedCode::" << ns << "::unit);\n";
+        }
+
+        stream << "    QQmlPrivate::RegisterQmlUnitCacheHook registration;\n";
+        stream << "    registration.version = 0;\n";
+        stream << "    registration.lookupCachedQmlUnit = &lookupCachedUnit;\n";
+        stream << "    QQmlPrivate::qmlregister(QQmlPrivate::QmlUnitCacheHookRegistration, &registration);\n";
+
+        if (!resourceRegisterCall.isEmpty())
+            stream << resourceRegisterCall;
+
+        stream << "}\n";
+        stream << "const QQmlPrivate::CachedQmlUnit *Registry::lookupCachedUnit(const QUrl &url) {\n";
+        stream << "    if (url.scheme() != QLatin1String(\"qrc\"))\n";
+        stream << "        return nullptr;\n";
+        stream << "    QString resourcePath = QDir::cleanPath(url.path());\n";
+        stream << "    if (resourcePath.isEmpty())\n";
+        stream << "        return nullptr;\n";
+        stream << "    if (!resourcePath.startsWith(QLatin1Char('/')))\n";
+        stream << "        resourcePath.prepend(QLatin1Char('/'));\n";
+        stream << "    return unitRegistry()->resourcePathToCachedUnit.value(resourcePath, nullptr);\n";
+        stream << "}\n";
+        stream << "}\n";
+
+        for (const QString &mapping: resourceFileMappings) {
+            QString originalResourceFile = mapping;
+            QString newResourceFile;
+            const int mappingSplit = originalResourceFile.indexOf(QLatin1Char('='));
+            if (mappingSplit != -1) {
+                newResourceFile = originalResourceFile.mid(mappingSplit + 1);
+                originalResourceFile.truncate(mappingSplit);
+            }
+
+            const QString function = QLatin1String("qInitResources_") + qtResourceNameForFile(originalResourceFile);
+
+            stream << QStringLiteral("int QT_MANGLE_NAMESPACE(%1)() {\n").arg(function);
+            stream << "    ::unitRegistry();\n";
+            if (!newResourceFile.isEmpty())
+                stream << "    Q_INIT_RESOURCE(" << qtResourceNameForFile(newResourceFile) << ");\n";
+            stream << "    return 1;\n";
+            stream << "}\n";
+            stream << "Q_CONSTRUCTOR_FUNCTION(QT_MANGLE_NAMESPACE(" << function << "));\n";
+        }
+    }
+
+    QSaveFile f(outputFileName);
+    if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
+        *errorString = f.errorString();
+        return false;
+    }
+
+    if (f.write(generatedLoaderCode) != generatedLoaderCode.size()) {
+        *errorString = f.errorString();
+        return false;
+    }
+
+    if (!f.commit()) {
+        *errorString = f.errorString();
+        return false;
+    }
+
+    return true;
+}
diff --git a/tools/qmlcachegen/qmlcachegen.cpp b/tools/qmlcachegen/qmlcachegen.cpp
index 18f8f79ba3..054a08b084 100644
--- a/tools/qmlcachegen/qmlcachegen.cpp
+++ b/tools/qmlcachegen/qmlcachegen.cpp
@@ -33,10 +33,17 @@
 #include <QFileInfo>
 #include <QDateTime>
 #include <QHashFunctions>
+#include <QSaveFile>
 
 #include <private/qqmlirbuilder_p.h>
 #include <private/qqmljsparser_p.h>
 
+#include "resourcefilemapper.h"
+
+int filterResourceFile(const QString &input, const QString &output);
+bool generateLoader(const QStringList &compiledFiles, const QString &output, const QStringList &resourceFileMappings, QString *errorString);
+QString symbolNamespaceForPath(const QString &relativePath);
+
 struct Error
 {
     QString message;
@@ -135,7 +142,9 @@ static bool checkArgumentsObjectUseInSignalHandlers(const QmlIR::Document &doc,
     return true;
 }
 
-static bool compileQmlFile(const QString &inputFileName, const QString &outputFileName, Error *error)
+using SaveFunction = std::function<bool (QV4::CompiledData::CompilationUnit *, QString *)>;
+
+static bool compileQmlFile(const QString &inputFileName, SaveFunction saveFunction, Error *error)
 {
     QmlIR::Document irDocument(/*debugMode*/false);
 
@@ -209,7 +218,7 @@ static bool compileQmlFile(const QString &inputFileName, const QString &outputFi
         unit->flags |= QV4::CompiledData::Unit::PendingTypeCompilation;
         irDocument.javaScriptCompilationUnit->data = unit;
 
-        if (!irDocument.javaScriptCompilationUnit->saveToDisk(outputFileName, &error->message))
+        if (!saveFunction(irDocument.javaScriptCompilationUnit, &error->message))
             return false;
 
         free(unit);
@@ -217,7 +226,7 @@ static bool compileQmlFile(const QString &inputFileName, const QString &outputFi
     return true;
 }
 
-static bool compileJSFile(const QString &inputFileName, const QString &outputFileName, Error *error)
+static bool compileJSFile(const QString &inputFileName, const QString &inputFileUrl, SaveFunction saveFunction, Error *error)
 {
     QmlIR::Document irDocument(/*debugMode*/false);
 
@@ -271,13 +280,12 @@ static bool compileJSFile(const QString &inputFileName, const QString &outputFil
     }
 
     {
-        irDocument.jsModule.fileName = inputFileName;
         QmlIR::JSCodeGen v4CodeGen(irDocument.code, &irDocument.jsGenerator,
                                    &irDocument.jsModule, &irDocument.jsParserEngine,
                                    irDocument.program, /*import cache*/0,
                                    &irDocument.jsGenerator.stringTable);
         v4CodeGen.setUseFastLookups(false); // Disable lookups in non-standalone (aka QML) mode
-        v4CodeGen.generateFromProgram(inputFileName, sourceCode, program, &irDocument.jsModule, QV4::Compiler::GlobalCode);
+        v4CodeGen.generateFromProgram(inputFileUrl, sourceCode, program, &irDocument.jsModule, QV4::Compiler::GlobalCode);
         QList<QQmlJS::DiagnosticMessage> jsErrors = v4CodeGen.errors();
         if (!jsErrors.isEmpty()) {
             for (const QQmlJS::DiagnosticMessage &e: qAsConst(jsErrors)) {
@@ -296,7 +304,7 @@ static bool compileJSFile(const QString &inputFileName, const QString &outputFil
         unit->flags |= QV4::CompiledData::Unit::StaticData;
         irDocument.javaScriptCompilationUnit->data = unit;
 
-        if (!irDocument.javaScriptCompilationUnit->saveToDisk(outputFileName, &error->message)) {
+        if (!saveFunction(irDocument.javaScriptCompilationUnit, &error->message)) {
             engine->setDirectives(oldDirs);
             return false;
         }
@@ -307,6 +315,81 @@ static bool compileJSFile(const QString &inputFileName, const QString &outputFil
     return true;
 }
 
+static bool saveUnitAsCpp(const QString &inputFileName, const QString &outputFileName, QV4::CompiledData::CompilationUnit *unit, QString *errorString)
+{
+    QSaveFile f(outputFileName);
+    if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
+        *errorString = f.errorString();
+        return false;
+    }
+
+    auto writeStr = [&f, errorString](const QByteArray &data) {
+        if (f.write(data) != data.size()) {
+            *errorString = f.errorString();
+            return false;
+        }
+        return true;
+    };
+
+    if (!writeStr("// "))
+        return false;
+
+    if (!writeStr(inputFileName.toUtf8()))
+        return false;
+
+    if (!writeStr("\n"))
+        return false;
+
+    if (!writeStr(QByteArrayLiteral("namespace QmlCacheGeneratedCode {\nnamespace ")))
+        return false;
+
+    if (!writeStr(symbolNamespaceForPath(inputFileName).toUtf8()))
+        return false;
+
+    if (!writeStr(QByteArrayLiteral(" {\nextern const unsigned char qmlData alignas(16) [] = {\n")))
+        return false;
+
+    QByteArray hexifiedData;
+    {
+        QByteArray modifiedUnit;
+        modifiedUnit.resize(unit->data->unitSize);
+        memcpy(modifiedUnit.data(), unit->data, unit->data->unitSize);
+        const char *dataPtr = modifiedUnit.data();
+        QV4::CompiledData::Unit *unitPtr;
+        memcpy(&unitPtr, &dataPtr, sizeof(unitPtr));
+        unitPtr->flags |= QV4::CompiledData::Unit::StaticData;
+
+        QTextStream stream(&hexifiedData);
+        const uchar *begin = reinterpret_cast<const uchar *>(modifiedUnit.constData());
+        const uchar *end = begin + unit->data->unitSize;
+        stream << hex;
+        int col = 0;
+        for (const uchar *data = begin; data < end; ++data, ++col) {
+            if (data > begin)
+                stream << ',';
+            if (col % 8 == 0) {
+                stream << '\n';
+                col = 0;
+            }
+            stream << "0x" << *data;
+        }
+        stream << '\n';
+    };
+
+    if (!writeStr(hexifiedData))
+        return false;
+
+    if (!writeStr("};\n}\n}\n"))
+        return false;
+
+    if (!f.commit()) {
+        *errorString = f.errorString();
+        return false;
+    }
+
+    return true;
+}
+
 int main(int argc, char **argv)
 {
     // Produce reliably the same output for the same input by disabling QHash's random seeding.
@@ -320,6 +403,15 @@ int main(int argc, char **argv)
     parser.addHelpOption();
     parser.addVersionOption();
 
+    QCommandLineOption filterResourceFileOption(QStringLiteral("filter-resource-file"), QCoreApplication::translate("main", "Filter out QML/JS files from a resource file that can be cached ahead of time instead"));
+    parser.addOption(filterResourceFileOption);
+    QCommandLineOption resourceFileMappingOption(QStringLiteral("resource-file-mapping"), QCoreApplication::translate("main", "Path from original resource file to new one"), QCoreApplication::translate("main", "old-name:new-name"));
+    parser.addOption(resourceFileMappingOption);
+    QCommandLineOption resourceOption(QStringLiteral("resource"), QCoreApplication::translate("main", "Qt resource file that might later contain one of the compiled files"), QCoreApplication::translate("main", "resource-file-name"));
+    parser.addOption(resourceOption);
+    QCommandLineOption resourcePathOption(QStringLiteral("resource-path"), QCoreApplication::translate("main", "Qt resource file path corresponding to the file being compiled"), QCoreApplication::translate("main", "resource-path"));
+    parser.addOption(resourcePathOption);
+
     QCommandLineOption outputFileOption(QStringLiteral("o"), QCoreApplication::translate("main", "Output file name"), QCoreApplication::translate("main", "file name"));
     parser.addOption(outputFileOption);
 
@@ -329,6 +421,8 @@ int main(int argc, char **argv)
     parser.addPositionalArgument(QStringLiteral("[qml file]"),
             QStringLiteral("QML source file to generate cache for."));
 
+    parser.setSingleDashWordOptionMode(QCommandLineParser::ParseAsLongOptions);
+
     parser.process(app);
 
     const QStringList sources = parser.positionalArguments();
@@ -346,13 +440,76 @@ int main(int argc, char **argv)
     if (parser.isSet(outputFileOption))
         outputFileName = parser.value(outputFileOption);
 
+    if (parser.isSet(filterResourceFileOption)) {
+        return filterResourceFile(inputFile, outputFileName);
+    }
+
+    if (outputFileName.endsWith(QLatin1String("qmlcache_loader.cpp"))) {
+        ResourceFileMapper mapper(inputFile);
+
+        if (!generateLoader(mapper.qmlCompilerFiles(), outputFileName, parser.values(resourceFileMappingOption), &error.message)) {
+            error.augment(QLatin1String("Error generating loader stub: ")).print();
+            return EXIT_FAILURE;
+        }
+        return EXIT_SUCCESS;
+    }
+
+    QString inputFileUrl = inputFile;
+
+    SaveFunction saveFunction;
+    if (outputFileName.endsWith(QLatin1String(".cpp"))) {
+        ResourceFileMapper fileMapper(parser.values(resourceOption));
+        QString inputResourcePath = parser.value(resourcePathOption);
+
+        if (!inputResourcePath.isEmpty() && !fileMapper.isEmpty()) {
+            fprintf(stderr, "--%s and --%s are mutually exclusive.\n",
+                    qPrintable(resourcePathOption.names().first()),
+                    qPrintable(resourceOption.names().first()));
+            return EXIT_FAILURE;
+        }
+
+        // If the user didn't specify the resource path corresponding to the file on disk being
+        // compiled, try to determine it from the resource file, if one was supplied.
+        if (inputResourcePath.isEmpty()) {
+            const QStringList resourcePaths = fileMapper.resourcePaths(inputFile);
+            if (resourcePaths.isEmpty()) {
+                fprintf(stderr, "No resource path for file: %s\n", qPrintable(inputFile));
+                return EXIT_FAILURE;
+            }
+
+            if (resourcePaths.size() != 1) {
+                fprintf(stderr, "Multiple resource paths for file %s. "
+                                "Use the --%s option to disambiguate:\n",
+                        qPrintable(inputFile),
+                        qPrintable(resourcePathOption.names().first()));
+                for (const QString &resourcePath: resourcePaths)
+                    fprintf(stderr, "\t%s\n", qPrintable(resourcePath));
+                return EXIT_FAILURE;
+            }
+
+            inputResourcePath = resourcePaths.first();
+        }
+
+        inputFileUrl = QStringLiteral("qrc://") + inputResourcePath;
+
+        saveFunction = [inputResourcePath, outputFileName](QV4::CompiledData::CompilationUnit *unit, QString *errorString) {
+            return saveUnitAsCpp(inputResourcePath, outputFileName, unit, errorString);
+        };
+
+    } else {
+        saveFunction = [outputFileName](QV4::CompiledData::CompilationUnit *unit, QString *errorString) {
+            return unit->saveToDisk(outputFileName, errorString);
+        };
+    }
+
+
     if (inputFile.endsWith(QLatin1String(".qml"))) {
-        if (!compileQmlFile(inputFile, outputFileName, &error)) {
+        if (!compileQmlFile(inputFile, saveFunction, &error)) {
             error.augment(QLatin1String("Error compiling qml file: ")).print();
             return EXIT_FAILURE;
         }
     } else if (inputFile.endsWith(QLatin1String(".js"))) {
-        if (!compileJSFile(inputFile, outputFileName, &error)) {
+        if (!compileJSFile(inputFile, inputFileUrl, saveFunction, &error)) {
             error.augment(QLatin1String("Error compiling qml file: ")).print();
             return EXIT_FAILURE;
         }
diff --git a/tools/qmlcachegen/qmlcachegen.pro b/tools/qmlcachegen/qmlcachegen.pro
index be92ef435e..1b76833b5a 100644
--- a/tools/qmlcachegen/qmlcachegen.pro
+++ b/tools/qmlcachegen/qmlcachegen.pro
@@ -3,10 +3,13 @@ option(host_build)
 QT = qmldevtools-private
 DEFINES += QT_NO_CAST_TO_ASCII QT_NO_CAST_FROM_ASCII
 
-SOURCES = qmlcachegen.cpp
+SOURCES = qmlcachegen.cpp \
+    resourcefilter.cpp \
+    generateloader.cpp \
+    resourcefilemapper.cpp
 TARGET = qmlcachegen
 
-build_integration.files = qmlcache.prf
+build_integration.files = qmlcache.prf qtquickcompiler.prf
 build_integration.path = $$[QT_HOST_DATA]/mkspecs/features
 prefix_build: INSTALLS += build_integration
 else: COPIES += build_integration
@@ -14,3 +17,6 @@ else: COPIES += build_integration
 QMAKE_TARGET_DESCRIPTION = QML Cache Generator
 
 load(qt_tool)
+
+HEADERS += \
+    resourcefilemapper.h
diff --git a/tools/qmlcachegen/qtquickcompiler.prf b/tools/qmlcachegen/qtquickcompiler.prf
new file mode 100644
index 0000000000..985a0a5cdc
--- /dev/null
+++ b/tools/qmlcachegen/qtquickcompiler.prf
@@ -0,0 +1,92 @@
+
+qtPrepareTool(QML_CACHEGEN, qmlcachegen, _FILTER)
+qtPrepareTool(QMAKE_RCC, rcc, _DEP)
+
+QMLCACHE_DIR = .qmlcache
+
+defineReplace(qmlCacheResourceFileOutputName) {
+    name = $$relative_path($$1, $$_PRO_FILE_PWD_)
+    name = $$replace(name,/,_)
+    name = $$replace(name, \\.qrc$, _qmlcache.qrc)
+    name = $$replace(name,\.\.,)
+    name = $$replace(name,-,_)
+    name = $$absolute_path($$name, $$OUT_PWD)
+    return($${name})
+}
+
+# Flatten RESOURCES that may contain individual files or objects
+load(resources)
+
+NEWRESOURCES =
+QMLCACHE_RESOURCE_FILES =
+
+for(res, RESOURCES) {
+    absRes = $$absolute_path($$res, $$_PRO_FILE_PWD_)
+    rccContents = $$system($$QMAKE_RCC_DEP -list $$absRes,lines)
+    contains(rccContents,.*\\.js$)|contains(rccContents,.*\\.qml$) {
+        new_resource = $$qmlCacheResourceFileOutputName($$res)
+        mkpath($$dirname(new_resource))
+        remaining_files = $$system($$QML_CACHEGEN_FILTER -filter-resource-file -o $$new_resource $$absRes,lines)
+        !isEmpty(remaining_files) {
+            NEWRESOURCES += $$new_resource
+            QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$absRes=$$new_resource
+        } else {
+            QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$absRes
+        }
+
+        QMLCACHE_RESOURCE_FILES += $$absRes
+
+        for(candidate, $$list($$rccContents)) {
+            contains(candidate,.*\\.js$)|contains(candidate,.*\\.qml$) {
+                QMLCACHE_FILES += $$candidate
+            }
+        }
+    } else {
+        NEWRESOURCES += $$res
+    }
+}
+
+RESOURCES = $$NEWRESOURCES
+
+QMLCACHE_RESOURCE_FLAGS =
+for(res, QMLCACHE_RESOURCE_FILES) {
+    QMLCACHE_RESOURCE_FLAGS += --resource=$$res
+}
+
+!isEmpty(QMLCACHE_FILES) {
+    QT_PRIVATE *= core-private qml-private
+}
+
+defineReplace(qmlCacheOutputName) {
+    name = $$absolute_path($$1, $$OUT_PWD)
+    name = $$relative_path($$name, $$_PRO_FILE_PWD_)
+    name = $$replace(name, \\.qml$, _qml)
+    name = $$replace(name, \\.js$, _js)
+    name = $$replace(name,/,_)
+    name = $$QMLCACHE_DIR/$${name}
+    return($${name})
+}
+
+qmlcache.input = QMLCACHE_FILES
+qmlcache.output = ${QMAKE_FUNC_FILE_IN_qmlCacheOutputName}$${first(QMAKE_EXT_CPP)}
+qmlcache.commands = $$QML_CACHEGEN $$QMLCACHE_RESOURCE_FLAGS $$QMLCACHE_FLAGS -o ${QMAKE_FILE_OUT} ${QMAKE_FILE_IN}
+qmlcache.name = qmlcachegen ${QMAKE_FILE_IN}
+qmlcache.variable_out = GENERATED_SOURCES
+qmlcache.dependency_type = TYPE_C
+
+qmlcache_loader.input = QMLCACHE_RESOURCE_FILES
+qmlcache_loader.output = $$QMLCACHE_DIR/qmlcache_loader.cpp
+qmlcache_loader.commands = $$QML_CACHEGEN $$QMLCACHE_LOADER_FLAGS $$QMLCACHE_FLAGS -o ${QMAKE_FILE_OUT} ${QMAKE_FILE_IN}
+qmlcache_loader.name = qmlcachengen_loader ${QMAKE_FILE_IN}
+qmlcache_loader.variable_out = SOURCES
+qmlcache_loader.dependency_type = TYPE_C
+qmlcache_loader.CONFIG = combine
+
+unix:!no_qmlcache_depend {
+    compiler_bin = $$[QT_HOST_BINS]/qmlcachegen
+    contains(QMAKE_HOST.os, Windows): compiler_bin = $${compiler_bin}.exe
+    qmlcache.depends += $$compiler_bin
+    qmlcache_loader.depends += $$compiler_bin
+}
+
+QMAKE_EXTRA_COMPILERS += qmlcache qmlcache_loader
diff --git a/tools/qmlcachegen/resourcefilemapper.cpp b/tools/qmlcachegen/resourcefilemapper.cpp
new file mode 100644
index 0000000000..d7daf33de8
--- /dev/null
+++ b/tools/qmlcachegen/resourcefilemapper.cpp
@@ -0,0 +1,177 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtQml module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "resourcefilemapper.h"
+
+#include <QFileInfo>
+#include <QDir>
+#include <QXmlStreamReader>
+
+ResourceFileMapper::ResourceFileMapper(const QStringList &resourceFiles)
+{
+    for (const QString &fileName: resourceFiles) {
+        QFile f(fileName);
+        if (!f.open(QIODevice::ReadOnly))
+            continue;
+        populateFromQrcFile(f);
+    }
+}
+
+ResourceFileMapper::ResourceFileMapper(const QString &resourceFile)
+{
+    QFile f(resourceFile);
+    if (!f.open(QIODevice::ReadOnly))
+        return;
+    populateFromQrcFile(f);
+}
+
+bool ResourceFileMapper::isEmpty() const
+{
+    return qrcPathToFileSystemPath.isEmpty();
+}
+
+QStringList ResourceFileMapper::resourcePaths(const QString &fileName)
+{
+    const QString absPath = QDir::cleanPath(QDir::current().absoluteFilePath(fileName));
+    QHashIterator<QString, QString> it(qrcPathToFileSystemPath);
+    QStringList resourcePaths;
+    while (it.hasNext()) {
+        it.next();
+        if (QFileInfo(it.value()) == QFileInfo(absPath))
+            resourcePaths.append(it.key());
+    }
+    return resourcePaths;
+}
+
+QStringList ResourceFileMapper::qmlCompilerFiles() const
+{
+    QStringList files;
+    for (auto it = qrcPathToFileSystemPath.constBegin(), end = qrcPathToFileSystemPath.constEnd();
+         it != end; ++it) {
+        const QString &qrcPath = it.key();
+        const QString suffix = QFileInfo(qrcPath).suffix();
+        if (suffix != QStringLiteral("qml") && suffix != QStringLiteral("js"))
+            continue;
+        files << qrcPath;
+    }
+    return files;
+}
+
+void ResourceFileMapper::populateFromQrcFile(QFile &file)
+{
+    enum State {
+        InitialState,
+        InRCC,
+        InResource,
+        InFile
+    };
+    State state = InitialState;
+
+    QDir qrcDir = QFileInfo(file).absoluteDir();
+
+    QString prefix;
+    QString currentFileName;
+    QXmlStreamAttributes currentFileAttributes;
+
+    QXmlStreamReader reader(&file);
+    while (!reader.atEnd()) {
+        switch (reader.readNext()) {
+        case QXmlStreamReader::StartElement:
+            if (reader.name() == QStringLiteral("RCC")) {
+                if (state != InitialState)
+                    return;
+                state = InRCC;
+                continue;
+            } else if (reader.name() == QStringLiteral("qresource")) {
+                if (state != InRCC)
+                    return;
+                state = InResource;
+                QXmlStreamAttributes attributes = reader.attributes();
+                if (attributes.hasAttribute(QStringLiteral("prefix")))
+                    prefix = attributes.value(QStringLiteral("prefix")).toString();
+                if (!prefix.startsWith(QLatin1Char('/')))
+                    prefix.prepend(QLatin1Char('/'));
+                if (!prefix.endsWith(QLatin1Char('/')))
+                    prefix.append(QLatin1Char('/'));
+                continue;
+            } else if (reader.name() == QStringLiteral("file")) {
+                if (state != InResource)
+                    return;
+                state = InFile;
+                currentFileAttributes = reader.attributes();
+                continue;
+            }
+            return;
+
+        case QXmlStreamReader::EndElement:
+            if (reader.name() == QStringLiteral("file")) {
+                if (state != InFile)
+                    return;
+                state = InResource;
+                continue;
+            } else if (reader.name() == QStringLiteral("qresource")) {
+                if (state != InResource)
+                    return;
+                state = InRCC;
+                continue;
+            } else if (reader.name() == QStringLiteral("RCC")) {
+                if (state != InRCC)
+                    return;
+                state = InitialState;
+                continue;
+            }
+            return;
+
+        case QXmlStreamReader::Characters: {
+            if (reader.isWhitespace())
+                break;
+            if (state != InFile)
+                return;
+            currentFileName = reader.text().toString();
+            if (currentFileName.isEmpty())
+                continue;
+
+            const QString fsPath = QDir::cleanPath(qrcDir.absoluteFilePath(currentFileName));
+
+            if (currentFileAttributes.hasAttribute(QStringLiteral("alias")))
+                currentFileName = currentFileAttributes.value(QStringLiteral("alias")).toString();
+
+            currentFileName = QDir::cleanPath(currentFileName);
+            while (currentFileName.startsWith(QLatin1String("../")))
+                currentFileName.remove(0, 3);
+
+            const QString qrcPath = prefix + currentFileName;
+            if (QFile::exists(fsPath))
+                qrcPathToFileSystemPath.insert(qrcPath, fsPath);
+            continue;
+        }
+
+        default: break;
+    }
+    }
+}
diff --git a/tools/qmlcachegen/resourcefilemapper.h b/tools/qmlcachegen/resourcefilemapper.h
new file mode 100644
index 0000000000..f6e82bacb0
--- /dev/null
+++ b/tools/qmlcachegen/resourcefilemapper.h
@@ -0,0 +1,51 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtQml module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#ifndef RESOURCEFILEMAPPER_H
+#define RESOURCEFILEMAPPER_H
+
+#include <QStringList>
+#include <QHash>
+#include <QFile>
+
+struct ResourceFileMapper
+{
+    ResourceFileMapper(const QStringList &resourceFiles);
+    ResourceFileMapper(const QString &resourceFile);
+
+    bool isEmpty() const;
+
+    QStringList resourcePaths(const QString &fileName);
+    QStringList qmlCompilerFiles() const;
+
+private:
+    void populateFromQrcFile(QFile &file);
+
+    QHash<QString, QString> qrcPathToFileSystemPath;
+};
+
+#endif // RESOURCEFILEMAPPER_H
diff --git a/tools/qmlcachegen/resourcefilter.cpp b/tools/qmlcachegen/resourcefilter.cpp
new file mode 100644
index 0000000000..196dbd4a75
--- /dev/null
+++ b/tools/qmlcachegen/resourcefilter.cpp
@@ -0,0 +1,183 @@
+/****************************************************************************
+**
+** Copyright (C) 2016 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtQml module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#include <QString>
+#include <QXmlStreamReader>
+#include <QFile>
+#include <QDir>
+
+int filterResourceFile(const QString &input, const QString &output)
+{
+    enum State {
+        InitialState,
+        InRCC,
+        InResource,
+        InFile
+    };
+    State state = InitialState;
+
+    QString prefix;
+    QString currentFileName;
+    QXmlStreamAttributes fileAttributes;
+
+    QFile file(input);
+    if (!file.open(QIODevice::ReadOnly)) {
+        fprintf(stderr, "Cannot open %s for reading.\n", qPrintable(input));
+        return EXIT_FAILURE;
+    }
+
+    QDir inputDirectory = QFileInfo(file).absoluteDir();
+    QDir outputDirectory = QFileInfo(output).absoluteDir();
+
+    QString outputString;
+    QXmlStreamWriter writer(&outputString);
+    writer.setAutoFormatting(true);
+
+    QStringList remainingFiles;
+
+    QXmlStreamReader reader(&file);
+    while (!reader.atEnd()) {
+        switch (reader.readNext()) {
+        case QXmlStreamReader::StartDocument: {
+            QStringRef version = reader.documentVersion();
+            if (!version.isEmpty())
+                writer.writeStartDocument(version.toString());
+            else
+                writer.writeStartDocument();
+            break;
+        }
+        case QXmlStreamReader::EndDocument:
+            writer.writeEndDocument();
+            break;
+        case QXmlStreamReader::StartElement:
+            if (reader.name() == QStringLiteral("RCC")) {
+                if (state != InitialState) {
+                    fprintf(stderr, "Unexpected RCC tag in line %d\n", int(reader.lineNumber()));
+                    return EXIT_FAILURE;
+                }
+                state = InRCC;
+            } else if (reader.name() == QStringLiteral("qresource")) {
+                if (state != InRCC) {
+                    fprintf(stderr, "Unexpected qresource tag in line %d\n", int(reader.lineNumber()));
+                    return EXIT_FAILURE;
+                }
+                state = InResource;
+                QXmlStreamAttributes attributes = reader.attributes();
+                if (attributes.hasAttribute(QStringLiteral("prefix")))
+                    prefix = attributes.value(QStringLiteral("prefix")).toString();
+                if (!prefix.startsWith(QLatin1Char('/')))
+                    prefix.prepend(QLatin1Char('/'));
+                if (!prefix.endsWith(QLatin1Char('/')))
+                    prefix.append(QLatin1Char('/'));
+            } else if (reader.name() == QStringLiteral("file")) {
+                if (state != InResource) {
+                    fprintf(stderr, "Unexpected file tag in line %d\n", int(reader.lineNumber()));
+                    return EXIT_FAILURE;
+                }
+                state = InFile;
+                fileAttributes = reader.attributes();
+                continue;
+            }
+            writer.writeStartElement(reader.name().toString());
+            writer.writeAttributes(reader.attributes());
+            continue;
+
+        case QXmlStreamReader::EndElement:
+            if (reader.name() == QStringLiteral("file")) {
+                if (state != InFile) {
+                    fprintf(stderr, "Unexpected end of file tag in line %d\n", int(reader.lineNumber()));
+                    return EXIT_FAILURE;
+                }
+                state = InResource;
+                continue;
+            } else if (reader.name() == QStringLiteral("qresource")) {
+                if (state != InResource) {
+                    fprintf(stderr, "Unexpected end of qresource tag in line %d\n", int(reader.lineNumber()));
+                    return EXIT_FAILURE;
+                }
+                state = InRCC;
+            } else if (reader.name() == QStringLiteral("RCC")) {
+                if (state != InRCC) {
+                    fprintf(stderr, "Unexpected end of RCC tag in line %d\n", int(reader.lineNumber()));
+                    return EXIT_FAILURE;
+                }
+                state = InitialState;
+            }
+            writer.writeEndElement();
+            continue;
+
+        case QXmlStreamReader::Characters:
+            if (reader.isWhitespace())
+                break;
+            if (state != InFile)
+                return EXIT_FAILURE;
+            currentFileName = reader.text().toString();
+            if (currentFileName.isEmpty())
+                continue;
+
+            if (!currentFileName.endsWith(QStringLiteral(".qml")) && !currentFileName.endsWith(QStringLiteral(".js"))) {
+                writer.writeStartElement(QStringLiteral("file"));
+
+                if (!fileAttributes.hasAttribute(QStringLiteral("alias")))
+                    fileAttributes.append(QStringLiteral("alias"), currentFileName);
+
+                currentFileName = inputDirectory.absoluteFilePath(currentFileName);
+                currentFileName = outputDirectory.relativeFilePath(currentFileName);
+
+                remainingFiles << currentFileName;
+
+                writer.writeAttributes(fileAttributes);
+                writer.writeCharacters(currentFileName);
+                writer.writeEndElement();
+            }
+            continue;
+
+        default: break;
+    }
+    }
+
+    if (!remainingFiles.isEmpty()) {
+        QFile outputFile(output);
+        if (!outputFile.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
+            fprintf(stderr, "Cannot open %s for writing.\n", qPrintable(output));
+            return EXIT_FAILURE;
+        }
+        const QByteArray outputStringUtf8 = outputString.toUtf8();
+        if (outputFile.write(outputStringUtf8) != outputStringUtf8.size())
+            return EXIT_FAILURE;
+
+        outputFile.close();
+        if (outputFile.error() != QFileDevice::NoError)
+            return EXIT_FAILURE;
+
+        // The build system expects this output if we wrote a qrc file and no output
+        // if no files remain.
+        fprintf(stdout, "New resource file written with %d files.\n", remainingFiles.count());
+    }
+
+    return EXIT_SUCCESS;
+}