From e5b157275d7b924e1a0f39d27958a7ad1eed9cef Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 22 Jan 2018 16:46:03 +0100 Subject: Add support for compiling QML/JS files ahead of time in resources This is bringing over the loading infrastructure from the Qt Quick Compiler that allows embedding qml/js files in resources and compiling them ahead of time. At the moment, the approach of generating one cpp file per qml/js file and the loader stub is needed because the build system does not support dynamic resource generation. In addition, as per QTBUG-60961, we must ensure that the generated data structures are aligned. To retain compatibility this is enabled via CONFIG += qtquickcompiler, but we may need to find a new name (but should keep the old one in any case). Task-number: QTBUG-60961 Change-Id: Ia9839bf98d3af4c50636b6e06815364a9fc7ee57 Reviewed-by: Lars Knoll --- tools/qmlcachegen/generateloader.cpp | 398 +++++++++++++++++++++++++++++++ tools/qmlcachegen/qmlcachegen.cpp | 173 +++++++++++++- tools/qmlcachegen/qmlcachegen.pro | 10 +- tools/qmlcachegen/qtquickcompiler.prf | 92 +++++++ tools/qmlcachegen/resourcefilemapper.cpp | 177 ++++++++++++++ tools/qmlcachegen/resourcefilemapper.h | 51 ++++ tools/qmlcachegen/resourcefilter.cpp | 183 ++++++++++++++ 7 files changed, 1074 insertions(+), 10 deletions(-) create mode 100644 tools/qmlcachegen/generateloader.cpp create mode 100644 tools/qmlcachegen/qtquickcompiler.prf create mode 100644 tools/qmlcachegen/resourcefilemapper.cpp create mode 100644 tools/qmlcachegen/resourcefilemapper.h create mode 100644 tools/qmlcachegen/resourcefilter.cpp (limited to 'tools/qmlcachegen') diff --git a/tools/qmlcachegen/generateloader.cpp b/tools/qmlcachegen/generateloader.cpp new file mode 100644 index 0000000000..dd2e244b56 --- /dev/null +++ b/tools/qmlcachegen/generateloader.cpp @@ -0,0 +1,398 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtQml module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +QString symbolNamespaceForPath(const QString &relativePath) +{ + QFileInfo fi(relativePath); + QString symbol = fi.path(); + if (symbol.length() == 1 && symbol.startsWith(QLatin1Char('.'))) { + symbol.clear(); + } else { + symbol.replace(QLatin1Char('/'), QLatin1Char('_')); + symbol += QLatin1Char('_'); + } + symbol += fi.baseName(); + symbol += QLatin1Char('_'); + symbol += fi.suffix(); + symbol.replace(QLatin1Char('.'), QLatin1Char('_')); + symbol.replace(QLatin1Char('+'), QLatin1Char('_')); + symbol.replace(QLatin1Char('-'), QLatin1Char('_')); + return symbol; +} + +struct VirtualDirectoryEntry +{ + QString name; + QVector dirEntries; + int firstChildIndex; // node index inside generated data + bool isDirectory; + + VirtualDirectoryEntry() + : firstChildIndex(-1) + , isDirectory(true) + {} + + ~VirtualDirectoryEntry() + { + qDeleteAll(dirEntries); + } + + VirtualDirectoryEntry *append(const QString &name) + { + for (QVector::Iterator it = dirEntries.begin(), end = dirEntries.end(); + it != end; ++it) { + if ((*it)->name == name) + return *it; + } + + VirtualDirectoryEntry *subEntry = new VirtualDirectoryEntry; + subEntry->name = name; + dirEntries.append(subEntry); + return subEntry; + } + + void appendEmptyFile(const QString &name) + { + VirtualDirectoryEntry *subEntry = new VirtualDirectoryEntry; + subEntry->name = name; + subEntry->isDirectory = false; + dirEntries.append(subEntry); + } + + bool isEmpty() const { return dirEntries.isEmpty(); } +}; + +struct DataStream +{ + DataStream(QVector *data = 0) + : data(data) + {} + + qint64 currentOffset() const { return data->size(); } + + DataStream &operator<<(quint16 value) + { + unsigned char d[2]; + qToBigEndian(value, d); + data->append(d[0]); + data->append(d[1]); + return *this; + } + DataStream &operator<<(quint32 value) + { + unsigned char d[4]; + qToBigEndian(value, d); + data->append(d[0]); + data->append(d[1]); + data->append(d[2]); + data->append(d[3]); + return *this; + } +private: + QVector *data; +}; + +static bool resource_sort_order(const VirtualDirectoryEntry *lhs, const VirtualDirectoryEntry *rhs) +{ + return qt_hash(lhs->name) < qt_hash(rhs->name); +} + +struct ResourceTree +{ + ResourceTree() + {} + + void serialize(VirtualDirectoryEntry &root, QVector *treeData, QVector *stringData) + { + treeStream = DataStream(treeData); + stringStream = DataStream(stringData); + + QStack directories; + + { + directories.push(&root); + while (!directories.isEmpty()) { + VirtualDirectoryEntry *entry = directories.pop(); + registerString(entry->name); + if (entry->isDirectory) + directories << entry->dirEntries; + } + } + + { + quint32 currentDirectoryIndex = 1; + directories.push(&root); + while (!directories.isEmpty()) { + VirtualDirectoryEntry *entry = directories.pop(); + entry->firstChildIndex = currentDirectoryIndex; + currentDirectoryIndex += entry->dirEntries.count(); + std::sort(entry->dirEntries.begin(), entry->dirEntries.end(), resource_sort_order); + + for (QVector::ConstIterator child = entry->dirEntries.constBegin(), end = entry->dirEntries.constEnd(); + child != end; ++child) { + if ((*child)->isDirectory) + directories << *child; + } + } + } + + { + writeTreeEntry(&root); + directories.push(&root); + while (!directories.isEmpty()) { + VirtualDirectoryEntry *entry = directories.pop(); + + for (QVector::ConstIterator child = entry->dirEntries.constBegin(), end = entry->dirEntries.constEnd(); + child != end; ++child) { + writeTreeEntry(*child); + if ((*child)->isDirectory) + directories << (*child); + } + } + } + } + +private: + DataStream treeStream; + DataStream stringStream; + QHash stringOffsets; + + void registerString(const QString &name) + { + if (stringOffsets.contains(name)) + return; + const qint64 offset = stringStream.currentOffset(); + stringOffsets.insert(name, offset); + + stringStream << quint16(name.length()) + << quint32(qt_hash(name)); + for (int i = 0; i < name.length(); ++i) + stringStream << quint16(name.at(i).unicode()); + } + + void writeTreeEntry(VirtualDirectoryEntry *entry) + { + treeStream << quint32(stringOffsets.value(entry->name)) + << quint16(entry->isDirectory ? 0x2 : 0x0); // Flags: File or Directory + + if (entry->isDirectory) { + treeStream << quint32(entry->dirEntries.count()) + << quint32(entry->firstChildIndex); + } else { + treeStream << quint16(QLocale::AnyCountry) << quint16(QLocale::C) + << quint32(0x0); + } + } +}; + +static QByteArray generateResourceDirectoryTree(QTextStream &code, const QStringList &qrcFiles) +{ + QByteArray call; + if (qrcFiles.isEmpty()) + return call; + + VirtualDirectoryEntry resourceDirs; + resourceDirs.name = QStringLiteral("/"); + + foreach (const QString &entry, qrcFiles) { + const QStringList segments = entry.split(QLatin1Char('/'), QString::SkipEmptyParts); + + VirtualDirectoryEntry *dirEntry = &resourceDirs; + + for (int i = 0; i < segments.count() - 1; ++i) + dirEntry = dirEntry->append(segments.at(i)); + dirEntry->appendEmptyFile(segments.last()); + } + + if (resourceDirs.isEmpty()) + return call; + + QVector names; + QVector tree; + ResourceTree().serialize(resourceDirs, &tree, &names); + + code << "static const unsigned char qt_resource_tree[] = {\n"; + for (int i = 0; i < tree.count(); ++i) { + code << uint(tree.at(i)); + if (i < tree.count() - 1) + code << ','; + if (i % 16 == 0) + code << '\n'; + } + code << "};\n"; + + code << "static const unsigned char qt_resource_names[] = {\n"; + for (int i = 0; i < names.count(); ++i) { + code << uint(names.at(i)); + if (i < names.count() - 1) + code << ','; + if (i % 16 == 0) + code << '\n'; + } + code << "};\n"; + + code << "static const unsigned char qt_resource_empty_payout[] = { 0, 0, 0, 0, 0 };\n"; + + code << "QT_BEGIN_NAMESPACE\n"; + code << "extern Q_CORE_EXPORT bool qRegisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);\n"; + code << "QT_END_NAMESPACE\n"; + + call = "QT_PREPEND_NAMESPACE(qRegisterResourceData)(/*version*/0x01, qt_resource_tree, qt_resource_names, qt_resource_empty_payout);\n"; + + return call; +} + +static QString qtResourceNameForFile(const QString &fileName) +{ + QFileInfo fi(fileName); + QString name = fi.completeBaseName(); + if (name.isEmpty()) + name = fi.fileName(); + name.replace(QRegExp(QLatin1String("[^a-zA-Z0-9_]")), QLatin1String("_")); + return name; +} + +bool generateLoader(const QStringList &compiledFiles, const QString &outputFileName, const QStringList &resourceFileMappings, QString *errorString) +{ + QByteArray generatedLoaderCode; + + { + QTextStream stream(&generatedLoaderCode); + stream << "#include \n"; + stream << "#include \n"; + stream << "#include \n"; + stream << "#include \n"; + stream << "\n"; + + QByteArray resourceRegisterCall = generateResourceDirectoryTree(stream, compiledFiles); + + stream << "namespace QmlCacheGeneratedCode {\n"; + for (int i = 0; i < compiledFiles.count(); ++i) { + const QString compiledFile = compiledFiles.at(i); + const QString ns = symbolNamespaceForPath(compiledFile); + stream << "namespace " << symbolNamespaceForPath(compiledFile) << " { \n"; + stream << " extern const unsigned char qmlData[];\n"; + stream << " QV4::CompiledData::CompilationUnit *createCompilationUnit() {\n"; + stream << " QV4::CompiledData::CompilationUnit *unit = new QV4::CompiledData::CompilationUnit;\n"; + stream << " unit->data = reinterpret_cast(&qmlData);\n"; + stream << " return unit;\n"; + stream << " }\n"; + stream << " const QQmlPrivate::CachedQmlUnit unit = {\n"; + stream << " reinterpret_cast(&qmlData), &createCompilationUnit, nullptr\n"; + stream << " };\n"; + stream << "}\n"; + } + + stream << "\n}\n"; + stream << "namespace {\n"; + + stream << "struct Registry {\n"; + stream << " Registry();\n"; + stream << " QHash resourcePathToCachedUnit;\n"; + stream << " static const QQmlPrivate::CachedQmlUnit *lookupCachedUnit(const QUrl &url);\n"; + stream << "};\n\n"; + stream << "Q_GLOBAL_STATIC(Registry, unitRegistry);\n"; + stream << "\n\n"; + + stream << "Registry::Registry() {\n"; + + for (int i = 0; i < compiledFiles.count(); ++i) { + const QString qrcFile = compiledFiles.at(i); + const QString ns = symbolNamespaceForPath(qrcFile); + stream << " resourcePathToCachedUnit.insert(QStringLiteral(\"" << qrcFile << "\"), &QmlCacheGeneratedCode::" << ns << "::unit);\n"; + } + + stream << " QQmlPrivate::RegisterQmlUnitCacheHook registration;\n"; + stream << " registration.version = 0;\n"; + stream << " registration.lookupCachedQmlUnit = &lookupCachedUnit;\n"; + stream << " QQmlPrivate::qmlregister(QQmlPrivate::QmlUnitCacheHookRegistration, ®istration);\n"; + + if (!resourceRegisterCall.isEmpty()) + stream << resourceRegisterCall; + + stream << "}\n"; + stream << "const QQmlPrivate::CachedQmlUnit *Registry::lookupCachedUnit(const QUrl &url) {\n"; + stream << " if (url.scheme() != QLatin1String(\"qrc\"))\n"; + stream << " return nullptr;\n"; + stream << " QString resourcePath = QDir::cleanPath(url.path());\n"; + stream << " if (resourcePath.isEmpty())\n"; + stream << " return nullptr;\n"; + stream << " if (!resourcePath.startsWith(QLatin1Char('/')))\n"; + stream << " resourcePath.prepend(QLatin1Char('/'));\n"; + stream << " return unitRegistry()->resourcePathToCachedUnit.value(resourcePath, nullptr);\n"; + stream << "}\n"; + stream << "}\n"; + + for (const QString &mapping: resourceFileMappings) { + QString originalResourceFile = mapping; + QString newResourceFile; + const int mappingSplit = originalResourceFile.indexOf(QLatin1Char('=')); + if (mappingSplit != -1) { + newResourceFile = originalResourceFile.mid(mappingSplit + 1); + originalResourceFile.truncate(mappingSplit); + } + + const QString function = QLatin1String("qInitResources_") + qtResourceNameForFile(originalResourceFile); + + stream << QStringLiteral("int QT_MANGLE_NAMESPACE(%1)() {\n").arg(function); + stream << " ::unitRegistry();\n"; + if (!newResourceFile.isEmpty()) + stream << " Q_INIT_RESOURCE(" << qtResourceNameForFile(newResourceFile) << ");\n"; + stream << " return 1;\n"; + stream << "}\n"; + stream << "Q_CONSTRUCTOR_FUNCTION(QT_MANGLE_NAMESPACE(" << function << "));\n"; + } + } + + QSaveFile f(outputFileName); + if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + *errorString = f.errorString(); + return false; + } + + if (f.write(generatedLoaderCode) != generatedLoaderCode.size()) { + *errorString = f.errorString(); + return false; + } + + if (!f.commit()) { + *errorString = f.errorString(); + return false; + } + + return true; +} diff --git a/tools/qmlcachegen/qmlcachegen.cpp b/tools/qmlcachegen/qmlcachegen.cpp index 18f8f79ba3..054a08b084 100644 --- a/tools/qmlcachegen/qmlcachegen.cpp +++ b/tools/qmlcachegen/qmlcachegen.cpp @@ -33,10 +33,17 @@ #include #include #include +#include #include #include +#include "resourcefilemapper.h" + +int filterResourceFile(const QString &input, const QString &output); +bool generateLoader(const QStringList &compiledFiles, const QString &output, const QStringList &resourceFileMappings, QString *errorString); +QString symbolNamespaceForPath(const QString &relativePath); + struct Error { QString message; @@ -135,7 +142,9 @@ static bool checkArgumentsObjectUseInSignalHandlers(const QmlIR::Document &doc, return true; } -static bool compileQmlFile(const QString &inputFileName, const QString &outputFileName, Error *error) +using SaveFunction = std::function; + +static bool compileQmlFile(const QString &inputFileName, SaveFunction saveFunction, Error *error) { QmlIR::Document irDocument(/*debugMode*/false); @@ -209,7 +218,7 @@ static bool compileQmlFile(const QString &inputFileName, const QString &outputFi unit->flags |= QV4::CompiledData::Unit::PendingTypeCompilation; irDocument.javaScriptCompilationUnit->data = unit; - if (!irDocument.javaScriptCompilationUnit->saveToDisk(outputFileName, &error->message)) + if (!saveFunction(irDocument.javaScriptCompilationUnit, &error->message)) return false; free(unit); @@ -217,7 +226,7 @@ static bool compileQmlFile(const QString &inputFileName, const QString &outputFi return true; } -static bool compileJSFile(const QString &inputFileName, const QString &outputFileName, Error *error) +static bool compileJSFile(const QString &inputFileName, const QString &inputFileUrl, SaveFunction saveFunction, Error *error) { QmlIR::Document irDocument(/*debugMode*/false); @@ -271,13 +280,12 @@ static bool compileJSFile(const QString &inputFileName, const QString &outputFil } { - irDocument.jsModule.fileName = inputFileName; QmlIR::JSCodeGen v4CodeGen(irDocument.code, &irDocument.jsGenerator, &irDocument.jsModule, &irDocument.jsParserEngine, irDocument.program, /*import cache*/0, &irDocument.jsGenerator.stringTable); v4CodeGen.setUseFastLookups(false); // Disable lookups in non-standalone (aka QML) mode - v4CodeGen.generateFromProgram(inputFileName, sourceCode, program, &irDocument.jsModule, QV4::Compiler::GlobalCode); + v4CodeGen.generateFromProgram(inputFileUrl, sourceCode, program, &irDocument.jsModule, QV4::Compiler::GlobalCode); QList jsErrors = v4CodeGen.errors(); if (!jsErrors.isEmpty()) { for (const QQmlJS::DiagnosticMessage &e: qAsConst(jsErrors)) { @@ -296,7 +304,7 @@ static bool compileJSFile(const QString &inputFileName, const QString &outputFil unit->flags |= QV4::CompiledData::Unit::StaticData; irDocument.javaScriptCompilationUnit->data = unit; - if (!irDocument.javaScriptCompilationUnit->saveToDisk(outputFileName, &error->message)) { + if (!saveFunction(irDocument.javaScriptCompilationUnit, &error->message)) { engine->setDirectives(oldDirs); return false; } @@ -307,6 +315,81 @@ static bool compileJSFile(const QString &inputFileName, const QString &outputFil return true; } +static bool saveUnitAsCpp(const QString &inputFileName, const QString &outputFileName, QV4::CompiledData::CompilationUnit *unit, QString *errorString) +{ + QSaveFile f(outputFileName); + if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + *errorString = f.errorString(); + return false; + } + + auto writeStr = [&f, errorString](const QByteArray &data) { + if (f.write(data) != data.size()) { + *errorString = f.errorString(); + return false; + } + return true; + }; + + if (!writeStr("// ")) + return false; + + if (!writeStr(inputFileName.toUtf8())) + return false; + + if (!writeStr("\n")) + return false; + + if (!writeStr(QByteArrayLiteral("namespace QmlCacheGeneratedCode {\nnamespace "))) + return false; + + if (!writeStr(symbolNamespaceForPath(inputFileName).toUtf8())) + return false; + + if (!writeStr(QByteArrayLiteral(" {\nextern const unsigned char qmlData alignas(16) [] = {\n"))) + return false; + + QByteArray hexifiedData; + { + QByteArray modifiedUnit; + modifiedUnit.resize(unit->data->unitSize); + memcpy(modifiedUnit.data(), unit->data, unit->data->unitSize); + const char *dataPtr = modifiedUnit.data(); + QV4::CompiledData::Unit *unitPtr; + memcpy(&unitPtr, &dataPtr, sizeof(unitPtr)); + unitPtr->flags |= QV4::CompiledData::Unit::StaticData; + + QTextStream stream(&hexifiedData); + const uchar *begin = reinterpret_cast(modifiedUnit.constData()); + const uchar *end = begin + unit->data->unitSize; + stream << hex; + int col = 0; + for (const uchar *data = begin; data < end; ++data, ++col) { + if (data > begin) + stream << ','; + if (col % 8 == 0) { + stream << '\n'; + col = 0; + } + stream << "0x" << *data; + } + stream << '\n'; + }; + + if (!writeStr(hexifiedData)) + return false; + + if (!writeStr("};\n}\n}\n")) + return false; + + if (!f.commit()) { + *errorString = f.errorString(); + return false; + } + + return true; +} + int main(int argc, char **argv) { // Produce reliably the same output for the same input by disabling QHash's random seeding. @@ -320,6 +403,15 @@ int main(int argc, char **argv) parser.addHelpOption(); parser.addVersionOption(); + QCommandLineOption filterResourceFileOption(QStringLiteral("filter-resource-file"), QCoreApplication::translate("main", "Filter out QML/JS files from a resource file that can be cached ahead of time instead")); + parser.addOption(filterResourceFileOption); + QCommandLineOption resourceFileMappingOption(QStringLiteral("resource-file-mapping"), QCoreApplication::translate("main", "Path from original resource file to new one"), QCoreApplication::translate("main", "old-name:new-name")); + parser.addOption(resourceFileMappingOption); + QCommandLineOption resourceOption(QStringLiteral("resource"), QCoreApplication::translate("main", "Qt resource file that might later contain one of the compiled files"), QCoreApplication::translate("main", "resource-file-name")); + parser.addOption(resourceOption); + QCommandLineOption resourcePathOption(QStringLiteral("resource-path"), QCoreApplication::translate("main", "Qt resource file path corresponding to the file being compiled"), QCoreApplication::translate("main", "resource-path")); + parser.addOption(resourcePathOption); + QCommandLineOption outputFileOption(QStringLiteral("o"), QCoreApplication::translate("main", "Output file name"), QCoreApplication::translate("main", "file name")); parser.addOption(outputFileOption); @@ -329,6 +421,8 @@ int main(int argc, char **argv) parser.addPositionalArgument(QStringLiteral("[qml file]"), QStringLiteral("QML source file to generate cache for.")); + parser.setSingleDashWordOptionMode(QCommandLineParser::ParseAsLongOptions); + parser.process(app); const QStringList sources = parser.positionalArguments(); @@ -346,13 +440,76 @@ int main(int argc, char **argv) if (parser.isSet(outputFileOption)) outputFileName = parser.value(outputFileOption); + if (parser.isSet(filterResourceFileOption)) { + return filterResourceFile(inputFile, outputFileName); + } + + if (outputFileName.endsWith(QLatin1String("qmlcache_loader.cpp"))) { + ResourceFileMapper mapper(inputFile); + + if (!generateLoader(mapper.qmlCompilerFiles(), outputFileName, parser.values(resourceFileMappingOption), &error.message)) { + error.augment(QLatin1String("Error generating loader stub: ")).print(); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; + } + + QString inputFileUrl = inputFile; + + SaveFunction saveFunction; + if (outputFileName.endsWith(QLatin1String(".cpp"))) { + ResourceFileMapper fileMapper(parser.values(resourceOption)); + QString inputResourcePath = parser.value(resourcePathOption); + + if (!inputResourcePath.isEmpty() && !fileMapper.isEmpty()) { + fprintf(stderr, "--%s and --%s are mutually exclusive.\n", + qPrintable(resourcePathOption.names().first()), + qPrintable(resourceOption.names().first())); + return EXIT_FAILURE; + } + + // If the user didn't specify the resource path corresponding to the file on disk being + // compiled, try to determine it from the resource file, if one was supplied. + if (inputResourcePath.isEmpty()) { + const QStringList resourcePaths = fileMapper.resourcePaths(inputFile); + if (resourcePaths.isEmpty()) { + fprintf(stderr, "No resource path for file: %s\n", qPrintable(inputFile)); + return EXIT_FAILURE; + } + + if (resourcePaths.size() != 1) { + fprintf(stderr, "Multiple resource paths for file %s. " + "Use the --%s option to disambiguate:\n", + qPrintable(inputFile), + qPrintable(resourcePathOption.names().first())); + for (const QString &resourcePath: resourcePaths) + fprintf(stderr, "\t%s\n", qPrintable(resourcePath)); + return EXIT_FAILURE; + } + + inputResourcePath = resourcePaths.first(); + } + + inputFileUrl = QStringLiteral("qrc://") + inputResourcePath; + + saveFunction = [inputResourcePath, outputFileName](QV4::CompiledData::CompilationUnit *unit, QString *errorString) { + return saveUnitAsCpp(inputResourcePath, outputFileName, unit, errorString); + }; + + } else { + saveFunction = [outputFileName](QV4::CompiledData::CompilationUnit *unit, QString *errorString) { + return unit->saveToDisk(outputFileName, errorString); + }; + } + + if (inputFile.endsWith(QLatin1String(".qml"))) { - if (!compileQmlFile(inputFile, outputFileName, &error)) { + if (!compileQmlFile(inputFile, saveFunction, &error)) { error.augment(QLatin1String("Error compiling qml file: ")).print(); return EXIT_FAILURE; } } else if (inputFile.endsWith(QLatin1String(".js"))) { - if (!compileJSFile(inputFile, outputFileName, &error)) { + if (!compileJSFile(inputFile, inputFileUrl, saveFunction, &error)) { error.augment(QLatin1String("Error compiling qml file: ")).print(); return EXIT_FAILURE; } diff --git a/tools/qmlcachegen/qmlcachegen.pro b/tools/qmlcachegen/qmlcachegen.pro index be92ef435e..1b76833b5a 100644 --- a/tools/qmlcachegen/qmlcachegen.pro +++ b/tools/qmlcachegen/qmlcachegen.pro @@ -3,10 +3,13 @@ option(host_build) QT = qmldevtools-private DEFINES += QT_NO_CAST_TO_ASCII QT_NO_CAST_FROM_ASCII -SOURCES = qmlcachegen.cpp +SOURCES = qmlcachegen.cpp \ + resourcefilter.cpp \ + generateloader.cpp \ + resourcefilemapper.cpp TARGET = qmlcachegen -build_integration.files = qmlcache.prf +build_integration.files = qmlcache.prf qtquickcompiler.prf build_integration.path = $$[QT_HOST_DATA]/mkspecs/features prefix_build: INSTALLS += build_integration else: COPIES += build_integration @@ -14,3 +17,6 @@ else: COPIES += build_integration QMAKE_TARGET_DESCRIPTION = QML Cache Generator load(qt_tool) + +HEADERS += \ + resourcefilemapper.h diff --git a/tools/qmlcachegen/qtquickcompiler.prf b/tools/qmlcachegen/qtquickcompiler.prf new file mode 100644 index 0000000000..985a0a5cdc --- /dev/null +++ b/tools/qmlcachegen/qtquickcompiler.prf @@ -0,0 +1,92 @@ + +qtPrepareTool(QML_CACHEGEN, qmlcachegen, _FILTER) +qtPrepareTool(QMAKE_RCC, rcc, _DEP) + +QMLCACHE_DIR = .qmlcache + +defineReplace(qmlCacheResourceFileOutputName) { + name = $$relative_path($$1, $$_PRO_FILE_PWD_) + name = $$replace(name,/,_) + name = $$replace(name, \\.qrc$, _qmlcache.qrc) + name = $$replace(name,\.\.,) + name = $$replace(name,-,_) + name = $$absolute_path($$name, $$OUT_PWD) + return($${name}) +} + +# Flatten RESOURCES that may contain individual files or objects +load(resources) + +NEWRESOURCES = +QMLCACHE_RESOURCE_FILES = + +for(res, RESOURCES) { + absRes = $$absolute_path($$res, $$_PRO_FILE_PWD_) + rccContents = $$system($$QMAKE_RCC_DEP -list $$absRes,lines) + contains(rccContents,.*\\.js$)|contains(rccContents,.*\\.qml$) { + new_resource = $$qmlCacheResourceFileOutputName($$res) + mkpath($$dirname(new_resource)) + remaining_files = $$system($$QML_CACHEGEN_FILTER -filter-resource-file -o $$new_resource $$absRes,lines) + !isEmpty(remaining_files) { + NEWRESOURCES += $$new_resource + QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$absRes=$$new_resource + } else { + QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$absRes + } + + QMLCACHE_RESOURCE_FILES += $$absRes + + for(candidate, $$list($$rccContents)) { + contains(candidate,.*\\.js$)|contains(candidate,.*\\.qml$) { + QMLCACHE_FILES += $$candidate + } + } + } else { + NEWRESOURCES += $$res + } +} + +RESOURCES = $$NEWRESOURCES + +QMLCACHE_RESOURCE_FLAGS = +for(res, QMLCACHE_RESOURCE_FILES) { + QMLCACHE_RESOURCE_FLAGS += --resource=$$res +} + +!isEmpty(QMLCACHE_FILES) { + QT_PRIVATE *= core-private qml-private +} + +defineReplace(qmlCacheOutputName) { + name = $$absolute_path($$1, $$OUT_PWD) + name = $$relative_path($$name, $$_PRO_FILE_PWD_) + name = $$replace(name, \\.qml$, _qml) + name = $$replace(name, \\.js$, _js) + name = $$replace(name,/,_) + name = $$QMLCACHE_DIR/$${name} + return($${name}) +} + +qmlcache.input = QMLCACHE_FILES +qmlcache.output = ${QMAKE_FUNC_FILE_IN_qmlCacheOutputName}$${first(QMAKE_EXT_CPP)} +qmlcache.commands = $$QML_CACHEGEN $$QMLCACHE_RESOURCE_FLAGS $$QMLCACHE_FLAGS -o ${QMAKE_FILE_OUT} ${QMAKE_FILE_IN} +qmlcache.name = qmlcachegen ${QMAKE_FILE_IN} +qmlcache.variable_out = GENERATED_SOURCES +qmlcache.dependency_type = TYPE_C + +qmlcache_loader.input = QMLCACHE_RESOURCE_FILES +qmlcache_loader.output = $$QMLCACHE_DIR/qmlcache_loader.cpp +qmlcache_loader.commands = $$QML_CACHEGEN $$QMLCACHE_LOADER_FLAGS $$QMLCACHE_FLAGS -o ${QMAKE_FILE_OUT} ${QMAKE_FILE_IN} +qmlcache_loader.name = qmlcachengen_loader ${QMAKE_FILE_IN} +qmlcache_loader.variable_out = SOURCES +qmlcache_loader.dependency_type = TYPE_C +qmlcache_loader.CONFIG = combine + +unix:!no_qmlcache_depend { + compiler_bin = $$[QT_HOST_BINS]/qmlcachegen + contains(QMAKE_HOST.os, Windows): compiler_bin = $${compiler_bin}.exe + qmlcache.depends += $$compiler_bin + qmlcache_loader.depends += $$compiler_bin +} + +QMAKE_EXTRA_COMPILERS += qmlcache qmlcache_loader diff --git a/tools/qmlcachegen/resourcefilemapper.cpp b/tools/qmlcachegen/resourcefilemapper.cpp new file mode 100644 index 0000000000..d7daf33de8 --- /dev/null +++ b/tools/qmlcachegen/resourcefilemapper.cpp @@ -0,0 +1,177 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtQml module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "resourcefilemapper.h" + +#include +#include +#include + +ResourceFileMapper::ResourceFileMapper(const QStringList &resourceFiles) +{ + for (const QString &fileName: resourceFiles) { + QFile f(fileName); + if (!f.open(QIODevice::ReadOnly)) + continue; + populateFromQrcFile(f); + } +} + +ResourceFileMapper::ResourceFileMapper(const QString &resourceFile) +{ + QFile f(resourceFile); + if (!f.open(QIODevice::ReadOnly)) + return; + populateFromQrcFile(f); +} + +bool ResourceFileMapper::isEmpty() const +{ + return qrcPathToFileSystemPath.isEmpty(); +} + +QStringList ResourceFileMapper::resourcePaths(const QString &fileName) +{ + const QString absPath = QDir::cleanPath(QDir::current().absoluteFilePath(fileName)); + QHashIterator it(qrcPathToFileSystemPath); + QStringList resourcePaths; + while (it.hasNext()) { + it.next(); + if (QFileInfo(it.value()) == QFileInfo(absPath)) + resourcePaths.append(it.key()); + } + return resourcePaths; +} + +QStringList ResourceFileMapper::qmlCompilerFiles() const +{ + QStringList files; + for (auto it = qrcPathToFileSystemPath.constBegin(), end = qrcPathToFileSystemPath.constEnd(); + it != end; ++it) { + const QString &qrcPath = it.key(); + const QString suffix = QFileInfo(qrcPath).suffix(); + if (suffix != QStringLiteral("qml") && suffix != QStringLiteral("js")) + continue; + files << qrcPath; + } + return files; +} + +void ResourceFileMapper::populateFromQrcFile(QFile &file) +{ + enum State { + InitialState, + InRCC, + InResource, + InFile + }; + State state = InitialState; + + QDir qrcDir = QFileInfo(file).absoluteDir(); + + QString prefix; + QString currentFileName; + QXmlStreamAttributes currentFileAttributes; + + QXmlStreamReader reader(&file); + while (!reader.atEnd()) { + switch (reader.readNext()) { + case QXmlStreamReader::StartElement: + if (reader.name() == QStringLiteral("RCC")) { + if (state != InitialState) + return; + state = InRCC; + continue; + } else if (reader.name() == QStringLiteral("qresource")) { + if (state != InRCC) + return; + state = InResource; + QXmlStreamAttributes attributes = reader.attributes(); + if (attributes.hasAttribute(QStringLiteral("prefix"))) + prefix = attributes.value(QStringLiteral("prefix")).toString(); + if (!prefix.startsWith(QLatin1Char('/'))) + prefix.prepend(QLatin1Char('/')); + if (!prefix.endsWith(QLatin1Char('/'))) + prefix.append(QLatin1Char('/')); + continue; + } else if (reader.name() == QStringLiteral("file")) { + if (state != InResource) + return; + state = InFile; + currentFileAttributes = reader.attributes(); + continue; + } + return; + + case QXmlStreamReader::EndElement: + if (reader.name() == QStringLiteral("file")) { + if (state != InFile) + return; + state = InResource; + continue; + } else if (reader.name() == QStringLiteral("qresource")) { + if (state != InResource) + return; + state = InRCC; + continue; + } else if (reader.name() == QStringLiteral("RCC")) { + if (state != InRCC) + return; + state = InitialState; + continue; + } + return; + + case QXmlStreamReader::Characters: { + if (reader.isWhitespace()) + break; + if (state != InFile) + return; + currentFileName = reader.text().toString(); + if (currentFileName.isEmpty()) + continue; + + const QString fsPath = QDir::cleanPath(qrcDir.absoluteFilePath(currentFileName)); + + if (currentFileAttributes.hasAttribute(QStringLiteral("alias"))) + currentFileName = currentFileAttributes.value(QStringLiteral("alias")).toString(); + + currentFileName = QDir::cleanPath(currentFileName); + while (currentFileName.startsWith(QLatin1String("../"))) + currentFileName.remove(0, 3); + + const QString qrcPath = prefix + currentFileName; + if (QFile::exists(fsPath)) + qrcPathToFileSystemPath.insert(qrcPath, fsPath); + continue; + } + + default: break; + } + } +} diff --git a/tools/qmlcachegen/resourcefilemapper.h b/tools/qmlcachegen/resourcefilemapper.h new file mode 100644 index 0000000000..f6e82bacb0 --- /dev/null +++ b/tools/qmlcachegen/resourcefilemapper.h @@ -0,0 +1,51 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtQml module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef RESOURCEFILEMAPPER_H +#define RESOURCEFILEMAPPER_H + +#include +#include +#include + +struct ResourceFileMapper +{ + ResourceFileMapper(const QStringList &resourceFiles); + ResourceFileMapper(const QString &resourceFile); + + bool isEmpty() const; + + QStringList resourcePaths(const QString &fileName); + QStringList qmlCompilerFiles() const; + +private: + void populateFromQrcFile(QFile &file); + + QHash qrcPathToFileSystemPath; +}; + +#endif // RESOURCEFILEMAPPER_H diff --git a/tools/qmlcachegen/resourcefilter.cpp b/tools/qmlcachegen/resourcefilter.cpp new file mode 100644 index 0000000000..196dbd4a75 --- /dev/null +++ b/tools/qmlcachegen/resourcefilter.cpp @@ -0,0 +1,183 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtQml module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL-EXCEPT$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 as published by the Free Software +** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include +#include +#include +#include + +int filterResourceFile(const QString &input, const QString &output) +{ + enum State { + InitialState, + InRCC, + InResource, + InFile + }; + State state = InitialState; + + QString prefix; + QString currentFileName; + QXmlStreamAttributes fileAttributes; + + QFile file(input); + if (!file.open(QIODevice::ReadOnly)) { + fprintf(stderr, "Cannot open %s for reading.\n", qPrintable(input)); + return EXIT_FAILURE; + } + + QDir inputDirectory = QFileInfo(file).absoluteDir(); + QDir outputDirectory = QFileInfo(output).absoluteDir(); + + QString outputString; + QXmlStreamWriter writer(&outputString); + writer.setAutoFormatting(true); + + QStringList remainingFiles; + + QXmlStreamReader reader(&file); + while (!reader.atEnd()) { + switch (reader.readNext()) { + case QXmlStreamReader::StartDocument: { + QStringRef version = reader.documentVersion(); + if (!version.isEmpty()) + writer.writeStartDocument(version.toString()); + else + writer.writeStartDocument(); + break; + } + case QXmlStreamReader::EndDocument: + writer.writeEndDocument(); + break; + case QXmlStreamReader::StartElement: + if (reader.name() == QStringLiteral("RCC")) { + if (state != InitialState) { + fprintf(stderr, "Unexpected RCC tag in line %d\n", int(reader.lineNumber())); + return EXIT_FAILURE; + } + state = InRCC; + } else if (reader.name() == QStringLiteral("qresource")) { + if (state != InRCC) { + fprintf(stderr, "Unexpected qresource tag in line %d\n", int(reader.lineNumber())); + return EXIT_FAILURE; + } + state = InResource; + QXmlStreamAttributes attributes = reader.attributes(); + if (attributes.hasAttribute(QStringLiteral("prefix"))) + prefix = attributes.value(QStringLiteral("prefix")).toString(); + if (!prefix.startsWith(QLatin1Char('/'))) + prefix.prepend(QLatin1Char('/')); + if (!prefix.endsWith(QLatin1Char('/'))) + prefix.append(QLatin1Char('/')); + } else if (reader.name() == QStringLiteral("file")) { + if (state != InResource) { + fprintf(stderr, "Unexpected file tag in line %d\n", int(reader.lineNumber())); + return EXIT_FAILURE; + } + state = InFile; + fileAttributes = reader.attributes(); + continue; + } + writer.writeStartElement(reader.name().toString()); + writer.writeAttributes(reader.attributes()); + continue; + + case QXmlStreamReader::EndElement: + if (reader.name() == QStringLiteral("file")) { + if (state != InFile) { + fprintf(stderr, "Unexpected end of file tag in line %d\n", int(reader.lineNumber())); + return EXIT_FAILURE; + } + state = InResource; + continue; + } else if (reader.name() == QStringLiteral("qresource")) { + if (state != InResource) { + fprintf(stderr, "Unexpected end of qresource tag in line %d\n", int(reader.lineNumber())); + return EXIT_FAILURE; + } + state = InRCC; + } else if (reader.name() == QStringLiteral("RCC")) { + if (state != InRCC) { + fprintf(stderr, "Unexpected end of RCC tag in line %d\n", int(reader.lineNumber())); + return EXIT_FAILURE; + } + state = InitialState; + } + writer.writeEndElement(); + continue; + + case QXmlStreamReader::Characters: + if (reader.isWhitespace()) + break; + if (state != InFile) + return EXIT_FAILURE; + currentFileName = reader.text().toString(); + if (currentFileName.isEmpty()) + continue; + + if (!currentFileName.endsWith(QStringLiteral(".qml")) && !currentFileName.endsWith(QStringLiteral(".js"))) { + writer.writeStartElement(QStringLiteral("file")); + + if (!fileAttributes.hasAttribute(QStringLiteral("alias"))) + fileAttributes.append(QStringLiteral("alias"), currentFileName); + + currentFileName = inputDirectory.absoluteFilePath(currentFileName); + currentFileName = outputDirectory.relativeFilePath(currentFileName); + + remainingFiles << currentFileName; + + writer.writeAttributes(fileAttributes); + writer.writeCharacters(currentFileName); + writer.writeEndElement(); + } + continue; + + default: break; + } + } + + if (!remainingFiles.isEmpty()) { + QFile outputFile(output); + if (!outputFile.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + fprintf(stderr, "Cannot open %s for writing.\n", qPrintable(output)); + return EXIT_FAILURE; + } + const QByteArray outputStringUtf8 = outputString.toUtf8(); + if (outputFile.write(outputStringUtf8) != outputStringUtf8.size()) + return EXIT_FAILURE; + + outputFile.close(); + if (outputFile.error() != QFileDevice::NoError) + return EXIT_FAILURE; + + // The build system expects this output if we wrote a qrc file and no output + // if no files remain. + fprintf(stdout, "New resource file written with %d files.\n", remainingFiles.count()); + } + + return EXIT_SUCCESS; +} -- cgit v1.2.3