diff options
author | Simon Hausmann <simon.hausmann@qt.io> | 2018-07-27 11:47:53 +0200 |
---|---|---|
committer | Simon Hausmann <simon.hausmann@qt.io> | 2018-08-01 09:42:06 +0000 |
commit | e99038b24d8b9f4a8cc503d1e3f789a93a3e8330 (patch) | |
tree | 05a43cf4833f5eba65f54e7fbb61fa49575ba14d /src/qml/compiler | |
parent | 73fc89bb0dbe9a866e7b1cf47a42a85e5042fead (diff) |
Reduce memory consumption when loading AOT generated cache files
Separate the qml data (objects/imports) from the general compilation
unit data. It's only the former that needs to be re-generated as part of
the type re-compilation and by separating it we can allocate memory just
for that and keep using the mmap'ed general unit data for everything
else (including byte code).
Another upside of this change is that it allows eliminating the recently
introduced concept of a backing unit again.
Saves ~149K RAM with the QQC1 gallery.
Task-number: QTBUG-69588
Change-Id: Ie88a4286feb7e2f472f58a28fa5dd6ff0a91c4b6
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
Diffstat (limited to 'src/qml/compiler')
-rw-r--r-- | src/qml/compiler/qqmlirbuilder.cpp | 156 | ||||
-rw-r--r-- | src/qml/compiler/qqmltypecompiler.cpp | 6 | ||||
-rw-r--r-- | src/qml/compiler/qv4compileddata.cpp | 22 | ||||
-rw-r--r-- | src/qml/compiler/qv4compileddata_p.h | 73 | ||||
-rw-r--r-- | src/qml/compiler/qv4compiler.cpp | 5 |
5 files changed, 137 insertions, 125 deletions
diff --git a/src/qml/compiler/qqmlirbuilder.cpp b/src/qml/compiler/qqmlirbuilder.cpp index e544559e03..e9685597f6 100644 --- a/src/qml/compiler/qqmlirbuilder.cpp +++ b/src/qml/compiler/qqmlirbuilder.cpp @@ -1553,34 +1553,70 @@ bool IRBuilder::isRedundantNullInitializerForPropertyDeclaration(Property *prope void QmlUnitGenerator::generate(Document &output, const QV4::CompiledData::DependentTypesHasher &dependencyHasher) { + output.jsGenerator.stringTable.registerString(output.jsModule.fileName); + output.jsGenerator.stringTable.registerString(output.jsModule.finalUrl); + QQmlRefPointer<QV4::CompiledData::CompilationUnit> compilationUnit = output.javaScriptCompilationUnit; const QV4::CompiledData::Unit *jsUnit = nullptr; - uint jsUnitSize = 0; + std::function<QV4::CompiledData::QmlUnit *(QV4::CompiledData::QmlUnit *, uint)> unitFinalizer + = [](QV4::CompiledData::QmlUnit *unit, uint) { + return unit; + }; // We may already have unit data if we're loading an ahead-of-time generated cache file. if (compilationUnit->data) { - jsUnit = const_cast<QV4::CompiledData::Unit*>(compilationUnit->data); - // Discard the old QML tables we will re-create further down anyway. - quint32 unitSizeWithoutQMLTables = jsUnit->offsetToImports; - jsUnitSize = unitSizeWithoutQMLTables; + jsUnit = compilationUnit->data; +#ifndef V4_BOOTSTRAP + output.javaScriptCompilationUnit->dynamicStrings = output.jsGenerator.stringTable.allStrings(); +#endif } else { - jsUnit = output.jsGenerator.generateUnit(QV4::Compiler::JSUnitGenerator::GenerateWithoutStringTable); - jsUnitSize = jsUnit->unitSize; - } - - output.jsGenerator.stringTable.registerString(output.jsModule.fileName); - output.jsGenerator.stringTable.registerString(output.jsModule.finalUrl); + QV4::CompiledData::Unit *createdUnit; + jsUnit = createdUnit = output.jsGenerator.generateUnit(); - // No more new strings after this point, we're calculating offsets. - output.jsGenerator.stringTable.freeze(); + // enable flag if we encountered pragma Singleton + for (Pragma *p : qAsConst(output.pragmas)) { + if (p->type == Pragma::PragmaSingleton) { + createdUnit->flags |= QV4::CompiledData::Unit::IsSingleton; + break; + } + } + // This unit's memory was allocated with malloc on the heap, so it's + // definitely not suitable for StaticData access. + createdUnit->flags &= ~QV4::CompiledData::Unit::StaticData; - const bool writeStringTable = #ifndef V4_BOOTSTRAP - output.javaScriptCompilationUnit->backingUnit == nullptr; + if (dependencyHasher) { + QCryptographicHash hash(QCryptographicHash::Md5); + if (dependencyHasher(&hash)) { + QByteArray checksum = hash.result(); + Q_ASSERT(checksum.size() == sizeof(createdUnit->dependencyMD5Checksum)); + memcpy(createdUnit->dependencyMD5Checksum, checksum.constData(), sizeof(createdUnit->dependencyMD5Checksum)); + } + } #else - true; + Q_UNUSED(dependencyHasher); #endif + createdUnit->sourceFileIndex = output.jsGenerator.stringTable.getStringId(output.jsModule.fileName); + createdUnit->finalUrlIndex = output.jsGenerator.stringTable.getStringId(output.jsModule.finalUrl); + + // Combine the qml data into the general unit data. + unitFinalizer = [&jsUnit](QV4::CompiledData::QmlUnit *qmlUnit, uint qmlDataSize) { + void *ptr = const_cast<QV4::CompiledData::Unit*>(jsUnit); + QV4::CompiledData::Unit *newUnit = (QV4::CompiledData::Unit *)realloc(ptr, jsUnit->unitSize + qmlDataSize); + jsUnit = newUnit; + newUnit->offsetToQmlUnit = newUnit->unitSize; + newUnit->unitSize += qmlDataSize; + memcpy(const_cast<QV4::CompiledData::QmlUnit *>(newUnit->qmlUnit()), qmlUnit, qmlDataSize); + free(const_cast<QV4::CompiledData::QmlUnit*>(qmlUnit)); + qmlUnit = nullptr; + newUnit->generateChecksum(); + return const_cast<QV4::CompiledData::QmlUnit*>(newUnit->qmlUnit()); + }; + } + + // No more new strings after this point, we're calculating offsets. + output.jsGenerator.stringTable.freeze(); const int importSize = sizeof(QV4::CompiledData::Import) * output.imports.count(); const int objectOffsetTableSize = output.objects.count() * sizeof(quint32); @@ -1589,7 +1625,7 @@ void QmlUnitGenerator::generate(Document &output, const QV4::CompiledData::Depen int objectsSize = 0; for (Object *o : qAsConst(output.objects)) { - objectOffsets.insert(o, jsUnitSize + importSize + objectOffsetTableSize + objectsSize); + objectOffsets.insert(o, sizeof(QV4::CompiledData::QmlUnit) + importSize + objectOffsetTableSize + objectsSize); objectsSize += QV4::CompiledData::Object::calculateSizeExcludingSignalsAndEnums(o->functionCount(), o->propertyCount(), o->aliasCount(), o->enumCount(), o->signalCount(), o->bindingCount(), o->namedObjectsInComponent.count); int signalTableSize = 0; @@ -1605,43 +1641,14 @@ void QmlUnitGenerator::generate(Document &output, const QV4::CompiledData::Depen objectsSize += enumTableSize; } - const int totalSize = jsUnitSize + importSize + objectOffsetTableSize + objectsSize + (writeStringTable ? output.jsGenerator.stringTable.sizeOfTableAndData() : 0); + const uint totalSize = sizeof(QV4::CompiledData::QmlUnit) + importSize + objectOffsetTableSize + objectsSize; char *data = (char*)malloc(totalSize); - memcpy(data, jsUnit, jsUnitSize); - memset(data + jsUnitSize, 0, totalSize - jsUnitSize); - jsUnit = nullptr; - - QV4::CompiledData::Unit *qmlUnit = reinterpret_cast<QV4::CompiledData::Unit *>(data); - qmlUnit->unitSize = totalSize; - // This unit's memory was allocated with malloc on the heap, so it's - // definitely not suitable for StaticData access. - qmlUnit->flags &= ~QV4::CompiledData::Unit::StaticData; - qmlUnit->offsetToImports = jsUnitSize; + memset(data, 0, totalSize); + QV4::CompiledData::QmlUnit *qmlUnit = reinterpret_cast<QV4::CompiledData::QmlUnit *>(data); + qmlUnit->offsetToImports = sizeof(*qmlUnit); qmlUnit->nImports = output.imports.count(); - qmlUnit->offsetToObjects = jsUnitSize + importSize; + qmlUnit->offsetToObjects = qmlUnit->offsetToImports + importSize; qmlUnit->nObjects = output.objects.count(); - if (writeStringTable) { - qmlUnit->offsetToStringTable = totalSize - output.jsGenerator.stringTable.sizeOfTableAndData(); - qmlUnit->stringTableSize = output.jsGenerator.stringTable.stringCount(); - } else { - qmlUnit->offsetToStringTable = 0; - qmlUnit->stringTableSize = 0; - } - qmlUnit->sourceFileIndex = output.jsGenerator.stringTable.getStringId(output.jsModule.fileName); - qmlUnit->finalUrlIndex = output.jsGenerator.stringTable.getStringId(output.jsModule.finalUrl); - -#ifndef V4_BOOTSTRAP - if (dependencyHasher) { - QCryptographicHash hash(QCryptographicHash::Md5); - if (dependencyHasher(&hash)) { - QByteArray checksum = hash.result(); - Q_ASSERT(checksum.size() == sizeof(qmlUnit->dependencyMD5Checksum)); - memcpy(qmlUnit->dependencyMD5Checksum, checksum.constData(), sizeof(qmlUnit->dependencyMD5Checksum)); - } - } -#else - Q_UNUSED(dependencyHasher); -#endif // write imports char *importPtr = data + qmlUnit->offsetToImports; @@ -1775,46 +1782,31 @@ void QmlUnitGenerator::generate(Document &output, const QV4::CompiledData::Depen objectPtr += enumTableSize; } - // enable flag if we encountered pragma Singleton - for (Pragma *p : qAsConst(output.pragmas)) { - if (p->type == Pragma::PragmaSingleton) { - qmlUnit->flags |= QV4::CompiledData::Unit::IsSingleton; - break; - } - } - - if (writeStringTable) - output.jsGenerator.stringTable.serialize(qmlUnit); -#ifndef V4_BOOTSTRAP - else - output.javaScriptCompilationUnit->dynamicStrings = output.jsGenerator.stringTable.allStrings(); -#endif - - qmlUnit->generateChecksum(); + qmlUnit = unitFinalizer(qmlUnit, totalSize); static const bool showStats = qEnvironmentVariableIsSet("QML_SHOW_UNIT_STATS"); if (showStats) { - qDebug() << "Generated QML unit that is" << qmlUnit->unitSize << "bytes big contains:"; - qDebug() << " " << qmlUnit->functionTableSize << "functions"; - qDebug() << " " << jsUnitSize << "for JS unit"; + qDebug() << "Generated QML unit that is" << totalSize << "bytes big contains:"; + qDebug() << " " << jsUnit->functionTableSize << "functions"; + qDebug() << " " << jsUnit->unitSize << "for JS unit"; qDebug() << " " << importSize << "for imports"; qDebug() << " " << objectsSize << "for" << qmlUnit->nObjects << "objects"; quint32 totalBindingCount = 0; for (quint32 i = 0; i < qmlUnit->nObjects; ++i) - totalBindingCount += qmlUnit->objectAtInternal(i)->nBindings; + totalBindingCount += qmlUnit->objectAt(i)->nBindings; qDebug() << " " << totalBindingCount << "bindings"; quint32 totalCodeSize = 0; - for (quint32 i = 0; i < qmlUnit->functionTableSize; ++i) - totalCodeSize += qmlUnit->functionAt(i)->codeSize; + for (quint32 i = 0; i < jsUnit->functionTableSize; ++i) + totalCodeSize += jsUnit->functionAt(i)->codeSize; qDebug() << " " << totalCodeSize << "bytes total byte code"; - qDebug() << " " << qmlUnit->stringTableSize << "strings"; + qDebug() << " " << jsUnit->stringTableSize << "strings"; quint32 totalStringSize = 0; - for (quint32 i = 0; i < qmlUnit->stringTableSize; ++i) - totalStringSize += QV4::CompiledData::String::calculateSize(qmlUnit->stringAtInternal(i)); + for (quint32 i = 0; i < jsUnit->stringTableSize; ++i) + totalStringSize += QV4::CompiledData::String::calculateSize(jsUnit->stringAtInternal(i)); qDebug() << " " << totalStringSize << "bytes total strings"; } - compilationUnit->setUnitData(qmlUnit); + compilationUnit->setUnitData(jsUnit, qmlUnit, output.jsModule.fileName, output.jsModule.finalUrl); } char *QmlUnitGenerator::writeBindings(char *bindingPtr, const Object *o, BindingFilter filter) const @@ -2388,8 +2380,10 @@ void IRLoader::load() { output->jsGenerator.stringTable.initializeFromBackingUnit(unit); - for (quint32 i = 0; i < unit->nImports; ++i) - output->imports << unit->importAtInternal(i); + const QV4::CompiledData::QmlUnit *qmlUnit = unit->qmlUnit(); + + for (quint32 i = 0; i < qmlUnit->nImports; ++i) + output->imports << qmlUnit->importAt(i); if (unit->flags & QV4::CompiledData::Unit::IsSingleton) { QmlIR::Pragma *p = New<QmlIR::Pragma>(); @@ -2398,8 +2392,8 @@ void IRLoader::load() output->pragmas << p; } - for (uint i = 0; i < unit->nObjects; ++i) { - const QV4::CompiledData::Object *serializedObject = unit->objectAtInternal(i); + for (uint i = 0; i < qmlUnit->nObjects; ++i) { + const QV4::CompiledData::Object *serializedObject = qmlUnit->objectAt(i); QmlIR::Object *object = loadObject(serializedObject); output->objects.append(object); } diff --git a/src/qml/compiler/qqmltypecompiler.cpp b/src/qml/compiler/qqmltypecompiler.cpp index 44ffc864a8..1695f3e5d6 100644 --- a/src/qml/compiler/qqmltypecompiler.cpp +++ b/src/qml/compiler/qqmltypecompiler.cpp @@ -133,11 +133,7 @@ QQmlRefPointer<QV4::CompiledData::CompilationUnit> QQmlTypeCompiler::compile() return nullptr; } - if (document->javaScriptCompilationUnit) { - // If this file was loaded from an ahead-of-time built cache file, then set up the original - // unit data as backing unit and fetch strings from there to save memory. - document->javaScriptCompilationUnit->backingUnit = document->javaScriptCompilationUnit->unitData(); - } else { + if (!document->javaScriptCompilationUnit) { // Compile JS binding expressions and signal handlers if necessary { // We can compile script strings ahead of time, but they must be compiled diff --git a/src/qml/compiler/qv4compileddata.cpp b/src/qml/compiler/qv4compileddata.cpp index 767e8dfeb3..8289d92eb1 100644 --- a/src/qml/compiler/qv4compileddata.cpp +++ b/src/qml/compiler/qv4compileddata.cpp @@ -99,8 +99,15 @@ CompilationUnit::CompilationUnit(const Unit *unitData) CompilationUnit::~CompilationUnit() { unlink(); - if (data && !(data->flags & QV4::CompiledData::Unit::StaticData)) - free(const_cast<Unit *>(data)); + + if (data) { + if (data->qmlUnit() != qmlData) + free(const_cast<QmlUnit *>(qmlData)); + qmlData = nullptr; + + if (!(data->flags & QV4::CompiledData::Unit::StaticData)) + free(const_cast<Unit *>(data)); + } data = nullptr; #if Q_BYTE_ORDER == Q_BIG_ENDIAN delete [] constants; @@ -463,16 +470,22 @@ bool CompilationUnit::saveToDisk(const QUrl &unitUrl, QString *errorString) #endif // QT_CONFIG(temporaryfile) } -void CompilationUnit::setUnitData(const Unit *unitData) +void CompilationUnit::setUnitData(const Unit *unitData, const QmlUnit *qmlUnit, + const QString &fileName, const QString &finalUrlString) { data = unitData; + qmlData = nullptr; #if Q_BYTE_ORDER == Q_BIG_ENDIAN delete [] constants; #endif constants = nullptr; + m_fileName.clear(); + m_finalUrlString.clear(); if (!data) return; + qmlData = qmlUnit ? qmlUnit : data->qmlUnit(); + #if Q_BYTE_ORDER == Q_BIG_ENDIAN Value *bigEndianConstants = new Value[data->constantTableSize]; const quint64_le *littleEndianConstants = data->constants(); @@ -482,6 +495,9 @@ void CompilationUnit::setUnitData(const Unit *unitData) #else constants = reinterpret_cast<const Value*>(data->constants()); #endif + + m_fileName = !fileName.isEmpty() ? fileName : stringAt(data->sourceFileIndex); + m_finalUrlString = !finalUrlString.isEmpty() ? finalUrlString : stringAt(data->finalUrlIndex); } #ifndef V4_BOOTSTRAP diff --git a/src/qml/compiler/qv4compileddata_p.h b/src/qml/compiler/qv4compileddata_p.h index 54c7a242de..c3cc38e193 100644 --- a/src/qml/compiler/qv4compileddata_p.h +++ b/src/qml/compiler/qv4compileddata_p.h @@ -753,6 +753,25 @@ struct Import }; static_assert(sizeof(Import) == 24, "Import structure needs to have the expected size to be binary compatible on disk when generated by host compiler and loaded by target"); +struct QmlUnit +{ + quint32_le nImports; + quint32_le offsetToImports; + quint32_le nObjects; + quint32_le offsetToObjects; + + const Import *importAt(int idx) const { + return reinterpret_cast<const Import*>((reinterpret_cast<const char *>(this)) + offsetToImports + idx * sizeof(Import)); + } + + const Object *objectAt(int idx) const { + const quint32_le *offsetTable = reinterpret_cast<const quint32_le*>((reinterpret_cast<const char *>(this)) + offsetToObjects); + const quint32_le offset = offsetTable[idx]; + return reinterpret_cast<const Object*>(reinterpret_cast<const char*>(this) + offset); + } +}; +static_assert(sizeof(QmlUnit) == 16, "QmlUnit structure needs to have the expected size to be binary compatible on disk when generated by host compiler and loaded by target"); + enum { QmlCompileHashSpace = 48 }; static const char magic_str[] = "qv4cdata"; extern const char qml_compile_hash[QmlCompileHashSpace + 1]; @@ -804,24 +823,14 @@ struct Unit quint32_le sourceFileIndex; quint32_le finalUrlIndex; - /* QML specific fields */ - quint32_le nImports; - quint32_le offsetToImports; - quint32_le nObjects; - quint32_le offsetToObjects; - - quint32_le padding; + quint32_le offsetToQmlUnit; bool verifyHeader(QDateTime expectedSourceTimeStamp, QString *errorString) const; - const Import *importAtInternal(int idx) const { - return reinterpret_cast<const Import*>((reinterpret_cast<const char *>(this)) + offsetToImports + idx * sizeof(Import)); - } + /* QML specific fields */ - const Object *objectAtInternal(int idx) const { - const quint32_le *offsetTable = reinterpret_cast<const quint32_le*>((reinterpret_cast<const char *>(this)) + offsetToObjects); - const quint32_le offset = offsetTable[idx]; - return reinterpret_cast<const Object*>(reinterpret_cast<const char*>(this) + offset); + const QmlUnit *qmlUnit() const { + return reinterpret_cast<const QmlUnit *>(reinterpret_cast<const char *>(this) + offsetToQmlUnit); } bool isSingleton() const { @@ -897,7 +906,7 @@ struct Unit } }; -static_assert(sizeof(Unit) == 216, "Unit structure needs to have the expected size to be binary compatible on disk when generated by host compiler and loaded by target"); +static_assert(sizeof(Unit) == 200, "Unit structure needs to have the expected size to be binary compatible on disk when generated by host compiler and loaded by target"); struct TypeReference { @@ -992,6 +1001,7 @@ Q_STATIC_ASSERT(offsetof(CompilationUnitBase, runtimeRegularExpressions) == offs struct Q_QML_PRIVATE_EXPORT CompilationUnit final : public CompilationUnitBase { const Unit *data = nullptr; + const QmlUnit *qmlData = nullptr; public: CompilationUnit(const Unit *unitData = nullptr); #ifdef V4_BOOTSTRAP @@ -1018,7 +1028,8 @@ public: } const Unit *unitData() const { return data; } - void setUnitData(const Unit *unitData); + void setUnitData(const Unit *unitData, const QmlUnit *qmlUnit = nullptr, + const QString &fileName = QString(), const QString &finalUrlString = QString()); #ifndef V4_BOOTSTRAP QIntrusiveListNode nextCompilationUnit; @@ -1032,8 +1043,8 @@ public: // finalUrl() and finalUrlString() shall be used to resolve further URLs referred to in the code // They are _not_ intercepted and thus represent the "logical" name for the code. - QString fileName() const { return stringAt(data->sourceFileIndex); } - QString finalUrlString() const { return stringAt(data->finalUrlIndex); } + QString fileName() const { return m_fileName; } + QString finalUrlString() const { return m_finalUrlString; } QUrl url() const { if (m_url.isNull) m_url = QUrl(fileName()); return m_url; } QUrl finalUrl() const { @@ -1080,25 +1091,18 @@ public: bool isRegisteredWithEngine = false; QScopedPointer<CompilationUnitMapper> backingFile; - const QV4::CompiledData::Unit *backingUnit = nullptr; QStringList dynamicStrings; // --- interface for QQmlPropertyCacheCreator typedef Object CompiledObject; - int objectCount() const { return data->nObjects; } - const Object *objectAt(int index) const { return data->objectAtInternal(index); } - int importCount() const { return data->nImports; } - const Import *importAt(int index) const { return data->importAtInternal(index); } + int objectCount() const { return qmlData->nObjects; } + const Object *objectAt(int index) const { return qmlData->objectAt(index); } + int importCount() const { return qmlData->nImports; } + const Import *importAt(int index) const { return qmlData->importAt(index); } QString stringAt(int index) const { - if (backingUnit) { - const qint32 backingUnitStringTableSize = backingUnit->stringTableSize; - if (index < backingUnitStringTableSize) - return backingUnit->stringAtInternal(index); - index -= backingUnitStringTableSize; - Q_ASSERT(data->stringTableSize == 0); - return dynamicStrings.at(index); - } + if (uint(index) >= data->stringTableSize) + return dynamicStrings.at(index - data->stringTableSize); return data->stringAtInternal(index); } @@ -1131,13 +1135,18 @@ protected: void linkBackendToEngine(QV4::ExecutionEngine *engine); quint32 totalStringCount() const - { return data->stringTableSize + (backingUnit ? backingUnit->stringTableSize : 0); } + { return data->stringTableSize; } +#else // V4_BOOTSTRAP + QString stringAt(int index) const { return data->stringAtInternal(index); } #endif // V4_BOOTSTRAP private: void destroy(); + QString m_fileName; // initialized from data->sourceFileIndex + QString m_finalUrlString; // initialized from data->finalUrlIndex + QAtomicInt refCount = 1; Q_NEVER_INLINE IdentifierHash createNamedObjectsPerComponent(int componentObjectIndex); diff --git a/src/qml/compiler/qv4compiler.cpp b/src/qml/compiler/qv4compiler.cpp index 44b809899f..f301f867c1 100644 --- a/src/qml/compiler/qv4compiler.cpp +++ b/src/qml/compiler/qv4compiler.cpp @@ -590,10 +590,7 @@ QV4::CompiledData::Unit QV4::Compiler::JSUnitGenerator::generateHeader(QV4::Comp unit.sourceFileIndex = getStringId(module->fileName); unit.finalUrlIndex = getStringId(module->finalUrl); unit.sourceTimeStamp = module->sourceTimeStamp.isValid() ? module->sourceTimeStamp.toMSecsSinceEpoch() : 0; - unit.nImports = 0; - unit.offsetToImports = 0; - unit.nObjects = 0; - unit.offsetToObjects = 0; + unit.offsetToQmlUnit = 0; unit.unitSize = nextOffset; |