diff options
Diffstat (limited to 'src/lib')
114 files changed, 2203 insertions, 742 deletions
diff --git a/src/lib/corelib/api/internaljobs.cpp b/src/lib/corelib/api/internaljobs.cpp index 5122db8ee..048ae5656 100644 --- a/src/lib/corelib/api/internaljobs.cpp +++ b/src/lib/corelib/api/internaljobs.cpp @@ -161,7 +161,10 @@ void InternalJob::storeBuildGraph(const TopLevelProjectPtr &project) TimedActivityLogger storeTimer(m_logger, Tr::tr("Storing build graph"), timed()); project->store(logger()); } catch (const ErrorInfo &error) { - logger().printWarning(error); + ErrorInfo fullError = this->error(); + for (const ErrorItem &item : error.items()) + fullError.append(item); + setError(fullError); } } @@ -246,7 +249,7 @@ TopLevelProjectPtr InternalSetupProjectJob::project() const void InternalSetupProjectJob::start() { - BuildGraphLocker *bgLocker = m_existingProject ? m_existingProject->bgLocker : 0; + BuildGraphLocker *bgLocker = m_existingProject ? m_existingProject->bgLocker : nullptr; bool deleteLocker = false; try { const ErrorInfo err = m_parameters.expandBuildConfiguration(); diff --git a/src/lib/corelib/api/jobs.cpp b/src/lib/corelib/api/jobs.cpp index 28b8a40fa..32b7accc7 100644 --- a/src/lib/corelib/api/jobs.cpp +++ b/src/lib/corelib/api/jobs.cpp @@ -229,10 +229,8 @@ SetupProjectJob::SetupProjectJob(const Logger &logger, QObject *parent) */ Project SetupProjectJob::project() const { - const InternalJobThreadWrapper * const wrapper - = qobject_cast<InternalJobThreadWrapper *>(internalJob()); - const InternalSetupProjectJob * const job - = qobject_cast<InternalSetupProjectJob *>(wrapper->synchronousJob()); + auto const wrapper = qobject_cast<const InternalJobThreadWrapper *>(internalJob()); + auto const job = qobject_cast<const InternalSetupProjectJob *>(wrapper->synchronousJob()); return Project(job->project(), job->logger()); } @@ -244,20 +242,16 @@ void SetupProjectJob::resolve(const Project &existingProject, = existingProject.d ? existingProject.d->internalProject : TopLevelProjectPtr(); if (existingInternalProject && !lockProject(existingInternalProject)) return; - InternalJobThreadWrapper * const wrapper - = qobject_cast<InternalJobThreadWrapper *>(internalJob()); - InternalSetupProjectJob * const job - = qobject_cast<InternalSetupProjectJob *>(wrapper->synchronousJob()); + auto const wrapper = qobject_cast<InternalJobThreadWrapper *>(internalJob()); + auto const job = qobject_cast<InternalSetupProjectJob *>(wrapper->synchronousJob()); job->init(existingInternalProject, parameters); wrapper->start(); } void SetupProjectJob::reportError(const ErrorInfo &error) { - InternalJobThreadWrapper * const wrapper - = qobject_cast<InternalJobThreadWrapper *>(internalJob()); - InternalSetupProjectJob * const job - = qobject_cast<InternalSetupProjectJob *>(wrapper->synchronousJob()); + auto const wrapper = qobject_cast<const InternalJobThreadWrapper *>(internalJob()); + auto const job = qobject_cast<InternalSetupProjectJob *>(wrapper->synchronousJob()); job->reportError(error); } @@ -351,7 +345,7 @@ void CleanJob::clean(const TopLevelProjectPtr &project, const QList<ResolvedProd { if (!lockProject(project)) return; - InternalJobThreadWrapper * wrapper = qobject_cast<InternalJobThreadWrapper *>(internalJob()); + auto wrapper = qobject_cast<InternalJobThreadWrapper *>(internalJob()); qobject_cast<InternalCleanJob *>(wrapper->synchronousJob())->init(project, products, options); wrapper->start(); } @@ -371,8 +365,8 @@ void InstallJob::install(const TopLevelProjectPtr &project, { if (!lockProject(project)) return; - InternalJobThreadWrapper *wrapper = qobject_cast<InternalJobThreadWrapper *>(internalJob()); - InternalInstallJob *installJob = qobject_cast<InternalInstallJob *>(wrapper->synchronousJob()); + auto wrapper = qobject_cast<InternalJobThreadWrapper *>(internalJob()); + auto installJob = qobject_cast<InternalInstallJob *>(wrapper->synchronousJob()); installJob->init(project, std::vector<ResolvedProductPtr>(products.cbegin(), products.cend()), options); diff --git a/src/lib/corelib/api/jobs.h b/src/lib/corelib/api/jobs.h index 04a0093c1..f121cc403 100644 --- a/src/lib/corelib/api/jobs.h +++ b/src/lib/corelib/api/jobs.h @@ -118,7 +118,7 @@ private: void resolve(const Project &existingProject, const SetupProjectParameters ¶meters); void reportError(const ErrorInfo &error); - void finish(); + void finish() override; Project m_existingProject; }; @@ -140,7 +140,7 @@ private: const BuildOptions &options); void handleLauncherError(const ErrorInfo &error); - void finish(); + void finish() override; }; diff --git a/src/lib/corelib/api/project.cpp b/src/lib/corelib/api/project.cpp index 5bbfb31c7..d1cb351be 100644 --- a/src/lib/corelib/api/project.cpp +++ b/src/lib/corelib/api/project.cpp @@ -1031,7 +1031,7 @@ RunEnvironment Project::getRunEnvironment(const ProductData &product, BuildJob *Project::buildAllProducts(const BuildOptions &options, ProductSelection productSelection, QObject *jobOwner) const { - QBS_ASSERT(isValid(), return 0); + QBS_ASSERT(isValid(), return nullptr); const bool includingNonDefault = productSelection == ProductSelectionWithNonDefault; return d->buildProducts(d->allEnabledInternalProducts(includingNonDefault), options, !includingNonDefault, jobOwner); @@ -1046,7 +1046,7 @@ BuildJob *Project::buildAllProducts(const BuildOptions &options, ProductSelectio BuildJob *Project::buildSomeProducts(const QList<ProductData> &products, const BuildOptions &options, QObject *jobOwner) const { - QBS_ASSERT(isValid(), return 0); + QBS_ASSERT(isValid(), return nullptr); return d->buildProducts(d->internalProducts(products), options, true, jobOwner); } @@ -1067,7 +1067,7 @@ BuildJob *Project::buildOneProduct(const ProductData &product, const BuildOption */ CleanJob *Project::cleanAllProducts(const CleanOptions &options, QObject *jobOwner) const { - QBS_ASSERT(isValid(), return 0); + QBS_ASSERT(isValid(), return nullptr); return d->cleanProducts(d->allEnabledInternalProducts(true), options, jobOwner); } @@ -1078,7 +1078,7 @@ CleanJob *Project::cleanAllProducts(const CleanOptions &options, QObject *jobOwn CleanJob *Project::cleanSomeProducts(const QList<ProductData> &products, const CleanOptions &options, QObject *jobOwner) const { - QBS_ASSERT(isValid(), return 0); + QBS_ASSERT(isValid(), return nullptr); return d->cleanProducts(d->internalProducts(products), options, jobOwner); } @@ -1101,7 +1101,7 @@ CleanJob *Project::cleanOneProduct(const ProductData &product, const CleanOption InstallJob *Project::installAllProducts(const InstallOptions &options, ProductSelection productSelection, QObject *jobOwner) const { - QBS_ASSERT(isValid(), return 0); + QBS_ASSERT(isValid(), return nullptr); const bool includingNonDefault = productSelection == ProductSelectionWithNonDefault; return d->installProducts(d->allEnabledInternalProducts(includingNonDefault), options, !includingNonDefault, jobOwner); @@ -1114,7 +1114,7 @@ InstallJob *Project::installAllProducts(const InstallOptions &options, InstallJob *Project::installSomeProducts(const QList<ProductData> &products, const InstallOptions &options, QObject *jobOwner) const { - QBS_ASSERT(isValid(), return 0); + QBS_ASSERT(isValid(), return nullptr); return d->installProducts(d->internalProducts(products), options, true, jobOwner); } diff --git a/src/lib/corelib/api/project.h b/src/lib/corelib/api/project.h index b43eecfc7..05f08deee 100644 --- a/src/lib/corelib/api/project.h +++ b/src/lib/corelib/api/project.h @@ -104,25 +104,26 @@ public: enum ProductSelection { ProductSelectionDefaultOnly, ProductSelectionWithNonDefault }; BuildJob *buildAllProducts(const BuildOptions &options, ProductSelection productSelection = ProductSelectionDefaultOnly, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; BuildJob *buildSomeProducts(const QList<ProductData> &products, const BuildOptions &options, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; BuildJob *buildOneProduct(const ProductData &product, const BuildOptions &options, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; - CleanJob *cleanAllProducts(const CleanOptions &options, QObject *jobOwner = 0) const; + CleanJob *cleanAllProducts(const CleanOptions &options, QObject *jobOwner = nullptr) const; CleanJob *cleanSomeProducts(const QList<ProductData> &products, const CleanOptions &options, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; CleanJob *cleanOneProduct(const ProductData &product, const CleanOptions &options, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; InstallJob *installAllProducts(const InstallOptions &options, ProductSelection productSelection = ProductSelectionDefaultOnly, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; InstallJob *installSomeProducts(const QList<ProductData> &products, - const InstallOptions &options, QObject *jobOwner = 0) const; + const InstallOptions &options, + QObject *jobOwner = nullptr) const; InstallJob *installOneProduct(const ProductData &product, const InstallOptions &options, - QObject *jobOwner = 0) const; + QObject *jobOwner = nullptr) const; void updateTimestamps(const QList<ProductData> &products); @@ -136,7 +137,7 @@ public: std::set<QString> buildSystemFiles() const; RuleCommandList ruleCommands(const ProductData &product, const QString &inputFilePath, - const QString &outputFileTag, ErrorInfo *error = 0) const; + const QString &outputFileTag, ErrorInfo *error = nullptr) const; ProjectTransformerData transformerData(ErrorInfo *error = nullptr) const; ErrorInfo dumpNodesTree(QIODevice &outDevice, const QList<ProductData> &products); diff --git a/src/lib/corelib/api/projectfileupdater.cpp b/src/lib/corelib/api/projectfileupdater.cpp index b11c43673..25c2705d1 100644 --- a/src/lib/corelib/api/projectfileupdater.cpp +++ b/src/lib/corelib/api/projectfileupdater.cpp @@ -71,7 +71,7 @@ public: UiObjectDefinition *item() const { return m_item; } private: - bool visit(UiObjectDefinition *ast) + bool visit(UiObjectDefinition *ast) override { if (toCodeLocation(m_cl.filePath(), ast->firstSourceLocation()) == m_cl) { m_item = ast; @@ -95,14 +95,14 @@ public: UiScriptBinding *binding() const { return m_binding; } private: - bool visit(UiObjectDefinition *ast) + bool visit(UiObjectDefinition *ast) override { // We start with the direct parent of the binding, so do not descend into any // other item. return ast == m_startItem; } - bool visit(UiScriptBinding *ast) + bool visit(UiScriptBinding *ast) override { if (ast->qualifiedId->name.toString() != StringConstants::filesProperty()) return true; diff --git a/src/lib/corelib/api/projectfileupdater.h b/src/lib/corelib/api/projectfileupdater.h index 7615b45f9..bc8de30eb 100644 --- a/src/lib/corelib/api/projectfileupdater.h +++ b/src/lib/corelib/api/projectfileupdater.h @@ -94,7 +94,7 @@ public: ProjectFileGroupInserter(const ProductData &product, const QString &groupName); private: - void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast); + void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast) override; const ProductData m_product; const QString m_groupName; @@ -108,7 +108,7 @@ public: const QStringList &files); private: - void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast); + void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast) override; const ProductData m_product; const GroupData m_group; @@ -122,7 +122,7 @@ public: const QStringList &files); private: - void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast); + void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast) override; const ProductData m_product; const GroupData m_group; @@ -135,7 +135,7 @@ public: ProjectFileGroupRemover(const ProductData &product, const GroupData &group); private: - void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast); + void doApply(QString &fileContent, QbsQmlJS::AST::UiProgram *ast) override; const ProductData m_product; const GroupData m_group; diff --git a/src/lib/corelib/api/qmljsrewriter.cpp b/src/lib/corelib/api/qmljsrewriter.cpp index ebcc00b07..60c384004 100644 --- a/src/lib/corelib/api/qmljsrewriter.cpp +++ b/src/lib/corelib/api/qmljsrewriter.cpp @@ -177,11 +177,11 @@ UiObjectMemberList *Rewriter::searchMemberToInsertAfter(UiObjectMemberList *memb if (cast<UiObjectDefinition*>(member)) lastObjectDef = iter; - else if (UiArrayBinding *arrayBinding = cast<UiArrayBinding*>(member)) + else if (auto arrayBinding = cast<UiArrayBinding*>(member)) idx = propertyOrder.indexOf(toString(arrayBinding->qualifiedId)); - else if (UiObjectBinding *objectBinding = cast<UiObjectBinding*>(member)) + else if (auto objectBinding = cast<UiObjectBinding*>(member)) idx = propertyOrder.indexOf(toString(objectBinding->qualifiedId)); - else if (UiScriptBinding *scriptBinding = cast<UiScriptBinding*>(member)) + else if (auto scriptBinding = cast<UiScriptBinding*>(member)) idx = propertyOrder.indexOf(toString(scriptBinding->qualifiedId)); else if (cast<UiPublicMember*>(member)) idx = propertyOrder.indexOf(QLatin1String("property")); @@ -210,11 +210,11 @@ UiArrayMemberList *Rewriter::searchMemberToInsertAfter(UiArrayMemberList *member if (cast<UiObjectDefinition*>(member)) lastObjectDef = iter; - else if (UiArrayBinding *arrayBinding = cast<UiArrayBinding*>(member)) + else if (auto arrayBinding = cast<UiArrayBinding*>(member)) idx = propertyOrder.indexOf(toString(arrayBinding->qualifiedId)); - else if (UiObjectBinding *objectBinding = cast<UiObjectBinding*>(member)) + else if (auto objectBinding = cast<UiObjectBinding*>(member)) idx = propertyOrder.indexOf(toString(objectBinding->qualifiedId)); - else if (UiScriptBinding *scriptBinding = cast<UiScriptBinding*>(member)) + else if (auto scriptBinding = cast<UiScriptBinding*>(member)) idx = propertyOrder.indexOf(toString(scriptBinding->qualifiedId)); else if (cast<UiPublicMember*>(member)) idx = propertyOrder.indexOf(QLatin1String("property")); @@ -241,13 +241,13 @@ UiObjectMemberList *Rewriter::searchMemberToInsertAfter(UiObjectMemberList *memb for (UiObjectMemberList *iter = members; iter; iter = iter->next) { UiObjectMember *member = iter->member; - if (UiArrayBinding *arrayBinding = cast<UiArrayBinding*>(member)) + if (auto arrayBinding = cast<UiArrayBinding*>(member)) orderedMembers[toString(arrayBinding->qualifiedId)] = iter; - else if (UiObjectBinding *objectBinding = cast<UiObjectBinding*>(member)) + else if (auto objectBinding = cast<UiObjectBinding*>(member)) orderedMembers[toString(objectBinding->qualifiedId)] = iter; else if (cast<UiObjectDefinition*>(member)) orderedMembers[QString::null] = iter; - else if (UiScriptBinding *scriptBinding = cast<UiScriptBinding*>(member)) + else if (auto scriptBinding = cast<UiScriptBinding*>(member)) orderedMembers[toString(scriptBinding->qualifiedId)] = iter; else if (cast<UiPublicMember*>(member)) orderedMembers[QLatin1String("property")] = iter; @@ -262,7 +262,7 @@ UiObjectMemberList *Rewriter::searchMemberToInsertAfter(UiObjectMemberList *memb for (; idx > 0; --idx) { const QString prop = propertyOrder.at(idx - 1); UiObjectMemberList *candidate = orderedMembers.value(prop, 0); - if (candidate != 0) + if (candidate != nullptr) return candidate; } @@ -306,7 +306,7 @@ void Rewriter::changeBinding(UiObjectInitializer *ast, break; // for grouped properties: } else if (!prefix.isEmpty()) { - if (UiObjectDefinition *def = cast<UiObjectDefinition *>(member)) { + if (auto def = cast<UiObjectDefinition *>(member)) { if (toString(def->qualifiedTypeNameId) == prefix) changeBinding(def->initializer, suffix, newValue, binding); } @@ -321,16 +321,16 @@ void Rewriter::replaceMemberValue(UiObjectMember *propertyMember, QString replacement = newValue; int startOffset = -1; int endOffset = -1; - if (UiObjectBinding *objectBinding = AST::cast<UiObjectBinding *>(propertyMember)) { + if (auto objectBinding = AST::cast<UiObjectBinding *>(propertyMember)) { startOffset = objectBinding->qualifiedTypeNameId->identifierToken.offset; endOffset = objectBinding->initializer->rbraceToken.end(); - } else if (UiScriptBinding *scriptBinding = AST::cast<UiScriptBinding *>(propertyMember)) { + } else if (auto scriptBinding = AST::cast<UiScriptBinding *>(propertyMember)) { startOffset = scriptBinding->statement->firstSourceLocation().offset; endOffset = scriptBinding->statement->lastSourceLocation().end(); - } else if (UiArrayBinding *arrayBinding = AST::cast<UiArrayBinding *>(propertyMember)) { + } else if (auto arrayBinding = AST::cast<UiArrayBinding *>(propertyMember)) { startOffset = arrayBinding->lbracketToken.offset; endOffset = arrayBinding->rbracketToken.end(); - } else if (UiPublicMember *publicMember = AST::cast<UiPublicMember*>(propertyMember)) { + } else if (auto publicMember = AST::cast<UiPublicMember*>(propertyMember)) { if (publicMember->statement) { startOffset = publicMember->statement->firstSourceLocation().offset; if (publicMember->semicolonToken.isValid()) @@ -357,13 +357,13 @@ void Rewriter::replaceMemberValue(UiObjectMember *propertyMember, bool Rewriter::isMatchingPropertyMember(const QString &propertyName, UiObjectMember *member) { - if (UiPublicMember *publicMember = cast<UiPublicMember*>(member)) + if (auto publicMember = cast<UiPublicMember*>(member)) return publicMember->name == propertyName; - else if (UiObjectBinding *objectBinding = cast<UiObjectBinding*>(member)) + else if (auto objectBinding = cast<UiObjectBinding*>(member)) return toString(objectBinding->qualifiedId) == propertyName; - else if (UiScriptBinding *scriptBinding = cast<UiScriptBinding*>(member)) + else if (auto scriptBinding = cast<UiScriptBinding*>(member)) return toString(scriptBinding->qualifiedId) == propertyName; - else if (UiArrayBinding *arrayBinding = cast<UiArrayBinding*>(member)) + else if (auto arrayBinding = cast<UiArrayBinding*>(member)) return toString(arrayBinding->qualifiedId) == propertyName; else return false; @@ -409,7 +409,7 @@ void Rewriter::removeBindingByName(UiObjectInitializer *ast, const QString &prop removeMember(member); // check for grouped properties: } else if (!prefix.isEmpty()) { - if (UiObjectDefinition *def = cast<UiObjectDefinition *>(member)) { + if (auto def = cast<UiObjectDefinition *>(member)) { if (toString(def->qualifiedTypeNameId) == prefix) removeGroupedProperty(def, propertyName); } @@ -680,10 +680,10 @@ void Rewriter::removeObjectMember(UiObjectMember *member, UiObjectMember *parent int start = member->firstSourceLocation().offset; int end = member->lastSourceLocation().end(); - if (UiArrayBinding *parentArray = cast<UiArrayBinding *>(parent)) { + if (auto parentArray = cast<UiArrayBinding *>(parent)) { extendToLeadingOrTrailingComma(parentArray, member, start, end); } else { - if (UiObjectDefinition *parentObjectDefinition = cast<UiObjectDefinition *>(parent)) + if (auto parentObjectDefinition = cast<UiObjectDefinition *>(parent)) includeEmptyGroupedProperty(parentObjectDefinition, member, start, end); includeSurroundingWhitespace(m_originalText, start, end); } diff --git a/src/lib/corelib/buildgraph/artifact.cpp b/src/lib/corelib/buildgraph/artifact.cpp index 667194356..e82919560 100644 --- a/src/lib/corelib/buildgraph/artifact.cpp +++ b/src/lib/corelib/buildgraph/artifact.cpp @@ -42,6 +42,7 @@ #include "transformer.h" #include "buildgraphvisitor.h" #include "productbuilddata.h" +#include "rulenode.h" #include <language/language.h> #include <language/propertymapinternal.h> #include <tools/persistence.h> @@ -77,15 +78,21 @@ QString Artifact::toString() const void Artifact::addFileTag(const FileTag &t) { m_fileTags += t; - if (!product.expired() && product->buildData) + if (!product.expired() && product->buildData) { product->buildData->addFileTagToArtifact(this, t); + if (product->fileTags.contains(t)) + product->buildData->addRootNode(this); + } } void Artifact::removeFileTag(const FileTag &t) { m_fileTags -= t; - if (!product.expired() && product->buildData) + if (!product.expired() && product->buildData) { product->buildData->removeArtifactFromSetByFileTag(this, t); + if (product->fileTags.contains(t) && !product->fileTags.intersects(m_fileTags)) + product->buildData->removeFromRootNodes(this); + } } void Artifact::setFileTags(const FileTags &newFileTags) @@ -94,10 +101,23 @@ void Artifact::setFileTags(const FileTags &newFileTags) m_fileTags = newFileTags; return; } - for (const FileTag &t : qAsConst(m_fileTags)) - product->buildData->removeArtifactFromSetByFileTag(this, t); - m_fileTags = newFileTags; - product->buildData->addArtifactToSet(this); + if (m_fileTags == newFileTags) + return; + const Set<FileTag> addedTags = newFileTags - m_fileTags; + for (const FileTag &t : addedTags) + addFileTag(t); + const Set<FileTag> removedTags = m_fileTags - newFileTags; + for (const FileTag &t : removedTags) + removeFileTag(t); +} + +RuleNode *Artifact::producer() const +{ + if (artifactType == SourceFile) + return nullptr; + const auto ruleNodes = filterByType<RuleNode>(children); + QBS_CHECK(ruleNodes.begin() != ruleNodes.end()); + return *ruleNodes.begin(); } void Artifact::initialize() @@ -133,7 +153,7 @@ void Artifact::load(PersistentPool &pool) children.load(pool); // restore parents of the loaded children - for (NodeSet::const_iterator it = children.constBegin(); it != children.constEnd(); ++it) + for (auto it = children.constBegin(); it != children.constEnd(); ++it) (*it)->parents.insert(this); pool.load(childrenAddedByScanner); @@ -142,10 +162,11 @@ void Artifact::load(PersistentPool &pool) pool.load(targetOfModule); pool.load(transformer); pool.load(m_fileTags); + pool.load(pureFileTags); + pool.load(pureProperties); artifactType = static_cast<ArtifactType>(pool.load<quint8>()); alwaysUpdated = pool.load<bool>(); oldDataPossiblyPresent = pool.load<bool>(); - pool.load(knownOutOfDate); } void Artifact::store(PersistentPool &pool) @@ -160,10 +181,11 @@ void Artifact::store(PersistentPool &pool) pool.store(targetOfModule); pool.store(transformer); pool.store(m_fileTags); + pool.store(pureFileTags); + pool.store(pureProperties); pool.store(static_cast<quint8>(artifactType)); pool.store(alwaysUpdated); pool.store(oldDataPossiblyPresent); - pool.store(knownOutOfDate); } } // namespace Internal diff --git a/src/lib/corelib/buildgraph/artifact.h b/src/lib/corelib/buildgraph/artifact.h index b1669974c..4d2a20086 100644 --- a/src/lib/corelib/buildgraph/artifact.h +++ b/src/lib/corelib/buildgraph/artifact.h @@ -50,6 +50,9 @@ #include <QtCore/qstring.h> +#include <utility> +#include <vector> + namespace qbs { namespace Internal { class Logger; @@ -81,12 +84,23 @@ public: void setFileTags(const FileTags &newFileTags); const FileTags &fileTags() const { return m_fileTags; } + RuleNode *producer() const; + ArtifactSet childrenAddedByScanner; Set<FileDependency *> fileDependencies; TransformerPtr transformer; PropertyMapPtr properties; QString targetOfModule; + // The tags set directly via an Artifact item or an outputArtifacts script, + // not the result of file taggers or fileTagsFilter groups, nor the ones inherited from + // the product. + FileTags pureFileTags; + + // The properties attached directly to an artifact in an Artifact item or outputArtifacts + // script. + std::vector<std::pair<QStringList, QVariant>> pureProperties; + enum ArtifactType { Unknown = 1, @@ -100,8 +114,6 @@ public: bool alwaysUpdated : 1; bool oldDataPossiblyPresent : 1; - bool knownOutOfDate = false; - void initialize(); const TypeFilter<Artifact> parentArtifacts() const; const TypeFilter<Artifact> childArtifacts() const; @@ -118,7 +130,7 @@ private: template<> inline QString Set<Artifact *>::toString(Artifact * const &artifact) const { - return artifact->filePath(); + return artifact ? artifact->filePath() : QLatin1String("<null>"); } template<> inline const void *uniqueAddress(const Artifact *a) { diff --git a/src/lib/corelib/buildgraph/artifactcleaner.h b/src/lib/corelib/buildgraph/artifactcleaner.h index 34be4e0f4..4d8c86ea8 100644 --- a/src/lib/corelib/buildgraph/artifactcleaner.h +++ b/src/lib/corelib/buildgraph/artifactcleaner.h @@ -59,7 +59,7 @@ public: private: void removeEmptyDirectories(const QString &rootDir, const CleanOptions &options, - bool *isEmpty = 0); + bool *isEmpty = nullptr); Logger m_logger; bool m_hasError; diff --git a/src/lib/corelib/buildgraph/artifactvisitor.h b/src/lib/corelib/buildgraph/artifactvisitor.h index 6304d5f43..89bd4429a 100644 --- a/src/lib/corelib/buildgraph/artifactvisitor.h +++ b/src/lib/corelib/buildgraph/artifactvisitor.h @@ -56,8 +56,8 @@ public: void visitProduct(const ResolvedProductConstPtr &product); void visitProject(const ResolvedProjectConstPtr &project); - bool visit(RuleNode *ruleNode); - bool visit(Artifact *artifact); + bool visit(RuleNode *ruleNode) override; + bool visit(Artifact *artifact) override; private: virtual void doVisit(Artifact *artifact) = 0; diff --git a/src/lib/corelib/buildgraph/buildgraph.cpp b/src/lib/corelib/buildgraph/buildgraph.cpp index ed48b427d..e6d49f3d5 100644 --- a/src/lib/corelib/buildgraph/buildgraph.cpp +++ b/src/lib/corelib/buildgraph/buildgraph.cpp @@ -50,6 +50,7 @@ #include <jsextensions/jsextensions.h> #include <jsextensions/moduleproperties.h> +#include <language/artifactproperties.h> #include <language/language.h> #include <language/preparescriptobserver.h> #include <language/propertymapinternal.h> @@ -71,6 +72,7 @@ #include <QtScript/qscriptclass.h> #include <algorithm> +#include <iterator> #include <vector> namespace qbs { @@ -464,7 +466,7 @@ void connect(BuildGraphNode *p, BuildGraphNode *c) QBS_CHECK(p != c); qCDebug(lcBuildGraph).noquote() << "connect" << p->toString() << "->" << c->toString(); if (c->type() == BuildGraphNode::ArtifactNodeType) { - Artifact * const ac = static_cast<Artifact *>(c); + auto const ac = static_cast<Artifact *>(c); for (const Artifact *child : filterByType<Artifact>(p->children)) { if (child == ac) return; @@ -576,7 +578,7 @@ Artifact *lookupArtifact(const ResolvedProductConstPtr &product, it != lookupResults.constEnd(); ++it) { if ((*it)->fileType() != FileResourceBase::FileTypeArtifact) continue; - Artifact *artifact = static_cast<Artifact *>(*it); + auto artifact = static_cast<Artifact *>(*it); if (compareByName ? artifact->product->uniqueName() == product->uniqueName() : artifact->product == product) { @@ -620,12 +622,31 @@ Artifact *createArtifact(const ResolvedProductPtr &product, { auto artifact = new Artifact; artifact->artifactType = Artifact::SourceFile; + setArtifactData(artifact, sourceArtifact); + insertArtifact(product, artifact); + return artifact; +} + +void setArtifactData(Artifact *artifact, const SourceArtifactConstPtr &sourceArtifact) +{ artifact->targetOfModule = sourceArtifact->targetOfModule; artifact->setFilePath(sourceArtifact->absoluteFilePath); artifact->setFileTags(sourceArtifact->fileTags); artifact->properties = sourceArtifact->properties; - insertArtifact(product, artifact); - return artifact; +} + +void updateArtifactFromSourceArtifact(const ResolvedProductPtr &product, + const SourceArtifactConstPtr &sourceArtifact) +{ + Artifact * const artifact = lookupArtifact(product, sourceArtifact->absoluteFilePath, false); + QBS_CHECK(artifact); + const FileTags oldFileTags = artifact->fileTags(); + const QVariantMap oldModuleProperties = artifact->properties->value(); + setArtifactData(artifact, sourceArtifact); + if (oldFileTags != artifact->fileTags() + || oldModuleProperties != artifact->properties->value()) { + invalidateArtifactAsRuleInputIfNecessary(artifact); + } } void insertArtifact(const ResolvedProductPtr &product, Artifact *artifact) @@ -638,6 +659,76 @@ void insertArtifact(const ResolvedProductPtr &product, Artifact *artifact) product->buildData->addArtifact(artifact); } +void provideFullFileTagsAndProperties(Artifact *artifact) +{ + artifact->properties = artifact->product->moduleProperties; + FileTags allTags = artifact->pureFileTags.empty() + ? artifact->product->fileTagsForFileName(artifact->fileName()) : artifact->pureFileTags; + for (const ArtifactPropertiesConstPtr &props : artifact->product->artifactProperties) { + if (allTags.intersects(props->fileTagsFilter())) { + artifact->properties = props->propertyMap(); + allTags += props->extraFileTags(); + break; + } + } + artifact->setFileTags(allTags); + + // Let a positive value of qbs.install imply the file tag "installable". + if (artifact->properties->qbsPropertyValue(StringConstants::installProperty()).toBool()) + artifact->addFileTag("installable"); +} + +void applyPerArtifactProperties(Artifact *artifact) +{ + if (artifact->pureProperties.empty()) + return; + QVariantMap props = artifact->properties->value(); + for (const auto &property : artifact->pureProperties) + setConfigProperty(props, property.first, property.second); + artifact->properties = artifact->properties->clone(); + artifact->properties->setValue(props); +} + +void updateGeneratedArtifacts(ResolvedProduct *product) +{ + if (!product->buildData) + return; + for (Artifact * const artifact : filterByType<Artifact>(product->buildData->allNodes())) { + if (artifact->artifactType == Artifact::Generated) { + const FileTags oldFileTags = artifact->fileTags(); + const QVariantMap oldModuleProperties = artifact->properties->value(); + provideFullFileTagsAndProperties(artifact); + applyPerArtifactProperties(artifact); + if (oldFileTags != artifact->fileTags() + || oldModuleProperties != artifact->properties->value()) { + invalidateArtifactAsRuleInputIfNecessary(artifact); + } + } + } +} + +// This is needed for artifacts which are inputs to rules whose outputArtifacts script +// returned an empty array for this input. Since there is no transformer, our usual change +// tracking procedure will not notice if the artifact's file tags or module properties have +// changed, so we need to force a re-run of the outputArtifacts script. +void invalidateArtifactAsRuleInputIfNecessary(Artifact *artifact) +{ + for (RuleNode * const parentRuleNode : filterByType<RuleNode>(artifact->parents)) { + if (!parentRuleNode->rule()->isDynamic()) + continue; + bool artifactNeedsExplicitInvalidation = true; + for (Artifact * const output : filterByType<Artifact>(parentRuleNode->parents)) { + if (output->children.contains(artifact) + && !output->childrenAddedByScanner.contains(artifact)) { + artifactNeedsExplicitInvalidation = false; + break; + } + } + if (artifactNeedsExplicitInvalidation) + parentRuleNode->removeOldInputArtifact(artifact); + } +} + static void doSanityChecksForProduct(const ResolvedProductConstPtr &product, const Set<ResolvedProductPtr> &allProducts, const Logger &logger) { @@ -674,7 +765,7 @@ static void doSanityChecksForProduct(const ResolvedProductConstPtr &product, ? static_cast<Artifact *>(node) : nullptr; if (!artifact) { QBS_CHECK(node->type() == BuildGraphNode::RuleNodeType); - RuleNode * const ruleNode = static_cast<RuleNode *>(node); + auto const ruleNode = static_cast<RuleNode *>(node); QBS_CHECK(ruleNode->rule()); QBS_CHECK(ruleNode->rule()->product); QBS_CHECK(ruleNode->rule()->product == ruleNode->product.get()); @@ -695,6 +786,9 @@ static void doSanityChecksForProduct(const ResolvedProductConstPtr &product, if (artifact->artifactType == Artifact::SourceFile) continue; + const auto parentRuleNodes = filterByType<RuleNode>(artifact->children); + QBS_CHECK(std::distance(parentRuleNodes.begin(), parentRuleNodes.end()) == 1); + QBS_CHECK(transformer); QBS_CHECK(transformer->rule); QBS_CHECK(transformer->rule->product); diff --git a/src/lib/corelib/buildgraph/buildgraph.h b/src/lib/corelib/buildgraph/buildgraph.h index 50195903d..2909b06bb 100644 --- a/src/lib/corelib/buildgraph/buildgraph.h +++ b/src/lib/corelib/buildgraph/buildgraph.h @@ -69,9 +69,15 @@ Artifact *lookupArtifact(const ResolvedProductConstPtr &product, const Artifact Artifact *createArtifact(const ResolvedProductPtr &product, const SourceArtifactConstPtr &sourceArtifact); +void setArtifactData(Artifact *artifact, const SourceArtifactConstPtr &sourceArtifact); +void updateArtifactFromSourceArtifact(const ResolvedProductPtr &product, + const SourceArtifactConstPtr &sourceArtifact); void insertArtifact(const ResolvedProductPtr &product, Artifact *artifact); void dumpProductBuildData(const ResolvedProductConstPtr &product); - +void provideFullFileTagsAndProperties(Artifact *artifact); +void applyPerArtifactProperties(Artifact *artifact); +void updateGeneratedArtifacts(ResolvedProduct *product); +void invalidateArtifactAsRuleInputIfNecessary(Artifact *artifact); bool findPath(BuildGraphNode *u, BuildGraphNode *v, QList<BuildGraphNode*> &path); void QBS_AUTOTEST_EXPORT connect(BuildGraphNode *p, BuildGraphNode *c); diff --git a/src/lib/corelib/buildgraph/buildgraphloader.cpp b/src/lib/corelib/buildgraph/buildgraphloader.cpp index f9b9910e6..a1ca7afdb 100644 --- a/src/lib/corelib/buildgraph/buildgraphloader.cpp +++ b/src/lib/corelib/buildgraph/buildgraphloader.cpp @@ -380,7 +380,10 @@ void BuildGraphLoader::trackProjectChanges() // an a per-artifact basis by the Executor on the next build. QHash<QString, AllRescuableArtifactData> rescuableArtifactData; for (const ResolvedProductPtr &product : qAsConst(changedProducts)) { - ResolvedProductPtr freshProduct = m_freshProductsByName.value(product->uniqueName()); + const QString name = product->uniqueName(); + m_changedSourcesByProduct.erase(name); + m_productsWhoseArtifactsNeedUpdate.remove(name); + ResolvedProductPtr freshProduct = m_freshProductsByName.value(name); if (!freshProduct) continue; onProductRemoved(product, product->topLevelProject()->buildData.get(), false); @@ -433,6 +436,20 @@ void BuildGraphLoader::trackProjectChanges() allNewlyResolvedProducts); } + for (auto it = m_changedSourcesByProduct.cbegin(); it != m_changedSourcesByProduct.cend(); + ++it) { + const ResolvedProductPtr product = m_freshProductsByName.value(it->first); + QBS_CHECK(!!product); + for (const SourceArtifactConstPtr &sa : it->second) + updateArtifactFromSourceArtifact(product, sa); + } + + for (const QString &productName : m_productsWhoseArtifactsNeedUpdate) { + const ResolvedProductPtr product = m_freshProductsByName.value(productName); + QBS_CHECK(!!product); + updateGeneratedArtifacts(product.get()); + } + for (const ResolvedProductConstPtr &changedProduct : qAsConst(changedProducts)) { rescueOldBuildData(changedProduct, m_freshProductsByName.value(changedProduct->uniqueName()), @@ -663,8 +680,7 @@ void BuildGraphLoader::checkAllProductsForChanges( continue; } - if (!sourceArtifactSetsAreEqual(restoredProduct->allEnabledFiles(), - newlyResolvedProduct->allEnabledFiles())) { + if (checkProductForChangesInSourceFiles(restoredProduct, newlyResolvedProduct)) { qCDebug(lcBuildGraph) << "File list of product" << restoredProduct->uniqueName() << "was changed."; if (!contains(changedProducts, restoredProduct)) @@ -673,6 +689,38 @@ void BuildGraphLoader::checkAllProductsForChanges( } } +bool BuildGraphLoader::checkProductForChangesInSourceFiles( + const ResolvedProductPtr &restoredProduct, const ResolvedProductPtr &newlyResolvedProduct) +{ + std::vector<SourceArtifactPtr> oldFiles = restoredProduct->allEnabledFiles(); + std::vector<SourceArtifactPtr> newFiles = newlyResolvedProduct->allEnabledFiles(); + // TODO: Also handle added and removed files in a fine-grained manner. + if (oldFiles.size() != newFiles.size()) + return true; + static const auto cmp = [](const SourceArtifactConstPtr &a1, + const SourceArtifactConstPtr &a2) { + return a1->absoluteFilePath < a2->absoluteFilePath; + }; + std::sort(oldFiles.begin(), oldFiles.end(), cmp); + std::sort(newFiles.begin(), newFiles.end(), cmp); + std::vector<SourceArtifactConstPtr> changedFiles; + for (int i = 0; i < int(oldFiles.size()); ++i) { + const SourceArtifactConstPtr &oldFile = oldFiles.at(i); + const SourceArtifactConstPtr &newFile = newFiles.at(i); + if (oldFile->absoluteFilePath != newFile->absoluteFilePath) + return true; + if (*oldFile != *newFile) { + qCDebug(lcBuildGraph) << "source artifact" << oldFile->absoluteFilePath << "changed"; + changedFiles.push_back(newFile); + } + } + if (!changedFiles.empty()) { + m_changedSourcesByProduct.insert(std::make_pair(restoredProduct->uniqueName(), + changedFiles)); + } + return false; +} + static bool dependenciesAreEqual(const ResolvedProductConstPtr &p1, const ResolvedProductConstPtr &p2) { @@ -742,7 +790,14 @@ bool BuildGraphLoader::checkForPropertyChanges(const ResolvedProductPtr &restore return true; if (!artifactPropertyListsAreEqual(restoredProduct->artifactProperties, newlyResolvedProduct->artifactProperties)) { - return true; + qCDebug(lcBuildGraph) << "a fileTagFilter group changed for product" + << restoredProduct->uniqueName(); + m_productsWhoseArtifactsNeedUpdate << restoredProduct->uniqueName(); + } + if (restoredProduct->moduleProperties != newlyResolvedProduct->moduleProperties) { + qCDebug(lcBuildGraph) << "module properties changed for product" + << restoredProduct->uniqueName(); + m_productsWhoseArtifactsNeedUpdate << restoredProduct->uniqueName(); } return false; } @@ -857,7 +912,7 @@ void BuildGraphLoader::rescueOldBuildData(const ResolvedProductConstPtr &restore if (!newArtifact) { RescuableArtifactData rad; rad.timeStamp = oldArtifact->timestamp(); - rad.knownOutOfDate = oldArtifact->knownOutOfDate; + rad.knownOutOfDate = oldArtifact->transformer->markedForRerun; rad.fileTags = oldArtifact->fileTags(); rad.properties = oldArtifact->properties; rad.commands = oldArtifact->transformer->commands; diff --git a/src/lib/corelib/buildgraph/buildgraphloader.h b/src/lib/corelib/buildgraph/buildgraphloader.h index e83a4a088..30d7b9b45 100644 --- a/src/lib/corelib/buildgraph/buildgraphloader.h +++ b/src/lib/corelib/buildgraph/buildgraphloader.h @@ -51,6 +51,8 @@ #include <QtCore/qprocess.h> #include <QtCore/qvariant.h> +#include <unordered_map> + namespace qbs { namespace Internal { @@ -101,6 +103,8 @@ private: std::vector<ResolvedProductPtr> &changedProducts); bool checkProductForChanges(const ResolvedProductPtr &restoredProduct, const ResolvedProductPtr &newlyResolvedProduct); + bool checkProductForChangesInSourceFiles(const ResolvedProductPtr &restoredProduct, + const ResolvedProductPtr &newlyResolvedProduct); bool checkProductForInstallInfoChanges(const ResolvedProductPtr &restoredProduct, const ResolvedProductPtr &newlyResolvedProduct); bool checkForPropertyChanges(const ResolvedProductPtr &restoredProduct, @@ -135,6 +139,8 @@ private: BuildGraphLoadResult m_result; Logger m_logger; QStringList m_artifactsRemovedFromDisk; + std::unordered_map<QString, std::vector<SourceArtifactConstPtr>> m_changedSourcesByProduct; + Set<QString> m_productsWhoseArtifactsNeedUpdate; qint64 m_wildcardExpansionEffort; qint64 m_propertyComparisonEffort; diff --git a/src/lib/corelib/buildgraph/cycledetector.h b/src/lib/corelib/buildgraph/cycledetector.h index d280f6ecd..fffe6ed2f 100644 --- a/src/lib/corelib/buildgraph/cycledetector.h +++ b/src/lib/corelib/buildgraph/cycledetector.h @@ -58,8 +58,8 @@ public: void visitProduct(const ResolvedProductConstPtr &product); private: - bool visit(Artifact *artifact); - bool visit(RuleNode *ruleNode); + bool visit(Artifact *artifact) override; + bool visit(RuleNode *ruleNode) override; bool visitNode(BuildGraphNode *node); diff --git a/src/lib/corelib/buildgraph/depscanner.h b/src/lib/corelib/buildgraph/depscanner.h index fef71058f..5c8ef83d9 100644 --- a/src/lib/corelib/buildgraph/depscanner.h +++ b/src/lib/corelib/buildgraph/depscanner.h @@ -84,13 +84,13 @@ public: PluginDependencyScanner(ScannerPlugin *plugin); private: - QStringList collectSearchPaths(Artifact *artifact); - QStringList collectDependencies(FileResourceBase *file, const char *fileTags); - bool recursive() const; - const void *key() const; - QString createId() const; + QStringList collectSearchPaths(Artifact *artifact) override; + QStringList collectDependencies(FileResourceBase *file, const char *fileTags) override; + bool recursive() const override; + const void *key() const override; + QString createId() const override; bool areModulePropertiesCompatible(const PropertyMapConstPtr &m1, - const PropertyMapConstPtr &m2) const; + const PropertyMapConstPtr &m2) const override; ScannerPlugin* m_plugin; }; @@ -101,13 +101,13 @@ public: UserDependencyScanner(const ResolvedScannerConstPtr &scanner, ScriptEngine *engine); private: - QStringList collectSearchPaths(Artifact *artifact); - QStringList collectDependencies(FileResourceBase *file, const char *fileTags); - bool recursive() const; - const void *key() const; - QString createId() const; + QStringList collectSearchPaths(Artifact *artifact) override; + QStringList collectDependencies(FileResourceBase *file, const char *fileTags) override; + bool recursive() const override; + const void *key() const override; + QString createId() const override; bool areModulePropertiesCompatible(const PropertyMapConstPtr &m1, - const PropertyMapConstPtr &m2) const; + const PropertyMapConstPtr &m2) const override; QStringList evaluate(Artifact *artifact, const PrivateScriptFunction &script); diff --git a/src/lib/corelib/buildgraph/executor.cpp b/src/lib/corelib/buildgraph/executor.cpp index 42e3eee8c..0558980d6 100644 --- a/src/lib/corelib/buildgraph/executor.cpp +++ b/src/lib/corelib/buildgraph/executor.cpp @@ -61,10 +61,12 @@ #include <logging/translator.h> #include <tools/error.h> #include <tools/fileinfo.h> +#include <tools/preferences.h> #include <tools/profiling.h> #include <tools/progressobserver.h> #include <tools/qbsassert.h> #include <tools/qttools.h> +#include <tools/settings.h> #include <tools/stringconstants.h> #include <QtCore/qdir.h> @@ -223,7 +225,6 @@ void Executor::doBuild() } QBS_CHECK(m_state == ExecutorIdle); m_leaves = Leaves(); - m_changedSourceArtifacts.clear(); m_error.clear(); m_explicitlyCanceled = false; m_activeFileTags = FileTags::fromStringList(m_buildOptions.activeFileTags()); @@ -231,6 +232,9 @@ void Executor::doBuild() m_tagsNeededForFilesToConsider.clear(); m_productsOfFilesToConsider.clear(); m_artifactsRemovedFromDisk.clear(); + m_jobCountPerPool.clear(); + + setupJobLimits(); // TODO: The "filesToConsider" thing is badly designed; we should know exactly which artifact // it is. Remove this from the BuildOptions class and introduce Project::buildSomeFiles() @@ -243,7 +247,7 @@ void Executor::doBuild() for (const FileResourceBase * const file : files) { if (file->fileType() != FileResourceBase::FileTypeArtifact) continue; - const Artifact * const artifact = static_cast<const Artifact *>(file); + auto const artifact = static_cast<const Artifact *>(file); if (contains(m_productsToBuild, artifact->product.lock())) { m_tagsOfFilesToConsider.unite(artifact->fileTags()); m_productsOfFilesToConsider << artifact->product.lock(); @@ -323,7 +327,7 @@ void Executor::updateLeaves(BuildGraphNode *node, NodeSet &seenNodes) if (node->buildState == BuildGraphNode::Untouched) { node->buildState = BuildGraphNode::Buildable; if (node->type() == BuildGraphNode::ArtifactNodeType) { - Artifact * const artifact = static_cast<Artifact *>(node); + auto const artifact = static_cast<Artifact *>(node); if (artifact->artifactType == Artifact::SourceFile) retrieveSourceFileTimestamp(artifact); } @@ -347,6 +351,7 @@ void Executor::updateLeaves(BuildGraphNode *node, NodeSet &seenNodes) bool Executor::scheduleJobs() { QBS_CHECK(m_state == ExecutorRunning); + std::vector<BuildGraphNode *> delayedLeaves; while (!m_leaves.empty() && !m_availableJobs.empty()) { BuildGraphNode * const nodeToBuild = m_leaves.top(); m_leaves.pop(); @@ -356,9 +361,17 @@ bool Executor::scheduleJobs() QBS_ASSERT(!"untouched node in leaves list", qDebug("%s", qPrintable(nodeToBuild->toString()))); break; - case BuildGraphNode::Buildable: - // This is the only state in which we want to build a node. - nodeToBuild->accept(this); + case BuildGraphNode::Buildable: // This is the only state in which we want to build a node. + // TODO: It's a bit annoying that we have to check this here already, when we + // don't know whether the transformer needs to run at all. Investigate + // moving the whole job allocation logic to runTransformer(). + if (schedulingBlockedByJobLimit(nodeToBuild)) { + qCDebug(lcExec).noquote() << "node delayed due to occupied job pool:" + << nodeToBuild->toString(); + delayedLeaves.push_back(nodeToBuild); + } else { + nodeToBuild->accept(this); + } break; case BuildGraphNode::Building: qCDebug(lcExec).noquote() << nodeToBuild->toString(); @@ -370,9 +383,50 @@ bool Executor::scheduleJobs() break; } } + for (BuildGraphNode * const delayedLeaf : delayedLeaves) + m_leaves.push(delayedLeaf); return !m_leaves.empty() || !m_processingJobs.empty(); } +bool Executor::schedulingBlockedByJobLimit(const BuildGraphNode *node) +{ + if (node->type() != BuildGraphNode::ArtifactNodeType) + return false; + const Artifact * const artifact = static_cast<const Artifact *>(node); + if (artifact->artifactType == Artifact::SourceFile) + return false; + + const Transformer * const transformer = artifact->transformer.get(); + for (const QString &jobPool : transformer->jobPools()) { + const int currentJobCount = m_jobCountPerPool[jobPool]; + if (currentJobCount == 0) + continue; + const auto jobLimitIsExceeded = [currentJobCount, jobPool, this](const Transformer *t) { + const int maxJobCount = m_jobLimitsPerProduct.at(t->product().get()) + .getLimit(jobPool); + return maxJobCount > 0 && currentJobCount >= maxJobCount; + }; + + // Different products can set different limits. The effective limit is the minimum of what + // is set in this transformer's product and in the products of all currently + // running transformers. + if (jobLimitIsExceeded(transformer)) + return true; + for (const ExecutorJob * const runningJob : m_processingJobs.keys()) { + if (!runningJob->jobPools().contains(jobPool)) + continue; + const Transformer * const runningTransformer = runningJob->transformer(); + if (!runningTransformer) + continue; // This can happen if the ExecutorJob has already finished. + if (runningTransformer->product() == transformer->product()) + continue; // We have already checked this product's job limit. + if (jobLimitIsExceeded(runningTransformer)) + return true; + } + } + return false; +} + bool Executor::isUpToDate(Artifact *artifact) const { QBS_CHECK(artifact->artifactType == Artifact::Generated); @@ -391,11 +445,6 @@ bool Executor::isUpToDate(Artifact *artifact) const return false; } - if (artifact->knownOutOfDate) { - qCDebug(lcUpToDateCheck) << "Explicitly marked as out of date."; - return false; - } - for (Artifact *childArtifact : filterByType<Artifact>(artifact->children)) { QBS_CHECK(!childArtifact->alwaysUpdated || childArtifact->timestamp().isValid()); qCDebug(lcUpToDateCheck) << "child timestamp" @@ -407,13 +456,9 @@ bool Executor::isUpToDate(Artifact *artifact) const for (FileDependency *fileDependency : qAsConst(artifact->fileDependencies)) { if (!fileDependency->timestamp().isValid()) { - FileInfo fi(fileDependency->filePath()); - fileDependency->setTimestamp(fi.lastModified()); - if (!fileDependency->timestamp().isValid()) { - qCDebug(lcUpToDateCheck) << "file dependency doesn't exist" - << fileDependency->filePath(); - return false; - } + qCDebug(lcUpToDateCheck) << "file dependency doesn't exist" + << fileDependency->filePath(); + return false; } qCDebug(lcUpToDateCheck) << "file dependency timestamp" << fileDependency->timestamp().toString() @@ -429,6 +474,10 @@ bool Executor::mustExecuteTransformer(const TransformerPtr &transformer) const { if (transformer->alwaysRun) return true; + if (transformer->markedForRerun) { + qCDebug(lcUpToDateCheck) << "explicitly marked for re-run."; + return true; + } bool hasAlwaysUpdatedArtifacts = false; bool hasUpToDateNotAlwaysUpdatedArtifacts = false; @@ -488,70 +537,24 @@ void Executor::executeRuleNode(RuleNode *ruleNode) return; QBS_CHECK(!m_evalContext->engine()->isActive()); - ArtifactSet changedInputArtifacts; - if (ruleNode->rule()->isDynamic()) { - for (Artifact * const artifact : qAsConst(m_changedSourceArtifacts)) { - if (artifact->product != ruleNode->product) - continue; - if (artifact->isTargetOfModule()) - continue; - if (ruleNode->rule()->acceptsAsInput(artifact)) - changedInputArtifacts += artifact; - } - for (Artifact *artifact : filterByType<Artifact>(ruleNode->product->buildData->allNodes())) { - if (artifact->artifactType == Artifact::SourceFile) - continue; - if (ruleNode->rule()->acceptsAsInput(artifact)) { - for (const Artifact * const parent : artifact->parentArtifacts()) { - if (parent->transformer->rule != ruleNode->rule()) - continue; - if (!parent->alwaysUpdated) - continue; - if (parent->timestamp() < artifact->timestamp()) { - changedInputArtifacts += artifact; - break; - } - } - } - } - } RuleNode::ApplicationResult result; - ruleNode->apply(m_logger, changedInputArtifacts, m_productsByName, m_projectsByName, &result); + ruleNode->apply(m_logger, m_productsByName, m_projectsByName, &result); + updateLeaves(result.createdArtifacts); + updateLeaves(result.invalidatedArtifacts); + m_artifactsRemovedFromDisk << result.removedArtifacts; - if (result.upToDate) { - qCDebug(lcExec).noquote() << ruleNode->toString() << "is up to date. Skipping."; - } else { - qCDebug(lcExec).noquote() << ruleNode->toString(); - const WeakPointer<ResolvedProduct> &product = ruleNode->product; - Set<RuleNode *> parentRules; - if (!result.createdNodes.empty()) { - for (BuildGraphNode *parent : qAsConst(ruleNode->parents)) { - if (parent->type() == BuildGraphNode::RuleNodeType) - parentRules += static_cast<RuleNode *>(parent); - } - } - for (BuildGraphNode *node : qAsConst(result.createdNodes)) { - qCDebug(lcExec).noquote() << "rule created" << node->toString(); - Internal::connect(node, ruleNode); - if (node->type() != BuildGraphNode::ArtifactNodeType) - continue; - Artifact * const outputArtifact = static_cast<Artifact *>(node); - if (outputArtifact->fileTags().intersects(product->fileTags)) - product->buildData->addRootNode(outputArtifact); - - for (Artifact *inputArtifact : qAsConst(outputArtifact->transformer->inputs)) - Internal::connect(ruleNode, inputArtifact); - - for (RuleNode *parentRule : qAsConst(parentRules)) - Internal::connect(parentRule, outputArtifact); + if (m_progressObserver) { + const int transformerCount = ruleNode->transformerCount(); + if (transformerCount == 0) { + m_progressObserver->incrementProgressValue(); + } else { + m_pendingTransformersPerRule.insert(std::make_pair(ruleNode->rule().get(), + transformerCount)); } - updateLeaves(result.createdNodes); - updateLeaves(result.invalidatedNodes); } + finishNode(ruleNode); - if (m_progressObserver) - m_progressObserver->incrementProgressValue(); } void Executor::finishJob(ExecutorJob *job, bool success) @@ -564,14 +567,14 @@ void Executor::finishJob(ExecutorJob *job, bool success) const TransformerPtr transformer = it.value(); m_processingJobs.erase(it); m_availableJobs.push_back(job); + updateJobCounts(transformer.get(), -1); if (success) { m_project->buildData->setDirty(); for (Artifact * const artifact : qAsConst(transformer->outputs)) { if (artifact->alwaysUpdated) { artifact->setTimestamp(FileTime::currentTime()); - artifact->knownOutOfDate = false; for (Artifact * const parent : artifact->parentArtifacts()) - parent->knownOutOfDate = true; + parent->transformer->markedForRerun = true; if (m_buildOptions.forceOutputCheck() && !m_buildOptions.dryRun() && !FileInfo(artifact->filePath()).exists()) { if (transformer->rule) { @@ -692,6 +695,30 @@ bool Executor::transformerHasMatchingInputFiles(const TransformerConstPtr &trans return false; } +void Executor::setupJobLimits() +{ + Settings settings(m_buildOptions.settingsDirectory()); + for (const ResolvedProductConstPtr &p : m_productsToBuild) { + const Preferences prefs(&settings, p->profile()); + const JobLimits &jobLimitsFromSettings = prefs.jobLimits(); + JobLimits effectiveJobLimits; + if (m_buildOptions.projectJobLimitsTakePrecedence()) { + effectiveJobLimits.update(jobLimitsFromSettings).update(m_buildOptions.jobLimits()) + .update(p->jobLimits); + } else { + effectiveJobLimits.update(p->jobLimits).update(jobLimitsFromSettings) + .update(m_buildOptions.jobLimits()); + } + m_jobLimitsPerProduct.insert(std::make_pair(p.get(), effectiveJobLimits)); + } +} + +void Executor::updateJobCounts(const Transformer *transformer, int diff) +{ + for (const QString &jobPool : transformer->jobPools()) + m_jobCountPerPool[jobPool] += diff; +} + void Executor::cancelJobs() { if (m_state == ExecutorCanceling) @@ -754,7 +781,7 @@ void Executor::addExecutorJobs() } } -void Executor::rescueOldBuildData(Artifact *artifact, bool *childrenAdded = 0) +void Executor::rescueOldBuildData(Artifact *artifact, bool *childrenAdded = nullptr) { if (childrenAdded) *childrenAdded = false; @@ -823,9 +850,7 @@ void Executor::rescueOldBuildData(Artifact *artifact, bool *childrenAdded = 0) << "not in the project's list of dependencies anymore."; break; } - FileDependency * const dep = static_cast<FileDependency *>(*depIt); - dep->clearTimestamp(); - artifact->fileDependencies.insert(dep); + artifact->fileDependencies.insert(static_cast<FileDependency *>(*depIt)); } if (canRescue) { @@ -868,7 +893,8 @@ void Executor::rescueOldBuildData(Artifact *artifact, bool *childrenAdded = 0) artifact->transformer->lastPrepareScriptExecutionTime = rad.lastPrepareScriptExecutionTime; artifact->transformer->commandsNeedChangeTracking = true; artifact->setTimestamp(rad.timeStamp); - artifact->knownOutOfDate = artifact->knownOutOfDate || rad.knownOutOfDate; + artifact->transformer->markedForRerun + = artifact->transformer->markedForRerun || rad.knownOutOfDate; if (childrenAdded && !childrenToConnect.empty()) *childrenAdded = true; for (Artifact * const child : childrenToConnect) { @@ -986,15 +1012,25 @@ void Executor::runTransformer(const TransformerPtr &transformer) for (Artifact * const artifact : qAsConst(transformer->outputs)) artifact->buildState = BuildGraphNode::Building; m_processingJobs.insert(job, transformer); + updateJobCounts(transformer.get(), 1); job->run(transformer.get()); } void Executor::finishTransformer(const TransformerPtr &transformer) { + transformer->markedForRerun = false; for (Artifact * const artifact : qAsConst(transformer->outputs)) { possiblyInstallArtifact(artifact); finishArtifact(artifact); } + if (m_progressObserver) { + const auto it = m_pendingTransformersPerRule.find(transformer->rule.get()); + QBS_CHECK(it != m_pendingTransformersPerRule.cend()); + if (--it->second == 0) { + m_progressObserver->incrementProgressValue(); + m_pendingTransformersPerRule.erase(it); + } + } } void Executor::possiblyInstallArtifact(const Artifact *artifact) @@ -1013,7 +1049,7 @@ void Executor::possiblyInstallArtifact(const Artifact *artifact) void Executor::onJobFinished(const qbs::ErrorInfo &err) { try { - ExecutorJob * const job = qobject_cast<ExecutorJob *>(sender()); + auto const job = qobject_cast<ExecutorJob *>(sender()); QBS_CHECK(job); if (m_evalContext->engine()->isActive()) { qCDebug(lcExec) << "Executor job finished while rule execution is pausing. " @@ -1178,7 +1214,9 @@ void Executor::syncFileDependencies() Set<FileDependency *> &globalFileDepList = m_project->buildData->fileDependencies; for (auto it = globalFileDepList.begin(); it != globalFileDepList.end(); ) { FileDependency * const dep = *it; - if (FileInfo(dep->filePath()).exists()) { + FileInfo fi(dep->filePath()); + if (fi.exists()) { + dep->setTimestamp(fi.lastModified()); ++it; continue; } @@ -1203,6 +1241,7 @@ void Executor::syncFileDependencies() it = globalFileDepList.erase(it); delete dep; } else { + dep->clearTimestamp(); ++it; } } @@ -1214,19 +1253,9 @@ void Executor::prepareArtifact(Artifact *artifact) artifact->timestampRetrieved = false; if (artifact->artifactType == Artifact::SourceFile) { - const FileTime oldTimestamp = artifact->timestamp(); retrieveSourceFileTimestamp(artifact); - if (oldTimestamp != artifact->timestamp()) - m_changedSourceArtifacts.push_back(artifact); possiblyInstallArtifact(artifact); } - - // Timestamps of file dependencies must be invalid for every build. - // TODO: These should be a subset of ProjectBuildData::fileDependencies, so clear the - // timestamps in syncFileDepencencies() instead. - // TODO: Verify this assumption in the sanity checks. - for (FileDependency * const fileDependency : qAsConst(artifact->fileDependencies)) - fileDependency->clearTimestamp(); } void Executor::setupForBuildingSelectedFiles(const BuildGraphNode *node) diff --git a/src/lib/corelib/buildgraph/executor.h b/src/lib/corelib/buildgraph/executor.h index a73e5f812..2756a7120 100644 --- a/src/lib/corelib/buildgraph/executor.h +++ b/src/lib/corelib/buildgraph/executor.h @@ -99,8 +99,8 @@ private: void checkForCancellation(); // BuildGraphVisitor implementation - bool visit(Artifact *artifact); - bool visit(RuleNode *ruleNode); + bool visit(Artifact *artifact) override; + bool visit(RuleNode *ruleNode) override; enum ExecutorState { ExecutorIdle, ExecutorRunning, ExecutorCanceling }; @@ -154,6 +154,10 @@ private: bool artifactHasMatchingOutputTags(const Artifact *artifact) const; bool transformerHasMatchingInputFiles(const TransformerConstPtr &transformer) const; + void setupJobLimits(); + void updateJobCounts(const Transformer *transformer, int diff); + bool schedulingBlockedByJobLimit(const BuildGraphNode *node); + typedef QHash<ExecutorJob *, TransformerPtr> JobMap; JobMap m_processingJobs; @@ -169,9 +173,11 @@ private: std::vector<ResolvedProductPtr> m_allProducts; std::unordered_map<QString, const ResolvedProduct *> m_productsByName; std::unordered_map<QString, const ResolvedProject *> m_projectsByName; + std::unordered_map<QString, int> m_jobCountPerPool; + std::unordered_map<const ResolvedProduct *, JobLimits> m_jobLimitsPerProduct; + std::unordered_map<const Rule *, int> m_pendingTransformersPerRule; NodeSet m_roots; Leaves m_leaves; - QList<Artifact *> m_changedSourceArtifacts; InputArtifactScannerContext *m_inputArtifactScanContext; ErrorInfo m_error; bool m_explicitlyCanceled; diff --git a/src/lib/corelib/buildgraph/executorjob.cpp b/src/lib/corelib/buildgraph/executorjob.cpp index e90b498f6..79f17377d 100644 --- a/src/lib/corelib/buildgraph/executorjob.cpp +++ b/src/lib/corelib/buildgraph/executorjob.cpp @@ -113,6 +113,7 @@ void ExecutorJob::run(Transformer *t) m_processCommandExecutor->setProcessEnvironment( (*t->outputs.cbegin())->product->buildEnvironment); m_transformer = t; + m_jobPools = t->jobPools(); runNextCommand(); } @@ -171,6 +172,7 @@ void ExecutorJob::setFinished() void ExecutorJob::reset() { m_transformer = nullptr; + m_jobPools.clear(); m_currentCommandExecutor = nullptr; m_currentCommandIdx = -1; m_error.clear(); diff --git a/src/lib/corelib/buildgraph/executorjob.h b/src/lib/corelib/buildgraph/executorjob.h index b1701a11d..e28d42f7f 100644 --- a/src/lib/corelib/buildgraph/executorjob.h +++ b/src/lib/corelib/buildgraph/executorjob.h @@ -43,8 +43,10 @@ #include <language/forward_decls.h> #include <tools/commandechomode.h> #include <tools/error.h> +#include <tools/set.h> #include <QtCore/qobject.h> +#include <QtCore/qstring.h> namespace qbs { class CodeLocation; @@ -71,6 +73,8 @@ public: void setEchoMode(CommandEchoMode echoMode); void run(Transformer *t); void cancel(); + const Transformer *transformer() const { return m_transformer; } + Set<QString> jobPools() const { return m_jobPools; } signals: void reportCommandDescription(const QString &highlight, const QString &message); @@ -88,6 +92,7 @@ private: ProcessCommandExecutor *m_processCommandExecutor; JsCommandExecutor *m_jsCommandExecutor; Transformer *m_transformer; + Set<QString> m_jobPools; int m_currentCommandIdx; ErrorInfo m_error; }; diff --git a/src/lib/corelib/buildgraph/forward_decls.h b/src/lib/corelib/buildgraph/forward_decls.h index 4f84041af..f39c9eb6f 100644 --- a/src/lib/corelib/buildgraph/forward_decls.h +++ b/src/lib/corelib/buildgraph/forward_decls.h @@ -48,6 +48,7 @@ class Artifact; class BuildGraphNode; class ProjectBuildData; class ProductBuildData; +class RuleNode; class Transformer; typedef std::shared_ptr<Transformer> TransformerPtr; diff --git a/src/lib/corelib/buildgraph/inputartifactscanner.cpp b/src/lib/corelib/buildgraph/inputartifactscanner.cpp index e96f6fdde..143c99fb5 100644 --- a/src/lib/corelib/buildgraph/inputartifactscanner.cpp +++ b/src/lib/corelib/buildgraph/inputartifactscanner.cpp @@ -84,7 +84,7 @@ static void resolveDepencency(const RawScannedDependency &dependency, fileDependencyArtifact = static_cast<FileDependency *>(lookupResult); break; case FileResourceBase::FileTypeArtifact: { - Artifact * const foundArtifact = static_cast<Artifact *>(lookupResult); + auto const foundArtifact = static_cast<Artifact *>(lookupResult); if (foundArtifact->product == product) dependencyInProduct = foundArtifact; else @@ -170,7 +170,7 @@ void InputArtifactScanner::scanForFileDependencies(Artifact *inputArtifact) for (DependencyScanner * const scanner : scanners) { scanForScannerFileDependencies(scanner, inputArtifact, fileToBeScanned, - scanner->recursive() ? &filesToScan : 0, cacheItem[scanner->key()]); + scanner->recursive() ? &filesToScan : nullptr, cacheItem[scanner->key()]); } } } @@ -277,8 +277,7 @@ resolved: if (artifactsToScan && resolvedDependency.file) { if (resolvedDependency.file->fileType() == FileResourceBase::FileTypeArtifact) { // Do not scan an artifact that is not built yet: Its contents might still change. - Artifact * const artifactDependency - = static_cast<Artifact *>(resolvedDependency.file); + auto const artifactDependency = static_cast<Artifact *>(resolvedDependency.file); if (artifactDependency->artifactType == Artifact::SourceFile || artifactDependency->buildState == BuildGraphNode::Built) { artifactsToScan->push_back(artifactDependency); diff --git a/src/lib/corelib/buildgraph/jscommandexecutor.h b/src/lib/corelib/buildgraph/jscommandexecutor.h index ec9d43147..207e18097 100644 --- a/src/lib/corelib/buildgraph/jscommandexecutor.h +++ b/src/lib/corelib/buildgraph/jscommandexecutor.h @@ -64,9 +64,9 @@ signals: private: void onJavaScriptCommandFinished(); - void doReportCommandDescription(const QString &productName); - void doStart(); - void cancel(); + void doReportCommandDescription(const QString &productName) override; + void doStart() override; + void cancel() override; void waitForFinished(); diff --git a/src/lib/corelib/buildgraph/processcommandexecutor.cpp b/src/lib/corelib/buildgraph/processcommandexecutor.cpp index cee45bcd0..0818e1200 100644 --- a/src/lib/corelib/buildgraph/processcommandexecutor.cpp +++ b/src/lib/corelib/buildgraph/processcommandexecutor.cpp @@ -197,7 +197,7 @@ void ProcessCommandExecutor::doStart() void ProcessCommandExecutor::cancel() { // We don't want this command to be reported as failing, since we explicitly terminated it. - disconnect(this, &ProcessCommandExecutor::reportProcessResult, 0, 0); + disconnect(this, &ProcessCommandExecutor::reportProcessResult, nullptr, nullptr); m_process.cancel(); } diff --git a/src/lib/corelib/buildgraph/processcommandexecutor.h b/src/lib/corelib/buildgraph/processcommandexecutor.h index bebe3a5e2..67eb9f746 100644 --- a/src/lib/corelib/buildgraph/processcommandexecutor.h +++ b/src/lib/corelib/buildgraph/processcommandexecutor.h @@ -69,10 +69,10 @@ private: void onProcessError(); void onProcessFinished(); - void doSetup(); - void doReportCommandDescription(const QString &productName); - void doStart(); - void cancel(); + void doSetup() override; + void doReportCommandDescription(const QString &productName) override; + void doStart() override; + void cancel() override; void startProcessCommand(); QString filterProcessOutput(const QByteArray &output, const QString &filterFunctionSource); diff --git a/src/lib/corelib/buildgraph/productbuilddata.cpp b/src/lib/corelib/buildgraph/productbuilddata.cpp index 7445c93cb..db51b2b9f 100644 --- a/src/lib/corelib/buildgraph/productbuilddata.cpp +++ b/src/lib/corelib/buildgraph/productbuilddata.cpp @@ -100,27 +100,6 @@ void ProductBuildData::addFileTagToArtifact(Artifact *artifact, const FileTag &t m_jsArtifactsMapUpToDate = false; } -void ProductBuildData::addArtifactWithChangedInputsForRule(const RuleConstPtr &rule, - Artifact *artifact) -{ - m_artifactsWithChangedInputsPerRule[rule] += artifact; -} - -void ProductBuildData::removeArtifactWithChangedInputsForRule(const RuleConstPtr &rule, Artifact *artifact) -{ - m_artifactsWithChangedInputsPerRule[rule] -= artifact; -} - -void ProductBuildData::removeAllArtifactsWithChangedInputsForRule(const RuleConstPtr &rule) -{ - m_artifactsWithChangedInputsPerRule.remove(rule); -} - -bool ProductBuildData::ruleHasArtifactWithChangedInputs(const RuleConstPtr &rule) const -{ - return !m_artifactsWithChangedInputsPerRule.value(rule).empty(); -} - ArtifactSetByFileTag ProductBuildData::artifactsByFileTag() const { std::lock_guard<std::mutex> l(m_artifactsMapMutex); diff --git a/src/lib/corelib/buildgraph/productbuilddata.h b/src/lib/corelib/buildgraph/productbuilddata.h index 553f79159..bcc500de8 100644 --- a/src/lib/corelib/buildgraph/productbuilddata.h +++ b/src/lib/corelib/buildgraph/productbuilddata.h @@ -68,15 +68,12 @@ public: void addNode(BuildGraphNode *node) { m_nodes.insert(node); } void addRootNode(BuildGraphNode *node) { m_roots.insert(node); } + void removeFromRootNodes(BuildGraphNode *node) { m_roots.remove(node); } void addArtifact(Artifact *artifact); void addArtifactToSet(Artifact *artifact); void removeArtifact(Artifact *artifact); void removeArtifactFromSetByFileTag(Artifact *artifact, const FileTag &fileTag); void addFileTagToArtifact(Artifact *artifact, const FileTag &tag); - void addArtifactWithChangedInputsForRule(const RuleConstPtr &rule, Artifact *artifact); - void removeArtifactWithChangedInputsForRule(const RuleConstPtr &rule, Artifact *artifact); - void removeAllArtifactsWithChangedInputsForRule(const RuleConstPtr &rule); - bool ruleHasArtifactWithChangedInputs(const RuleConstPtr &rule) const; ArtifactSetByFileTag artifactsByFileTag() const; @@ -93,7 +90,7 @@ public: template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool) { pool.serializationOp<opType>(m_nodes, m_roots, m_rescuableArtifactData, - m_artifactsByFileTag, m_artifactsWithChangedInputsPerRule); + m_artifactsByFileTag); } private: @@ -114,9 +111,6 @@ private: ArtifactSetByFileTag m_artifactsByFileTag; mutable std::mutex m_artifactsMapMutex; - typedef QHash<RuleConstPtr, ArtifactSet> ArtifactSetByRule; - ArtifactSetByRule m_artifactsWithChangedInputsPerRule; - bool m_jsArtifactsMapUpToDate = true; }; diff --git a/src/lib/corelib/buildgraph/projectbuilddata.cpp b/src/lib/corelib/buildgraph/projectbuilddata.cpp index 88de0f392..ccb6c5123 100644 --- a/src/lib/corelib/buildgraph/projectbuilddata.cpp +++ b/src/lib/corelib/buildgraph/projectbuilddata.cpp @@ -167,12 +167,11 @@ static void disconnectArtifactParents(Artifact *artifact) parent->children.remove(artifact); if (parent->type() != BuildGraphNode::ArtifactNodeType) continue; - Artifact * const parentArtifact = static_cast<Artifact *>(parent); + auto const parentArtifact = static_cast<Artifact *>(parent); QBS_CHECK(parentArtifact->transformer); parentArtifact->childrenAddedByScanner.remove(artifact); parentArtifact->transformer->inputs.remove(artifact); parentArtifact->transformer->explicitlyDependsOn.remove(artifact); - parentArtifact->product->registerArtifactWithChangedInputs(parentArtifact); } artifact->parents.clear(); @@ -205,7 +204,6 @@ void ProjectBuildData::removeArtifactAndExclusiveDependents(Artifact *artifact, if (parent->children.empty()) { removeParent = true; } else if (parent->transformer) { - parent->product->registerArtifactWithChangedInputs(parent); parent->transformer->inputs.remove(artifact); removeParent = parent->transformer->inputs.empty(); } @@ -222,18 +220,8 @@ void ProjectBuildData::removeArtifactAndExclusiveDependents(Artifact *artifact, static void removeFromRuleNodes(Artifact *artifact) { - for (const ResolvedProductPtr &product : artifact->product->topLevelProject()->allProducts()) { - if (!product->buildData) - continue; - for (BuildGraphNode *n : qAsConst(product->buildData->allNodes())) { - if (n->type() != BuildGraphNode::RuleNodeType) - continue; - const auto ruleNode = static_cast<RuleNode *>(n); - qCDebug(lcBuildGraph) << "remove old input" << artifact->filePath() - << "from rule" << ruleNode->rule()->toString(); - ruleNode->removeOldInputArtifact(artifact); - } - } + for (RuleNode * const ruleNode : filterByType<RuleNode>(artifact->parents)) + ruleNode->removeOldInputArtifact(artifact); } void ProjectBuildData::removeArtifact(Artifact *artifact, @@ -245,10 +233,8 @@ void ProjectBuildData::removeArtifact(Artifact *artifact, removeFromLookupTable(artifact); removeFromRuleNodes(artifact); disconnectArtifact(artifact); - if (artifact->transformer) { - artifact->product->unregisterArtifactWithChangedInputs(artifact); + if (artifact->transformer) artifact->transformer->outputs.remove(artifact); - } if (removeFromProduct) artifact->product->buildData->removeArtifact(artifact); m_isDirty = false; diff --git a/src/lib/corelib/buildgraph/projectbuilddata.h b/src/lib/corelib/buildgraph/projectbuilddata.h index e63922bae..0df303c0b 100644 --- a/src/lib/corelib/buildgraph/projectbuilddata.h +++ b/src/lib/corelib/buildgraph/projectbuilddata.h @@ -62,7 +62,7 @@ class ScriptEngine; class QBS_AUTOTEST_EXPORT ProjectBuildData { public: - ProjectBuildData(const ProjectBuildData *other = 0); + ProjectBuildData(const ProjectBuildData *other = nullptr); ~ProjectBuildData(); static QString deriveBuildGraphFilePath(const QString &buildDir, const QString &projectId); @@ -75,7 +75,7 @@ public: QList<FileResourceBase *> lookupFiles(const Artifact *artifact) const; void insertFileDependency(FileDependency *dependency); void removeArtifactAndExclusiveDependents(Artifact *artifact, const Logger &logger, - bool removeFromProduct = true, ArtifactSet *removedArtifacts = 0); + bool removeFromProduct = true, ArtifactSet *removedArtifacts = nullptr); void removeArtifact(Artifact *artifact, const Logger &logger, bool removeFromDisk = true, bool removeFromProduct = true); diff --git a/src/lib/corelib/buildgraph/qtmocscanner.cpp b/src/lib/corelib/buildgraph/qtmocscanner.cpp index a2e9786aa..8c165cb80 100644 --- a/src/lib/corelib/buildgraph/qtmocscanner.cpp +++ b/src/lib/corelib/buildgraph/qtmocscanner.cpp @@ -219,8 +219,6 @@ QScriptValue QtMocScanner::apply(QScriptEngine *engine, const Artifact *artifact m_cppScanner = scanners.front(); } - findIncludedMocCppFiles(); - qCDebug(lcMocScan).noquote() << "scanning" << artifact->toString(); bool hasQObjectMacro = false; @@ -237,6 +235,7 @@ QScriptValue QtMocScanner::apply(QScriptEngine *engine, const Artifact *artifact hasQObjectMacro = true; hasPluginMetaDataMacro = true; } + findIncludedMocCppFiles(); if (!m_includedMocCppFiles.contains(FileInfo::completeBaseName(artifact->fileName()))) mustCompile = true; } else { @@ -257,6 +256,7 @@ QScriptValue QtMocScanner::apply(QScriptEngine *engine, const Artifact *artifact obj.setProperty(QStringLiteral("hasQObjectMacro"), hasQObjectMacro); obj.setProperty(QStringLiteral("mustCompile"), mustCompile); obj.setProperty(QStringLiteral("hasPluginMetaDataMacro"), hasPluginMetaDataMacro); + static_cast<ScriptEngine *>(engine)->setUsesIo(); return obj; } diff --git a/src/lib/corelib/buildgraph/rescuableartifactdata.h b/src/lib/corelib/buildgraph/rescuableartifactdata.h index a88d63273..068c543ed 100644 --- a/src/lib/corelib/buildgraph/rescuableartifactdata.h +++ b/src/lib/corelib/buildgraph/rescuableartifactdata.h @@ -104,7 +104,6 @@ public: FileTime timeStamp; std::vector<ChildData> children; std::vector<QString> fileDependencies; - bool knownOutOfDate = false; // Per-Transformer data CommandList commands; @@ -122,6 +121,7 @@ public: FileTime lastCommandExecutionTime; std::unordered_map<QString, ExportedModule> exportedModulesAccessedInPrepareScript; std::unordered_map<QString, ExportedModule> exportedModulesAccessedInCommands; + bool knownOutOfDate = false; // Only needed for API purposes FileTags fileTags; diff --git a/src/lib/corelib/buildgraph/rulecommands.cpp b/src/lib/corelib/buildgraph/rulecommands.cpp index c2aad1ce1..a15047290 100644 --- a/src/lib/corelib/buildgraph/rulecommands.cpp +++ b/src/lib/corelib/buildgraph/rulecommands.cpp @@ -103,6 +103,7 @@ bool AbstractCommand::equals(const AbstractCommand *other) const && m_highlight == other->m_highlight && m_ignoreDryRun == other->m_ignoreDryRun && m_silent == other->m_silent + && m_jobPool == other->m_jobPool && m_properties == other->m_properties; } @@ -113,6 +114,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const m_highlight = scriptValue->property(highlightProperty()).toString(); m_ignoreDryRun = scriptValue->property(ignoreDryRunProperty()).toBool(); m_silent = scriptValue->property(silentProperty()).toBool(); + m_jobPool = scriptValue->property(StringConstants::jobPoolProperty()).toString(); m_codeLocation = codeLocation; m_predefinedProperties @@ -120,6 +122,7 @@ void AbstractCommand::fillFromScriptValue(const QScriptValue *scriptValue, const << extendedDescriptionProperty() << highlightProperty() << ignoreDryRunProperty() + << StringConstants::jobPoolProperty() << silentProperty(); } @@ -146,7 +149,9 @@ void AbstractCommand::applyCommandProperties(const QScriptValue *scriptValue) if (m_predefinedProperties.contains(it.name())) continue; const QVariant value = it.value().toVariant(); - if (QMetaType::Type(value.type()) == QMetaType::QObjectStar) { + if (QMetaType::Type(value.type()) == QMetaType::QObjectStar + || it.value().scriptClass() + || it.value().data().isValid()) { throw ErrorInfo(Tr::tr("Property '%1' has a type unsuitable for storing in a command " "object.").arg(it.name()), m_codeLocation); } @@ -392,8 +397,7 @@ bool JavaScriptCommand::equals(const AbstractCommand *otherAbstractCommand) cons { if (!AbstractCommand::equals(otherAbstractCommand)) return false; - const JavaScriptCommand * const other - = static_cast<const JavaScriptCommand *>(otherAbstractCommand); + auto const other = static_cast<const JavaScriptCommand *>(otherAbstractCommand); return m_sourceCode == other->m_sourceCode; } diff --git a/src/lib/corelib/buildgraph/rulecommands.h b/src/lib/corelib/buildgraph/rulecommands.h index 7583de2b6..4e56359f4 100644 --- a/src/lib/corelib/buildgraph/rulecommands.h +++ b/src/lib/corelib/buildgraph/rulecommands.h @@ -81,6 +81,7 @@ public: const QString highlight() const { return m_highlight; } bool ignoreDryRun() const { return m_ignoreDryRun; } bool isSilent() const { return m_silent; } + QString jobPool() const { return m_jobPool; } CodeLocation codeLocation() const { return m_codeLocation; } const QVariantMap &properties() const { return m_properties; } @@ -98,7 +99,8 @@ private: template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) { pool.serializationOp<opType>(m_description, m_extendedDescription, m_highlight, - m_ignoreDryRun, m_silent, m_codeLocation, m_properties); + m_ignoreDryRun, m_silent, m_codeLocation, m_jobPool, + m_properties); } QString m_description; @@ -107,6 +109,7 @@ private: bool m_ignoreDryRun; bool m_silent; CodeLocation m_codeLocation; + QString m_jobPool; QVariantMap m_properties; }; @@ -116,9 +119,10 @@ public: static ProcessCommandPtr create() { return ProcessCommandPtr(new ProcessCommand); } static void setupForJavaScript(QScriptValue targetObject); - CommandType type() const { return ProcessCommandType; } - bool equals(const AbstractCommand *otherAbstractCommand) const; - void fillFromScriptValue(const QScriptValue *scriptValue, const CodeLocation &codeLocation); + CommandType type() const override { return ProcessCommandType; } + bool equals(const AbstractCommand *otherAbstractCommand) const override; + void fillFromScriptValue(const QScriptValue *scriptValue, + const CodeLocation &codeLocation) override; const QString program() const { return m_program; } const QStringList arguments() const { return m_arguments; } const QString workingDir() const { return m_workingDir; } @@ -136,8 +140,8 @@ public: QString stdoutFilePath() const { return m_stdoutFilePath; } QString stderrFilePath() const { return m_stderrFilePath; } - void load(PersistentPool &pool); - void store(PersistentPool &pool); + void load(PersistentPool &pool) override; + void store(PersistentPool &pool) override; private: ProcessCommand(); @@ -178,16 +182,17 @@ public: static JavaScriptCommandPtr create() { return JavaScriptCommandPtr(new JavaScriptCommand); } static void setupForJavaScript(QScriptValue targetObject); - virtual CommandType type() const { return JavaScriptCommandType; } - bool equals(const AbstractCommand *otherAbstractCommand) const; - void fillFromScriptValue(const QScriptValue *scriptValue, const CodeLocation &codeLocation); + CommandType type() const override { return JavaScriptCommandType; } + bool equals(const AbstractCommand *otherAbstractCommand) const override; + void fillFromScriptValue(const QScriptValue *scriptValue, + const CodeLocation &codeLocation) override; const QString &scopeName() const { return m_scopeName; } const QString &sourceCode() const { return m_sourceCode; } void setSourceCode(const QString &str) { m_sourceCode = str; } - void load(PersistentPool &pool); - void store(PersistentPool &pool); + void load(PersistentPool &pool) override; + void store(PersistentPool &pool) override; private: JavaScriptCommand(); diff --git a/src/lib/corelib/buildgraph/rulenode.cpp b/src/lib/corelib/buildgraph/rulenode.cpp index 27f2d2c55..516b9b830 100644 --- a/src/lib/corelib/buildgraph/rulenode.cpp +++ b/src/lib/corelib/buildgraph/rulenode.cpp @@ -78,15 +78,22 @@ QString RuleNode::toString() const + QLatin1String(" located at ") + m_rule->prepareScript.location().toString(); } -void RuleNode::apply(const Logger &logger, const ArtifactSet &changedInputs, +void RuleNode::apply(const Logger &logger, const std::unordered_map<QString, const ResolvedProduct *> &productsByName, const std::unordered_map<QString, const ResolvedProject *> &projectsByName, ApplicationResult *result) { ArtifactSet allCompatibleInputs = currentInputArtifacts(); + const ArtifactSet explicitlyDependsOn + = RulesApplicator::collectExplicitlyDependsOn(m_rule.get(), product.get()); + const ArtifactSet auxiliaryInputs + = RulesApplicator::collectAuxiliaryInputs(m_rule.get(), product.get()); const ArtifactSet addedInputs = allCompatibleInputs - m_oldInputArtifacts; const ArtifactSet removedInputs = m_oldInputArtifacts - allCompatibleInputs; - result->upToDate = changedInputs.empty() && addedInputs.empty() && removedInputs.empty(); + const ArtifactSet changedInputs = changedInputArtifacts(allCompatibleInputs, + explicitlyDependsOn, + auxiliaryInputs); + bool upToDate = changedInputs.empty() && addedInputs.empty() && removedInputs.empty(); qCDebug(lcBuildGraph).noquote().nospace() << "consider " << (m_rule->isDynamic() ? "dynamic " : "") @@ -98,13 +105,6 @@ void RuleNode::apply(const Logger &logger, const ArtifactSet &changedInputs, << "\n\tremoved: " << removedInputs.toString(); ArtifactSet inputs = changedInputs; - if (product->isMarkedForReapplication(m_rule)) { - QBS_CHECK(m_rule->multiplex); - result->upToDate = false; - product->unmarkForReapplication(m_rule); - qCDebug(lcBuildGraph) << "rule is marked for reapplication " << m_rule->toString(); - } - if (m_rule->multiplex) inputs = allCompatibleInputs; else @@ -117,7 +117,7 @@ void RuleNode::apply(const Logger &logger, const ArtifactSet &changedInputs, if (prepareScriptNeedsRerun(output->transformer.get(), output->transformer->product().get(), productsByName, projectsByName)) { - result->upToDate = false; + upToDate = false; inputs += input; } break; @@ -128,8 +128,7 @@ void RuleNode::apply(const Logger &logger, const ArtifactSet &changedInputs, // Handle rules without inputs: We want to run such a rule if and only if it has not run yet // or its transformer is not up to date regarding the prepare script. - if (result->upToDate && (!m_rule->declaresInputs() || !m_rule->requiresInputs) - && inputs.empty()) { + if (upToDate && (!m_rule->declaresInputs() || !m_rule->requiresInputs) && inputs.empty()) { bool hasOutputs = false; for (const Artifact * const output : filterByType<Artifact>(parents)) { if (output->transformer->rule != m_rule) @@ -138,46 +137,77 @@ void RuleNode::apply(const Logger &logger, const ArtifactSet &changedInputs, if (prepareScriptNeedsRerun(output->transformer.get(), output->transformer->product().get(), productsByName, projectsByName)) { - result->upToDate = false; + upToDate = false; break; } if (m_rule->multiplex) break; } if (!hasOutputs) - result->upToDate = false; + upToDate = false; } - if (result->upToDate) + if (upToDate) { + qCDebug(lcExec) << "rule is up to date. Skipping."; return; - if (!removedInputs.empty()) { - ArtifactSet outputArtifactsToRemove; - for (Artifact * const artifact : removedInputs) { - for (Artifact *parent : filterByType<Artifact>(artifact->parents)) { - if (parent->transformer->rule != m_rule) { - // parent was not created by our rule. - continue; - } - - // parent must always have a transformer, because it's generated. - QBS_CHECK(parent->transformer); - - // artifact is a former input of m_rule and parent was created by m_rule - // the inputs of the transformer must contain artifact - QBS_CHECK(parent->transformer->inputs.contains(artifact)); + } - outputArtifactsToRemove += parent; + const bool mustApplyRule = !inputs.empty() || !m_rule->declaresInputs() + || !m_rule->requiresInputs; + + // For a non-multiplex rule, the removal of an input always implies that the + // corresponding outputs disappear. + // For a multiplex rule, the outputs disappear only if *all* inputs are gone *and* + // the rule requires inputs. This is exactly the opposite condition of whether to + // re-apply the rule. + const bool removedInputForcesOutputRemoval = !m_rule->multiplex || !mustApplyRule; + ArtifactSet outputArtifactsToRemove; + std::vector<std::pair<Artifact *, Artifact *>> connectionsToBreak; + for (Artifact * const artifact : removedInputs) { + if (!artifact) // dummy artifact + continue; + for (Artifact *parent : filterByType<Artifact>(artifact->parents)) { + if (parent->transformer->rule != m_rule) { + // parent was not created by our rule. + continue; } + + // parent must always have a transformer, because it's generated. + QBS_CHECK(parent->transformer); + + // artifact is a former input of m_rule and parent was created by m_rule + // the inputs of the transformer must contain artifact + QBS_CHECK(parent->transformer->inputs.contains(artifact)); + + if (removedInputForcesOutputRemoval) + outputArtifactsToRemove += parent; + else + connectionsToBreak.push_back(std::make_pair(parent, artifact)); } - RulesApplicator::handleRemovedRuleOutputs(inputs, outputArtifactsToRemove, logger); + disconnect(this, artifact); + } + for (const auto &connection : connectionsToBreak) + disconnect(connection.first, connection.second); + if (!outputArtifactsToRemove.empty()) { + RulesApplicator::handleRemovedRuleOutputs(inputs, outputArtifactsToRemove, + result->removedArtifacts, logger); } - if (!inputs.empty() || !m_rule->declaresInputs() || !m_rule->requiresInputs) { + + if (mustApplyRule) { RulesApplicator applicator(product.lock(), productsByName, projectsByName, logger); - applicator.applyRule(m_rule, inputs); - result->createdNodes = applicator.createdArtifacts(); - result->invalidatedNodes = applicator.invalidatedArtifacts(); - m_oldInputArtifacts.unite(inputs); + applicator.applyRule(this, inputs, explicitlyDependsOn); + result->createdArtifacts = applicator.createdArtifacts(); + result->invalidatedArtifacts = applicator.invalidatedArtifacts(); + m_lastApplicationTime = FileTime::currentTime(); + if (applicator.ruleUsesIo()) + m_needsToConsiderChangedInputs = true; + } else { + qCDebug(lcExec).noquote() << "prepare script does not need to run"; } + m_oldInputArtifacts = allCompatibleInputs; + m_oldExplicitlyDependsOn = explicitlyDependsOn; + m_oldAuxiliaryInputs = auxiliaryInputs; + product->topLevelProject()->buildData->setDirty(); } void RuleNode::load(PersistentPool &pool) @@ -192,6 +222,14 @@ void RuleNode::store(PersistentPool &pool) serializationOp<PersistentPool::Store>(pool); } +int RuleNode::transformerCount() const +{ + Set<const Transformer *> transformers; + for (const Artifact * const output : filterByType<Artifact>(parents)) + transformers.insert(output->transformer.get()); + return transformers.size(); +} + ArtifactSet RuleNode::currentInputArtifacts() const { ArtifactSet s; @@ -237,5 +275,51 @@ ArtifactSet RuleNode::currentInputArtifacts() const return s; } +ArtifactSet RuleNode::changedInputArtifacts(const ArtifactSet &allCompatibleInputs, + const ArtifactSet &explicitlyDependsOn, + const ArtifactSet &auxiliaryInputs) const +{ + ArtifactSet changedInputArtifacts; + if (explicitlyDependsOn != m_oldExplicitlyDependsOn) + return allCompatibleInputs; + if (!m_needsToConsiderChangedInputs) + return changedInputArtifacts; + + for (Artifact * const artifact : explicitlyDependsOn) { + if (artifact->timestamp() > m_lastApplicationTime) + return allCompatibleInputs; + } + if (auxiliaryInputs != m_oldAuxiliaryInputs) + return allCompatibleInputs; + for (Artifact * const artifact : auxiliaryInputs) { + if (artifact->timestamp() > m_lastApplicationTime) + return allCompatibleInputs; + } + for (Artifact * const artifact : allCompatibleInputs) { + if (artifact->timestamp() > m_lastApplicationTime) + changedInputArtifacts.insert(artifact); + } + return changedInputArtifacts; +} + +void RuleNode::removeOldInputArtifact(Artifact *artifact) +{ + if (m_oldInputArtifacts.remove(artifact)) { + qCDebug(lcBuildGraph) << "remove old input" << artifact->filePath() + << "from rule" << rule()->toString(); + m_oldInputArtifacts.insert(nullptr); + } + if (m_oldExplicitlyDependsOn.remove(artifact)) { + qCDebug(lcBuildGraph) << "remove old explicitlyDependsOn" << artifact->filePath() + << "from rule" << rule()->toString(); + m_oldExplicitlyDependsOn.insert(nullptr); + } + if (m_oldAuxiliaryInputs.remove(artifact)) { + qCDebug(lcBuildGraph) << "remove old auxiliaryInput" << artifact->filePath() + << "from rule" << rule()->toString(); + m_oldAuxiliaryInputs.insert(nullptr); + } +} + } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/buildgraph/rulenode.h b/src/lib/corelib/buildgraph/rulenode.h index b502b7ac8..0585678ec 100644 --- a/src/lib/corelib/buildgraph/rulenode.h +++ b/src/lib/corelib/buildgraph/rulenode.h @@ -63,36 +63,52 @@ public: void setRule(const RuleConstPtr &rule) { m_rule = rule; } const RuleConstPtr &rule() const { return m_rule; } - Type type() const { return RuleNodeType; } - void accept(BuildGraphVisitor *visitor); - QString toString() const; + Type type() const override { return RuleNodeType; } + void accept(BuildGraphVisitor *visitor) override; + QString toString() const override; struct ApplicationResult { - bool upToDate; - NodeSet createdNodes; - NodeSet invalidatedNodes; + NodeSet createdArtifacts; + NodeSet invalidatedArtifacts; + QStringList removedArtifacts; }; - void apply(const Logger &logger, const ArtifactSet &changedInputs, + void apply(const Logger &logger, const std::unordered_map<QString, const ResolvedProduct *> &productsByName, const std::unordered_map<QString, const ResolvedProject *> &projectsByName, ApplicationResult *result); - void removeOldInputArtifact(Artifact *artifact) { m_oldInputArtifacts.remove(artifact); } + void removeOldInputArtifact(Artifact *artifact); - void load(PersistentPool &pool); - void store(PersistentPool &pool); + void load(PersistentPool &pool) override; + void store(PersistentPool &pool) override; + + int transformerCount() const; private: template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) { - pool.serializationOp<opType>(m_rule, m_oldInputArtifacts); + pool.serializationOp<opType>(m_rule, m_oldInputArtifacts, m_oldExplicitlyDependsOn, + m_oldAuxiliaryInputs, m_lastApplicationTime, + m_needsToConsiderChangedInputs); } ArtifactSet currentInputArtifacts() const; + ArtifactSet changedInputArtifacts(const ArtifactSet &allCompatibleInputs, + const ArtifactSet &explicitlyDependsOn, const ArtifactSet &auxiliaryInputs) const; RuleConstPtr m_rule; + + // These three can contain null pointers, which represent a "dummy artifact" encoding + // the information that an artifact that used to be in here has ceased to exist. + // This is okay, because no code outside this class has access to these sets, so + // we cannot break any assumptions about non-nullness. ArtifactSet m_oldInputArtifacts; + ArtifactSet m_oldExplicitlyDependsOn; + ArtifactSet m_oldAuxiliaryInputs; + + FileTime m_lastApplicationTime; + bool m_needsToConsiderChangedInputs = false; }; template<> inline bool hasDynamicType<RuleNode>(const BuildGraphNode *n) diff --git a/src/lib/corelib/buildgraph/rulesapplicator.cpp b/src/lib/corelib/buildgraph/rulesapplicator.cpp index 1144e5ff5..777ec4b9e 100644 --- a/src/lib/corelib/buildgraph/rulesapplicator.cpp +++ b/src/lib/corelib/buildgraph/rulesapplicator.cpp @@ -39,11 +39,11 @@ #include "rulesapplicator.h" #include "buildgraph.h" -#include "emptydirectoriesremover.h" #include "productbuilddata.h" #include "projectbuilddata.h" #include "qtmocscanner.h" #include "rulecommands.h" +#include "rulenode.h" #include "rulesevaluationcontext.h" #include "transformer.h" #include "transformerchangetracking.h" @@ -65,6 +65,7 @@ #include <tools/qttools.h> #include <tools/stringconstants.h> +#include <QtCore/qcryptographichash.h> #include <QtCore/qdir.h> #include <QtScript/qscriptvalueiterator.h> @@ -92,19 +93,22 @@ RulesApplicator::~RulesApplicator() delete m_mocScanner; } -void RulesApplicator::applyRule(const RuleConstPtr &rule, const ArtifactSet &inputArtifacts) +void RulesApplicator::applyRule(RuleNode *ruleNode, const ArtifactSet &inputArtifacts, + const ArtifactSet &explicitlyDependsOn) { - if (inputArtifacts.empty() && rule->declaresInputs() && rule->requiresInputs) - return; + m_ruleNode = ruleNode; + m_rule = ruleNode->rule(); + QBS_CHECK(!inputArtifacts.empty() || !m_rule->declaresInputs() || !m_rule->requiresInputs); m_product->topLevelProject()->buildData->setDirty(); m_createdArtifacts.clear(); m_invalidatedArtifacts.clear(); + m_removedArtifacts.clear(); + m_explicitlyDependsOn = explicitlyDependsOn; RulesEvaluationContext::Scope s(evalContext().get()); - m_rule = rule; m_completeInputSet = inputArtifacts; - if (rule->name == QLatin1String("QtCoreMocRule")) { + if (m_rule->name.startsWith(QLatin1String("QtCoreMocRule"))) { delete m_mocScanner; m_mocScanner = new QtMocScanner(m_product, scope()); } @@ -115,6 +119,7 @@ void RulesApplicator::applyRule(const RuleConstPtr &rule, const ArtifactSet &inp setupScriptEngineForProduct(engine(), m_product.get(), m_rule->module.get(), prepareScriptContext, true); + engine()->clearUsesIo(); if (m_rule->multiplex) { // apply the rule once for a set of inputs doApply(inputArtifacts, prepareScriptContext); } else { // apply the rule once for each input @@ -124,10 +129,13 @@ void RulesApplicator::applyRule(const RuleConstPtr &rule, const ArtifactSet &inp doApply(lst, prepareScriptContext); } } + if (engine()->usesIo()) + m_ruleUsesIo = true; } void RulesApplicator::handleRemovedRuleOutputs(const ArtifactSet &inputArtifacts, - const ArtifactSet &outputArtifactsToRemove, const Logger &logger) + const ArtifactSet &outputArtifactsToRemove, QStringList &removedArtifacts, + const Logger &logger) { ArtifactSet artifactsToRemove; const TopLevelProject *project = nullptr; @@ -139,14 +147,9 @@ void RulesApplicator::handleRemovedRuleOutputs(const ArtifactSet &inputArtifacts project->buildData->removeArtifactAndExclusiveDependents(removedArtifact, logger, true, &artifactsToRemove); } - // parents of removed artifacts must update their transformers - for (Artifact *removedArtifact : qAsConst(artifactsToRemove)) { - for (Artifact *parent : removedArtifact->parentArtifacts()) - parent->product->registerArtifactWithChangedInputs(parent); - } - EmptyDirectoriesRemover(project, logger).removeEmptyParentDirectories(artifactsToRemove); for (Artifact * const artifact : qAsConst(artifactsToRemove)) { QBS_CHECK(!inputArtifacts.contains(artifact)); + removedArtifacts << artifact->filePath(); delete artifact; } } @@ -183,13 +186,13 @@ void RulesApplicator::doApply(const ArtifactSet &inputArtifacts, QScriptValue &p qCDebug(lcBuildGraph) << "apply rule" << m_rule->toString() << toStringList(inputArtifacts).join(QLatin1String(",\n ")); - QList<std::pair<const RuleArtifact *, Artifact *> > ruleArtifactArtifactMap; + QList<std::pair<const RuleArtifact *, OutputArtifactInfo>> ruleArtifactArtifactMap; QList<Artifact *> outputArtifacts; m_transformer = Transformer::create(); m_transformer->rule = m_rule; m_transformer->inputs = inputArtifacts; - m_transformer->explicitlyDependsOn = collectExplicitlyDependsOn(); + m_transformer->explicitlyDependsOn = m_explicitlyDependsOn; m_transformer->alwaysRun = m_rule->alwaysRun; m_oldTransformer.reset(); @@ -206,19 +209,37 @@ void RulesApplicator::doApply(const ArtifactSet &inputArtifacts, QScriptValue &p if (m_rule->isDynamic()) { outputArtifacts = runOutputArtifactsScript(inputArtifacts, ScriptEngine::argumentList(Rule::argumentNamesForOutputArtifacts(), scope())); - ArtifactSet newOutputs = ArtifactSet::fromList(outputArtifacts); - const ArtifactSet oldOutputs = collectOldOutputArtifacts(inputArtifacts); - handleRemovedRuleOutputs(m_completeInputSet, oldOutputs - newOutputs, m_logger); } else { Set<QString> outputFilePaths; for (const RuleArtifactConstPtr &ruleArtifact : m_rule->artifacts) { - Artifact * const outputArtifact - = createOutputArtifactFromRuleArtifact(ruleArtifact, inputArtifacts, - &outputFilePaths); - if (!outputArtifact) + const OutputArtifactInfo outputInfo = createOutputArtifactFromRuleArtifact( + ruleArtifact, inputArtifacts, &outputFilePaths); + if (!outputInfo.artifact) continue; - outputArtifacts.push_back(outputArtifact); - ruleArtifactArtifactMap.push_back({ ruleArtifact.get(), outputArtifact }); + outputArtifacts.push_back(outputInfo.artifact); + ruleArtifactArtifactMap.push_back({ ruleArtifact.get(), outputInfo }); + } + if (m_rule->artifacts.empty()) { + outputArtifacts.push_back(createOutputArtifactFromRuleArtifact( + nullptr, inputArtifacts, &outputFilePaths).artifact); + } + } + + ArtifactSet newOutputs = ArtifactSet::fromList(outputArtifacts); + const ArtifactSet oldOutputs = collectOldOutputArtifacts(inputArtifacts); + handleRemovedRuleOutputs(m_completeInputSet, oldOutputs - newOutputs, m_removedArtifacts, + m_logger); + + // The inputs become children of the rule node. Generated artifacts in the same product + // already are children, because output artifacts become children of the producing + // rule node's parent rule node. + for (Artifact * const input : inputArtifacts) { + if (input->artifactType == Artifact::SourceFile || input->product != m_ruleNode->product + || input->producer()->rule()->collectedOutputFileTags().intersects( + m_ruleNode->rule()->excludedInputs)) { + connect(m_ruleNode, input); + } else { + QBS_CHECK(m_ruleNode->children.contains(input)); } } @@ -228,7 +249,6 @@ void RulesApplicator::doApply(const ArtifactSet &inputArtifacts, QScriptValue &p for (Artifact * const outputArtifact : qAsConst(outputArtifacts)) { for (Artifact * const dependency : qAsConst(m_transformer->explicitlyDependsOn)) connect(outputArtifact, dependency); - outputArtifact->product->unregisterArtifactWithChangedInputs(outputArtifact); } if (inputArtifacts != m_transformer->inputs) @@ -245,7 +265,8 @@ void RulesApplicator::doApply(const ArtifactSet &inputArtifacts, QScriptValue &p continue; // expose attributes of this artifact - Artifact *outputArtifact = it->second; + const OutputArtifactInfo outputInfo = it->second; + Artifact *outputArtifact = outputInfo.artifact; outputArtifact->properties = outputArtifact->properties->clone(); scope().setProperty(StringConstants::fileNameProperty(), @@ -262,9 +283,15 @@ void RulesApplicator::doApply(const ArtifactSet &inputArtifacts, QScriptValue &p engine()->lastErrorString(scriptValue)), engine()->lastErrorLocation(scriptValue, binding.location)); } - setConfigProperty(artifactModulesCfg, binding.name, scriptValue.toVariant()); + const QVariant value = scriptValue.toVariant(); + setConfigProperty(artifactModulesCfg, binding.name, value); + outputArtifact->pureProperties.push_back(std::make_pair(binding.name, value)); } outputArtifact->properties->setValue(artifactModulesCfg); + if (!outputInfo.newlyCreated && (outputArtifact->fileTags() != outputInfo.oldFileTags + || outputArtifact->properties->value() != outputInfo.oldProperties)) { + invalidateArtifactAsRuleInputIfNecessary(outputArtifact); + } } if (!ruleArtifactArtifactMap.empty()) engine()->setGlobalObject(prepareScriptContext.prototype()); @@ -276,6 +303,8 @@ void RulesApplicator::doApply(const ArtifactSet &inputArtifacts, QScriptValue &p throw ErrorInfo(Tr::tr("There is a rule without commands: %1.") .arg(m_rule->toString()), m_rule->prepareScript.location()); if (!m_oldTransformer || m_oldTransformer->outputs != m_transformer->outputs + || m_oldTransformer->inputs != m_transformer->inputs + || m_oldTransformer->explicitlyDependsOn != m_transformer->explicitlyDependsOn || m_oldTransformer->commands != m_transformer->commands || commandsNeedRerun(m_transformer.get(), m_product.get(), m_productsByName, m_projectsByName)) { @@ -336,43 +365,72 @@ ArtifactSet RulesApplicator::collectAdditionalInputs(const FileTags &tags, const return artifacts; } -ArtifactSet RulesApplicator::collectExplicitlyDependsOn() +ArtifactSet RulesApplicator::collectExplicitlyDependsOn(const Rule *rule, + const ResolvedProduct *product) { ArtifactSet first = collectAdditionalInputs( - m_rule->explicitlyDependsOn, m_rule.get(), m_product.get(), CurrentProduct); + rule->explicitlyDependsOn, rule, product, CurrentProduct); ArtifactSet second = collectAdditionalInputs( - m_rule->explicitlyDependsOnFromDependencies, m_rule.get(), m_product.get(), - Dependencies); + rule->explicitlyDependsOnFromDependencies, rule, product, Dependencies); return first.unite(second); } -Artifact *RulesApplicator::createOutputArtifactFromRuleArtifact( +RulesApplicator::OutputArtifactInfo RulesApplicator::createOutputArtifactFromRuleArtifact( const RuleArtifactConstPtr &ruleArtifact, const ArtifactSet &inputArtifacts, Set<QString> *outputFilePaths) { - QScriptValue scriptValue = engine()->evaluate(ruleArtifact->filePath, - ruleArtifact->filePathLocation.filePath(), - ruleArtifact->filePathLocation.line()); - if (Q_UNLIKELY(engine()->hasErrorOrException(scriptValue))) - throw engine()->lastError(scriptValue, ruleArtifact->filePathLocation); - QString outputPath = FileInfo::resolvePath(m_product->buildDirectory(), scriptValue.toString()); + QString outputPath; + FileTags fileTags; + bool alwaysUpdated; + if (ruleArtifact) { + QScriptValue scriptValue = engine()->evaluate(ruleArtifact->filePath, + ruleArtifact->filePathLocation.filePath(), + ruleArtifact->filePathLocation.line()); + if (Q_UNLIKELY(engine()->hasErrorOrException(scriptValue))) + throw engine()->lastError(scriptValue, ruleArtifact->filePathLocation); + outputPath = scriptValue.toString(); + fileTags = ruleArtifact->fileTags; + alwaysUpdated = ruleArtifact->alwaysUpdated; + } else { + outputPath = QStringLiteral("__dummyoutput__"); + QByteArray hashInput = m_rule->toString().toLatin1(); + for (const Artifact * const input : inputArtifacts) + hashInput += input->filePath().toLatin1(); + outputPath += QLatin1String(QCryptographicHash::hash(hashInput, QCryptographicHash::Sha1) + .toHex().left(16)); + fileTags = m_rule->outputFileTags; + alwaysUpdated = false; + } + outputPath = FileInfo::resolvePath(m_product->buildDirectory(), outputPath); if (Q_UNLIKELY(!outputFilePaths->insert(outputPath).second)) { throw ErrorInfo(Tr::tr("Rule %1 already created '%2'.") .arg(m_rule->toString(), outputPath)); } - return createOutputArtifact(outputPath, ruleArtifact->fileTags, ruleArtifact->alwaysUpdated, - inputArtifacts); + return createOutputArtifact(outputPath, fileTags, alwaysUpdated, inputArtifacts); } -Artifact *RulesApplicator::createOutputArtifact(const QString &filePath, const FileTags &fileTags, - bool alwaysUpdated, const ArtifactSet &inputArtifacts) +RulesApplicator::OutputArtifactInfo RulesApplicator::createOutputArtifact(const QString &filePath, + const FileTags &fileTags, bool alwaysUpdated, const ArtifactSet &inputArtifacts) { QString outputPath = filePath; // don't let the output artifact "escape" its build dir outputPath.replace(StringConstants::dotDot(), QStringLiteral("dotdot")); outputPath = resolveOutPath(outputPath); - Artifact *outputArtifact = lookupArtifact(m_product, outputPath); + if (m_rule->isDynamic()) { + const Set<FileTag> undeclaredTags = fileTags - m_rule->collectedOutputFileTags(); + if (!undeclaredTags.empty()) { + throw ErrorInfo(Tr::tr("Artifact '%1' has undeclared file tags [\"%2\"].") + .arg(outputPath, undeclaredTags.toStringList() + .join(QLatin1String("\",\""))), + m_rule->prepareScript.location()); + } + } + + OutputArtifactInfo outputInfo; + Artifact *& outputArtifact = outputInfo.artifact; + outputArtifact = lookupArtifact(m_product, outputPath); + outputInfo.newlyCreated = !outputArtifact; if (outputArtifact) { const Transformer * const transformer = outputArtifact->transformer.get(); if (transformer && transformer->rule != m_rule) { @@ -413,6 +471,8 @@ Artifact *RulesApplicator::createOutputArtifact(const QString &filePath, const F } m_transformer->rescueChangeTrackingData(outputArtifact->transformer); m_oldTransformer = outputArtifact->transformer; + outputInfo.oldFileTags = outputArtifact->fileTags(); + outputInfo.oldProperties = outputArtifact->properties->value(); } else { std::unique_ptr<Artifact> newArtifact(new Artifact); newArtifact->artifactType = Artifact::Generated; @@ -420,25 +480,17 @@ Artifact *RulesApplicator::createOutputArtifact(const QString &filePath, const F insertArtifact(m_product, newArtifact.get()); m_createdArtifacts += newArtifact.get(); outputArtifact = newArtifact.release(); + qCDebug(lcExec).noquote() << "rule created" << outputArtifact->toString(); + connect(outputArtifact, m_ruleNode); } outputArtifact->alwaysUpdated = alwaysUpdated; - outputArtifact->properties = m_product->moduleProperties; - - FileTags outputArtifactFileTags = fileTags.empty() - ? m_product->fileTagsForFileName(outputArtifact->fileName()) : fileTags; - for (const ArtifactPropertiesConstPtr &props : m_product->artifactProperties) { - if (outputArtifactFileTags.intersects(props->fileTagsFilter())) { - outputArtifact->properties = props->propertyMap(); - outputArtifactFileTags += props->extraFileTags(); - break; - } + outputArtifact->pureFileTags = fileTags; + provideFullFileTagsAndProperties(outputArtifact); + if (outputInfo.newlyCreated || outputInfo.oldFileTags != outputArtifact->fileTags()) { + for (RuleNode * const parentRule : filterByType<RuleNode>(m_ruleNode->parents)) + connect(parentRule, outputArtifact); } - outputArtifact->setFileTags(outputArtifactFileTags); - - // Let a positive value of qbs.install imply the file tag "installable". - if (outputArtifact->properties->qbsPropertyValue(StringConstants::installProperty()).toBool()) - outputArtifact->addFileTag("installable"); for (Artifact * const inputArtifact : inputArtifacts) { QBS_CHECK(outputArtifact != inputArtifact); @@ -449,7 +501,7 @@ Artifact *RulesApplicator::createOutputArtifact(const QString &filePath, const F m_transformer->outputs.insert(outputArtifact); QBS_CHECK(m_rule->multiplex || m_transformer->inputs.size() == 1); - return outputArtifact; + return outputInfo; } class RuleOutputArtifactsException : public ErrorInfo @@ -550,8 +602,11 @@ public: outputArtifact->properties = outputArtifact->properties->clone(); QVariantMap artifactCfg = outputArtifact->properties->value(); - for (const auto &e : m_propertyValues) - setConfigProperty(artifactCfg, {e.module, e.name}, e.value); + for (const auto &e : m_propertyValues) { + const QStringList key{e.module, e.name}; + setConfigProperty(artifactCfg, key, e.value); + outputArtifact->pureProperties.push_back(std::make_pair(key, e.value)); + } outputArtifact->properties->setValue(artifactCfg); } }; @@ -575,8 +630,9 @@ Artifact *RulesApplicator::createOutputArtifactFromScriptValue(const QScriptValu const QVariant alwaysUpdatedVar = obj.property(StringConstants::alwaysUpdatedProperty()).toVariant(); const bool alwaysUpdated = alwaysUpdatedVar.isValid() ? alwaysUpdatedVar.toBool() : true; - Artifact *output = createOutputArtifact(filePath, fileTags, alwaysUpdated, inputArtifacts); - if (output->fileTags().empty()) { + OutputArtifactInfo outputInfo = createOutputArtifact(filePath, fileTags, alwaysUpdated, + inputArtifacts); + if (outputInfo.artifact->fileTags().empty()) { // Check the file tags after file taggers were run. throw RuleOutputArtifactsException( Tr::tr("Property fileTags for artifact '%1' must be a non-empty string list. " @@ -588,10 +644,14 @@ Artifact *RulesApplicator::createOutputArtifactFromScriptValue(const QScriptValu .toVariant().toStringList()); for (const FileTag &tag : explicitlyDependsOn) { for (Artifact * const dependency : m_product->lookupArtifactsByFileTag(tag)) - connect(output, dependency); + connect(outputInfo.artifact, dependency); + } + ArtifactBindingsExtractor().apply(outputInfo.artifact, obj); + if (!outputInfo.newlyCreated && (outputInfo.artifact->fileTags() != outputInfo.oldFileTags + || outputInfo.artifact->properties->value() != outputInfo.oldProperties)) { + invalidateArtifactAsRuleInputIfNecessary(outputInfo.artifact); } - ArtifactBindingsExtractor().apply(output, obj); - return output; + return outputInfo.artifact; } QString RulesApplicator::resolveOutPath(const QString &path) const diff --git a/src/lib/corelib/buildgraph/rulesapplicator.h b/src/lib/corelib/buildgraph/rulesapplicator.h index adc58f3d0..f6fadb1ec 100644 --- a/src/lib/corelib/buildgraph/rulesapplicator.h +++ b/src/lib/corelib/buildgraph/rulesapplicator.h @@ -70,11 +70,16 @@ public: const NodeSet &createdArtifacts() const { return m_createdArtifacts; } const NodeSet &invalidatedArtifacts() const { return m_invalidatedArtifacts; } + QStringList removedArtifacts() const { return m_removedArtifacts; } + bool ruleUsesIo() const { return m_ruleUsesIo; } - void applyRule(const RuleConstPtr &rule, const ArtifactSet &inputArtifacts); + void applyRule(RuleNode *ruleNode, const ArtifactSet &inputArtifacts, + const ArtifactSet &explicitlyDependsOn); static void handleRemovedRuleOutputs(const ArtifactSet &inputArtifacts, - const ArtifactSet &artifactsToRemove, const Logger &logger); + const ArtifactSet &artifactsToRemove, QStringList &removedArtifacts, + const Logger &logger); static ArtifactSet collectAuxiliaryInputs(const Rule *rule, const ResolvedProduct *product); + static ArtifactSet collectExplicitlyDependsOn(const Rule *rule, const ResolvedProduct *product); enum InputsSourceFlag { CurrentProduct = 1, Dependencies = 2 }; Q_DECLARE_FLAGS(InputsSources, InputsSourceFlag) @@ -82,11 +87,17 @@ public: private: void doApply(const ArtifactSet &inputArtifacts, QScriptValue &prepareScriptContext); ArtifactSet collectOldOutputArtifacts(const ArtifactSet &inputArtifacts) const; - ArtifactSet collectExplicitlyDependsOn(); - ArtifactSet collectExplicitlyDependsOnFromDependencies(); - Artifact *createOutputArtifactFromRuleArtifact(const RuleArtifactConstPtr &ruleArtifact, - const ArtifactSet &inputArtifacts, Set<QString> *outputFilePaths); - Artifact *createOutputArtifact(const QString &filePath, const FileTags &fileTags, + + struct OutputArtifactInfo { + Artifact *artifact = nullptr; + bool newlyCreated = false; + FileTags oldFileTags; + QVariantMap oldProperties; + }; + OutputArtifactInfo createOutputArtifactFromRuleArtifact( + const RuleArtifactConstPtr &ruleArtifact, const ArtifactSet &inputArtifacts, + Set<QString> *outputFilePaths); + OutputArtifactInfo createOutputArtifact(const QString &filePath, const FileTags &fileTags, bool alwaysUpdated, const ArtifactSet &inputArtifacts); QList<Artifact *> runOutputArtifactsScript(const ArtifactSet &inputArtifacts, const QScriptValueList &args); @@ -104,14 +115,18 @@ private: const ResolvedProductPtr m_product; const std::unordered_map<QString, const ResolvedProduct *> &m_productsByName; const std::unordered_map<QString, const ResolvedProject *> &m_projectsByName; + ArtifactSet m_explicitlyDependsOn; NodeSet m_createdArtifacts; NodeSet m_invalidatedArtifacts; + QStringList m_removedArtifacts; + RuleNode *m_ruleNode = nullptr; RuleConstPtr m_rule; ArtifactSet m_completeInputSet; TransformerPtr m_transformer; TransformerConstPtr m_oldTransformer; QtMocScanner *m_mocScanner; Logger m_logger; + bool m_ruleUsesIo = false; }; Q_DECLARE_OPERATORS_FOR_FLAGS(RulesApplicator::InputsSources) diff --git a/src/lib/corelib/buildgraph/transformer.cpp b/src/lib/corelib/buildgraph/transformer.cpp index 009e8cde4..ccde486ab 100644 --- a/src/lib/corelib/buildgraph/transformer.cpp +++ b/src/lib/corelib/buildgraph/transformer.cpp @@ -41,6 +41,7 @@ #include "artifact.h" #include <jsextensions/moduleproperties.h> #include <language/language.h> +#include <language/preparescriptobserver.h> #include <language/scriptengine.h> #include <logging/translator.h> #include <tools/error.h> @@ -97,6 +98,18 @@ static QScriptValue js_baseDir(QScriptContext *ctx, QScriptEngine *engine, return basedir; } +static QScriptValue js_children(QScriptContext *ctx, QScriptEngine *engine, const Artifact *artifact) +{ + Q_UNUSED(ctx); + QScriptValue sv = engine->newArray(); + uint idx = 0; + for (const Artifact *parent : artifact->childArtifacts()) { + sv.setProperty(idx++, Transformer::translateFileConfig(static_cast<ScriptEngine *>(engine), + parent, QString())); + } + return sv; +} + static void setArtifactProperty(QScriptValue &obj, const QString &name, QScriptValue (*func)(QScriptContext *, QScriptEngine *, const Artifact *), const Artifact *artifact) @@ -117,8 +130,11 @@ QScriptValue Transformer::translateFileConfig(ScriptEngine *scriptEngine, const setArtifactProperty(obj, StringConstants::completeBaseNameProperty(), js_completeBaseName, artifact); setArtifactProperty(obj, QStringLiteral("baseDir"), js_baseDir, artifact); + setArtifactProperty(obj, QStringLiteral("children"), js_children, artifact); const QStringList fileTags = sorted(artifact->fileTags().toStringList()); - obj.setProperty(StringConstants::fileTagsProperty(), scriptEngine->toScriptValue(fileTags)); + scriptEngine->setObservedProperty(obj, StringConstants::fileTagsProperty(), + scriptEngine->toScriptValue(fileTags)); + scriptEngine->observer()->addArtifactId(obj.objectId()); if (!defaultModuleName.isEmpty()) obj.setProperty(StringConstants::moduleNameProperty(), defaultModuleName); return obj; @@ -224,7 +240,7 @@ AbstractCommandPtr Transformer::createCommandFromScriptValue(const QScriptValue if (cmdBase) cmdBase->fillFromScriptValue(&scriptValue, codeLocation); if (className == StringConstants::commandType()) { - ProcessCommand *procCmd = static_cast<ProcessCommand *>(cmdBase.get()); + auto procCmd = static_cast<ProcessCommand *>(cmdBase.get()); procCmd->clearRelevantEnvValues(); for (const QString &key : procCmd->relevantEnvVars()) procCmd->addRelevantEnvValue(key, product()->buildEnvironment.value(key)); @@ -297,9 +313,20 @@ void Transformer::rescueChangeTrackingData(const TransformerConstPtr &other) lastPrepareScriptExecutionTime = other->lastPrepareScriptExecutionTime; prepareScriptNeedsChangeTracking = other->prepareScriptNeedsChangeTracking; commandsNeedChangeTracking = other->commandsNeedChangeTracking; + markedForRerun = other->markedForRerun; exportedModulesAccessedInPrepareScript = other->exportedModulesAccessedInPrepareScript; exportedModulesAccessedInCommands = other->exportedModulesAccessedInCommands; } +Set<QString> Transformer::jobPools() const +{ + Set<QString> pools; + for (const AbstractCommandPtr &c : commands.commands()) { + if (!c->jobPool().isEmpty()) + pools.insert(c->jobPool()); + } + return pools; +} + } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/buildgraph/transformer.h b/src/lib/corelib/buildgraph/transformer.h index 1b6d36253..2f6a8e56d 100644 --- a/src/lib/corelib/buildgraph/transformer.h +++ b/src/lib/corelib/buildgraph/transformer.h @@ -89,6 +89,7 @@ public: bool alwaysRun; bool prepareScriptNeedsChangeTracking = false; bool commandsNeedChangeTracking = false; + bool markedForRerun = false; static QScriptValue translateFileConfig(ScriptEngine *scriptEngine, const Artifact *artifact, @@ -101,6 +102,8 @@ public: const QScriptValueList &args); void rescueChangeTrackingData(const TransformerConstPtr &other); + Set<QString> jobPools() const; + template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool) { pool.serializationOp<opType>(rule, inputs, outputs, explicitlyDependsOn, @@ -116,7 +119,7 @@ public: exportedModulesAccessedInPrepareScript, exportedModulesAccessedInCommands, alwaysRun, prepareScriptNeedsChangeTracking, - commandsNeedChangeTracking); + commandsNeedChangeTracking, markedForRerun); } private: diff --git a/src/lib/corelib/buildgraph/transformerchangetracking.cpp b/src/lib/corelib/buildgraph/transformerchangetracking.cpp index d784ed5b7..551d2b1b1 100644 --- a/src/lib/corelib/buildgraph/transformerchangetracking.cpp +++ b/src/lib/corelib/buildgraph/transformerchangetracking.cpp @@ -124,6 +124,7 @@ QVariantMap TrafoChangeTracker::propertyMapByKind(const Property &property) cons return v; return getParameterValue(p->moduleParameters, depName); } + case Property::PropertyInArtifact: default: QBS_CHECK(false); } @@ -151,7 +152,10 @@ bool TrafoChangeTracker::checkForPropertyChange(const Property &restoredProperty while (!moduleName.empty()) map = map.value(moduleName.takeFirst()).toMap(); v = map.value(restoredProperty.propertyName); + break; } + case Property::PropertyInArtifact: + QBS_CHECK(false); } if (restoredProperty.value != v) { qCDebug(lcBuildGraph).noquote().nospace() @@ -227,7 +231,7 @@ const Artifact *TrafoChangeTracker::getArtifact(const QString &filePath, const Artifact *artifact = nullptr; for (const FileResourceBase * const candidate : candidates) { if (candidate->fileType() == FileResourceBase::FileTypeArtifact) { - const Artifact * const a = static_cast<const Artifact *>(candidate); + auto const a = static_cast<const Artifact *>(candidate); if (a->product.get() == product) { m_lastArtifact = artifact = a; break; @@ -271,6 +275,11 @@ bool TrafoChangeTracker::prepareScriptNeedsRerun() const const Artifact * const artifact = getArtifact(it.key(), property.productName); if (!artifact) return true; + if (property.kind == Property::PropertyInArtifact) { + if (sorted(artifact->fileTags().toStringList()) != property.value.toStringList()) + return true; + continue; + } if (checkForPropertyChange(property, artifact->properties->value())) return true; } @@ -302,6 +311,11 @@ bool TrafoChangeTracker::commandsNeedRerun() const const Artifact * const artifact = getArtifact(it.key(), property.productName); if (!artifact) return true; + if (property.kind == Property::PropertyInArtifact) { + if (sorted(artifact->fileTags().toStringList()) != property.value.toStringList()) + return true; + continue; + } if (checkForPropertyChange(property, artifact->properties->value())) return true; } diff --git a/src/lib/corelib/corelib.qbs b/src/lib/corelib/corelib.qbs index 93509763b..c947cb484 100644 --- a/src/lib/corelib/corelib.qbs +++ b/src/lib/corelib/corelib.qbs @@ -397,6 +397,7 @@ QbsLibrary { "id.cpp", "id.h", "iosutils.h", + "joblimits.cpp", "jsliterals.cpp", "jsliterals.h", "installoptions.cpp", @@ -467,6 +468,7 @@ QbsLibrary { "error.h", "generateoptions.h", "installoptions.h", + "joblimits.h", "preferences.h", "processresult.h", "profile.h", diff --git a/src/lib/corelib/generators/generator.h b/src/lib/corelib/generators/generator.h index f5e70f36f..8d4e4f21b 100644 --- a/src/lib/corelib/generators/generator.h +++ b/src/lib/corelib/generators/generator.h @@ -59,7 +59,7 @@ class QBS_EXPORT ProjectGenerator : public QObject Q_OBJECT Q_DISABLE_COPY(ProjectGenerator) public: - virtual ~ProjectGenerator(); + ~ProjectGenerator(); /*! * Returns the name of the generator used to create the external build system files. diff --git a/src/lib/corelib/jsextensions/binaryfile.cpp b/src/lib/corelib/jsextensions/binaryfile.cpp index eb8497e6f..a2c095dcb 100644 --- a/src/lib/corelib/jsextensions/binaryfile.cpp +++ b/src/lib/corelib/jsextensions/binaryfile.cpp @@ -115,6 +115,7 @@ QScriptValue BinaryFile::ctor(QScriptContext *context, QScriptEngine *engine) DubiousContext(EvalContext::PropertyEvaluation, DubiousContext::SuggestMoving) }; se->checkContext(QLatin1String("qbs.BinaryFile"), dubiousContexts); + se->setUsesIo(); return engine->newQObject(t, QScriptEngine::QtOwnership); } diff --git a/src/lib/corelib/jsextensions/domxml.cpp b/src/lib/corelib/jsextensions/domxml.cpp index 09acccbab..dc84fbdaf 100644 --- a/src/lib/corelib/jsextensions/domxml.cpp +++ b/src/lib/corelib/jsextensions/domxml.cpp @@ -38,6 +38,8 @@ ** ****************************************************************************/ +#include <language/scriptengine.h> + #include <QtCore/qfile.h> #include <QtCore/qobject.h> #include <QtCore/qvariant.h> @@ -132,6 +134,7 @@ QScriptValue XmlDomDocument::ctor(QScriptContext *context, QScriptEngine *engine return context->throwError(QLatin1String("DomXml(QString file = QLatin1String(\"\"))")); } QScriptValue obj = engine->newQObject(xml, QScriptEngine::ScriptOwnership); + static_cast<ScriptEngine *>(engine)->setUsesIo(); return obj; } @@ -364,7 +367,7 @@ QScriptValue XmlDomNode::nextSibling(const QString &tagName) const QScriptValue XmlDomNode::appendChild(QScriptValue newChild) { - XmlDomNode *newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); + auto newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); if (!newNode) { context()->throwError(QString::fromLatin1("First argument is not a XmlDomNode object")); return QScriptValue(); @@ -374,13 +377,13 @@ QScriptValue XmlDomNode::appendChild(QScriptValue newChild) QScriptValue XmlDomNode::insertBefore(const QScriptValue &newChild, const QScriptValue &refChild) { - XmlDomNode *newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); + auto newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); if (!newNode) { context()->throwError(QString::fromLatin1("First argument is not a XmlDomNode object")); return QScriptValue(); } - XmlDomNode *refNode = qobject_cast<XmlDomNode*>(refChild.toQObject()); + auto refNode = qobject_cast<XmlDomNode*>(refChild.toQObject()); if (!refNode) { context()->throwError(QString::fromLatin1("Second argument is not a XmlDomNode object")); return QScriptValue(); @@ -391,13 +394,13 @@ QScriptValue XmlDomNode::insertBefore(const QScriptValue &newChild, const QScrip QScriptValue XmlDomNode::insertAfter(const QScriptValue &newChild, const QScriptValue &refChild) { - XmlDomNode *newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); + auto newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); if (!newNode) { context()->throwError(QString::fromLatin1("First argument is not a XmlDomNode object")); return QScriptValue(); } - XmlDomNode *refNode = qobject_cast<XmlDomNode*>(refChild.toQObject()); + auto refNode = qobject_cast<XmlDomNode*>(refChild.toQObject()); if (!refNode) { context()->throwError(QString::fromLatin1("Second argument is not a XmlDomNode object")); return QScriptValue(); @@ -408,13 +411,13 @@ QScriptValue XmlDomNode::insertAfter(const QScriptValue &newChild, const QScript QScriptValue XmlDomNode::replaceChild(const QScriptValue &newChild, const QScriptValue &oldChild) { - XmlDomNode *newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); + auto newNode = qobject_cast<XmlDomNode*>(newChild.toQObject()); if (!newNode) { context()->throwError(QString::fromLatin1("First argument is not a XmlDomNode object")); return QScriptValue(); } - XmlDomNode *oldNode = qobject_cast<XmlDomNode*>(oldChild.toQObject()); + auto oldNode = qobject_cast<XmlDomNode*>(oldChild.toQObject()); if (!oldNode) { context()->throwError(QString::fromLatin1("Second argument is not a XmlDomNode object")); return QScriptValue(); @@ -425,7 +428,7 @@ QScriptValue XmlDomNode::replaceChild(const QScriptValue &newChild, const QScrip QScriptValue XmlDomNode::removeChild(const QScriptValue &oldChild) { - XmlDomNode *oldNode = qobject_cast<XmlDomNode*>(oldChild.toQObject()); + auto oldNode = qobject_cast<XmlDomNode*>(oldChild.toQObject()); if (!oldNode) { context()->throwError(QString::fromLatin1("First argument is not a XmlDomNode object")); return QScriptValue(); diff --git a/src/lib/corelib/jsextensions/environmentextension.cpp b/src/lib/corelib/jsextensions/environmentextension.cpp index e40831f5d..22c4f069b 100644 --- a/src/lib/corelib/jsextensions/environmentextension.cpp +++ b/src/lib/corelib/jsextensions/environmentextension.cpp @@ -72,7 +72,7 @@ static QProcessEnvironment *getProcessEnvironment(QScriptContext *context, QScri const QString &func, bool doThrow = true) { QVariant v = engine->property(StringConstants::qbsProcEnvVarInternal()); - QProcessEnvironment *procenv = reinterpret_cast<QProcessEnvironment *>(v.value<void *>()); + auto procenv = reinterpret_cast<QProcessEnvironment *>(v.value<void *>()); if (!procenv && doThrow) throw context->throwError(QScriptContext::UnknownError, QStringLiteral("%1 can only be called from ").arg(func) + diff --git a/src/lib/corelib/jsextensions/moduleproperties.cpp b/src/lib/corelib/jsextensions/moduleproperties.cpp index 7607b3dc2..4deed3fd1 100644 --- a/src/lib/corelib/jsextensions/moduleproperties.cpp +++ b/src/lib/corelib/jsextensions/moduleproperties.cpp @@ -261,7 +261,7 @@ void ModuleProperties::init(QScriptValue artifactObject, const Artifact *artifac void ModuleProperties::setModuleScriptValue(QScriptValue targetObject, const QScriptValue &moduleObject, const QString &moduleName) { - ScriptEngine * const e = static_cast<ScriptEngine *>(targetObject.engine()); + auto const e = static_cast<ScriptEngine *>(targetObject.engine()); const QualifiedId name = QualifiedId::fromString(moduleName); QScriptValue obj = targetObject; for (int i = 0; i < name.size() - 1; ++i) { diff --git a/src/lib/corelib/jsextensions/process.cpp b/src/lib/corelib/jsextensions/process.cpp index ced69a3e6..1c714fe64 100644 --- a/src/lib/corelib/jsextensions/process.cpp +++ b/src/lib/corelib/jsextensions/process.cpp @@ -135,6 +135,7 @@ QScriptValue Process::ctor(QScriptContext *context, QScriptEngine *engine) t->m_environment = QProcessEnvironment(*reinterpret_cast<QProcessEnvironment*>(v.value<void*>())); } + se->setUsesIo(); return obj; } diff --git a/src/lib/corelib/jsextensions/propertylist.mm b/src/lib/corelib/jsextensions/propertylist.mm index a7c7eb98d..75154d252 100644 --- a/src/lib/corelib/jsextensions/propertylist.mm +++ b/src/lib/corelib/jsextensions/propertylist.mm @@ -101,13 +101,13 @@ public: QScriptValue PropertyList::ctor(QScriptContext *context, QScriptEngine *engine) { - ScriptEngine * const se = static_cast<ScriptEngine *>(engine); + auto const se = static_cast<ScriptEngine *>(engine); const DubiousContextList dubiousContexts({ DubiousContext(EvalContext::PropertyEvaluation, DubiousContext::SuggestMoving) }); se->checkContext(QLatin1String("qbs.PropertyList"), dubiousContexts); - PropertyList *p = new PropertyList(context); + auto p = new PropertyList(context); QScriptValue obj = engine->newQObject(p, QScriptEngine::ScriptOwnership); return obj; } @@ -132,14 +132,14 @@ PropertyList::PropertyList(QScriptContext *context) bool PropertyList::isEmpty() const { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); return p->d->propertyListObject.isNull(); } void PropertyList::clear() { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); p->d->propertyListObject = QVariant(); p->d->propertyListFormat = 0; } @@ -147,7 +147,7 @@ void PropertyList::clear() void PropertyList::readFromObject(const QScriptValue &value) { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); p->d->propertyListObject = value.toVariant(); p->d->propertyListFormat = 0; // wasn't deserialized from any external format } @@ -160,7 +160,7 @@ void PropertyList::readFromString(const QString &input) void PropertyList::readFromFile(const QString &filePath) { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); QFile file(filePath); if (file.open(QIODevice::ReadOnly)) { @@ -177,14 +177,14 @@ void PropertyList::readFromFile(const QString &filePath) void PropertyList::readFromData(const QByteArray &data) { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); p->d->readFromData(p->context(), data); } void PropertyList::writeToFile(const QString &filePath, const QString &plistFormat) { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); QFile file(filePath); QByteArray data = p->d->writeToData(p->context(), plistFormat); @@ -200,7 +200,7 @@ void PropertyList::writeToFile(const QString &filePath, const QString &plistForm QScriptValue PropertyList::format() const { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); switch (p->d->propertyListFormat) { case QPropertyListOpenStepFormat: @@ -219,14 +219,14 @@ QScriptValue PropertyList::format() const QScriptValue PropertyList::toObject() const { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); return p->engine()->toScriptValue(p->d->propertyListObject); } QString PropertyList::toString(const QString &plistFormat) const { Q_ASSERT(thisObject().engine() == engine()); - PropertyList *p = qscriptvalue_cast<PropertyList*>(thisObject()); + auto p = qscriptvalue_cast<PropertyList*>(thisObject()); if (plistFormat == QLatin1String("binary1")) { p->context()->throwError(QLatin1String("Property list object cannot be converted to a " diff --git a/src/lib/corelib/jsextensions/textfile.cpp b/src/lib/corelib/jsextensions/textfile.cpp index e52fb9daf..0ba3c80ef 100644 --- a/src/lib/corelib/jsextensions/textfile.cpp +++ b/src/lib/corelib/jsextensions/textfile.cpp @@ -125,6 +125,7 @@ QScriptValue TextFile::ctor(QScriptContext *context, QScriptEngine *engine) DubiousContext(EvalContext::PropertyEvaluation, DubiousContext::SuggestMoving) }); se->checkContext(QLatin1String("qbs.TextFile"), dubiousContexts); + se->setUsesIo(); return engine->newQObject(t, QScriptEngine::QtOwnership); } diff --git a/src/lib/corelib/language/artifactproperties.cpp b/src/lib/corelib/language/artifactproperties.cpp index 4897c6cff..dd61bf1a2 100644 --- a/src/lib/corelib/language/artifactproperties.cpp +++ b/src/lib/corelib/language/artifactproperties.cpp @@ -57,9 +57,9 @@ FileTags ArtifactProperties::extraFileTags() const return m_extraFileTags; } -void ArtifactProperties::setExtraFileTags(const FileTags &extraFileTags) +void ArtifactProperties::addExtraFileTags(const FileTags &extraFileTags) { - m_extraFileTags = extraFileTags; + m_extraFileTags.unite(extraFileTags); } bool operator==(const ArtifactProperties &ap1, const ArtifactProperties &ap2) diff --git a/src/lib/corelib/language/artifactproperties.h b/src/lib/corelib/language/artifactproperties.h index a255950d0..856aa80c7 100644 --- a/src/lib/corelib/language/artifactproperties.h +++ b/src/lib/corelib/language/artifactproperties.h @@ -59,7 +59,7 @@ public: void setPropertyMapInternal(const PropertyMapPtr &pmap) { m_propertyMap = pmap; } FileTags extraFileTags() const; - void setExtraFileTags(const FileTags &extraFileTags); + void addExtraFileTags(const FileTags &extraFileTags); template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool) { diff --git a/src/lib/corelib/language/astimportshandler.cpp b/src/lib/corelib/language/astimportshandler.cpp index f1d103718..960af4cda 100644 --- a/src/lib/corelib/language/astimportshandler.cpp +++ b/src/lib/corelib/language/astimportshandler.cpp @@ -75,14 +75,22 @@ void ASTImportsHandler::handleImports(const QbsQmlJS::AST::UiImportList *uiImpor // files in the same directory are available as prototypes collectPrototypes(m_directory, QString()); + bool baseImported = false; for (const auto *it = uiImportList; it; it = it->next) - handleImport(it->import); + handleImport(it->import, &baseImported); + if (!baseImported) { + QStringRef qbsref(&StringConstants::qbsModule()); + QbsQmlJS::AST::UiQualifiedId qbsURI(qbsref); + qbsURI.finish(); + QbsQmlJS::AST::UiImport imp(&qbsURI); + handleImport(&imp, &baseImported); + } for (auto it = m_jsImports.constBegin(); it != m_jsImports.constEnd(); ++it) m_file->addJsImport(it.value()); } -void ASTImportsHandler::handleImport(const QbsQmlJS::AST::UiImport *import) +void ASTImportsHandler::handleImport(const QbsQmlJS::AST::UiImport *import, bool *baseImported) { QStringList importUri; bool isBase = false; @@ -92,6 +100,7 @@ void ASTImportsHandler::handleImport(const QbsQmlJS::AST::UiImport *import) || (importUri.size() == 2 && importUri.front() == StringConstants::qbsModule() && importUri.last() == StringConstants::baseVar()); if (isBase) { + *baseImported = true; checkImportVersion(import->versionToken); } else if (import->versionToken.length) { m_logger.printWarning(ErrorInfo(Tr::tr("Superfluous version specification."), diff --git a/src/lib/corelib/language/astimportshandler.h b/src/lib/corelib/language/astimportshandler.h index b69a8fab7..e9c2b6c27 100644 --- a/src/lib/corelib/language/astimportshandler.h +++ b/src/lib/corelib/language/astimportshandler.h @@ -75,7 +75,7 @@ private: void collectPrototypes(const QString &path, const QString &as); void collectPrototypesAndJsCollections(const QString &path, const QString &as, const CodeLocation &location); - void handleImport(const QbsQmlJS::AST::UiImport *import); + void handleImport(const QbsQmlJS::AST::UiImport *import, bool *baseImported); ItemReaderVisitorState &m_visitorState; Logger &m_logger; diff --git a/src/lib/corelib/language/builtindeclarations.cpp b/src/lib/corelib/language/builtindeclarations.cpp index 4be405112..4886675d0 100644 --- a/src/lib/corelib/language/builtindeclarations.cpp +++ b/src/lib/corelib/language/builtindeclarations.cpp @@ -67,6 +67,7 @@ BuiltinDeclarations::BuiltinDeclarations() { QLatin1String("Export"), ItemType::Export }, { QLatin1String("FileTagger"), ItemType::FileTagger }, { QLatin1String("Group"), ItemType::Group }, + { QLatin1String("JobLimit"), ItemType::JobLimit }, { QLatin1String("Module"), ItemType::Module }, { QLatin1String("Parameter"), ItemType::Parameter }, { QLatin1String("Parameters"), ItemType::Parameters }, @@ -87,6 +88,7 @@ BuiltinDeclarations::BuiltinDeclarations() addExportItem(); addFileTaggerItem(); addGroupItem(); + addJobLimitItem(); addModuleItem(); addProbeItem(); addProductItem(); @@ -298,6 +300,15 @@ void BuiltinDeclarations::addGroupItem() insert(item); } +void BuiltinDeclarations::addJobLimitItem() +{ + ItemDeclaration item(ItemType::JobLimit); + item << conditionProperty(); + item << PropertyDeclaration(StringConstants::jobPoolProperty(), PropertyDeclaration::String); + item << PropertyDeclaration(StringConstants::jobCountProperty(), PropertyDeclaration::Integer); + insert(item); +} + void BuiltinDeclarations::addModuleItem() { ItemDeclaration item = moduleLikeItem(ItemType::Module); @@ -312,6 +323,7 @@ ItemDeclaration BuiltinDeclarations::moduleLikeItem(ItemType type) << ItemType::Group << ItemType::Depends << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Rule << ItemType::Parameter << ItemType::Probe @@ -360,6 +372,7 @@ void BuiltinDeclarations::addProductItem() << ItemType::Depends << ItemType::Group << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Export << ItemType::Probe << ItemType::Profile @@ -429,6 +442,7 @@ void BuiltinDeclarations::addProjectItem() << ItemType::Profile << ItemType::Probe << ItemType::FileTagger + << ItemType::JobLimit << ItemType::Rule); item << nameProperty(); item << conditionProperty(); diff --git a/src/lib/corelib/language/builtindeclarations.h b/src/lib/corelib/language/builtindeclarations.h index c75475df7..ff16b395a 100644 --- a/src/lib/corelib/language/builtindeclarations.h +++ b/src/lib/corelib/language/builtindeclarations.h @@ -76,6 +76,7 @@ private: void addExportItem(); void addFileTaggerItem(); void addGroupItem(); + void addJobLimitItem(); void addModuleItem(); static ItemDeclaration moduleLikeItem(ItemType type); void addProbeItem(); diff --git a/src/lib/corelib/language/evaluator.cpp b/src/lib/corelib/language/evaluator.cpp index e02720e3a..c4b747a0c 100644 --- a/src/lib/corelib/language/evaluator.cpp +++ b/src/lib/corelib/language/evaluator.cpp @@ -175,7 +175,7 @@ QScriptValue Evaluator::scriptValue(const Item *item) void Evaluator::onItemPropertyChanged(Item *item) { - EvaluationData *data = attachedPointer<EvaluationData>(m_scriptValueMap.value(item)); + auto data = attachedPointer<EvaluationData>(m_scriptValueMap.value(item)); if (data) data->valueCache.clear(); } diff --git a/src/lib/corelib/language/evaluator.h b/src/lib/corelib/language/evaluator.h index 0fd4003d8..d8931a37e 100644 --- a/src/lib/corelib/language/evaluator.h +++ b/src/lib/corelib/language/evaluator.h @@ -69,14 +69,15 @@ public: ScriptEngine *engine() const { return m_scriptEngine; } QScriptValue property(const Item *item, const QString &name); - QScriptValue value(const Item *item, const QString &name, bool *propertySet = 0); + QScriptValue value(const Item *item, const QString &name, bool *propertySet = nullptr); bool boolValue(const Item *item, const QString &name, bool *propertyWasSet = nullptr); int intValue(const Item *item, const QString &name, int defaultValue = 0, - bool *propertyWasSet = 0); - FileTags fileTagsValue(const Item *item, const QString &name, bool *propertySet = 0); + bool *propertyWasSet = nullptr); + FileTags fileTagsValue(const Item *item, const QString &name, bool *propertySet = nullptr); QString stringValue(const Item *item, const QString &name, - const QString &defaultValue = QString(), bool *propertyWasSet = 0); - QStringList stringListValue(const Item *item, const QString &name, bool *propertyWasSet = 0); + const QString &defaultValue = QString(), bool *propertyWasSet = nullptr); + QStringList stringListValue(const Item *item, const QString &name, + bool *propertyWasSet = nullptr); void convertToPropertyType(const PropertyDeclaration& decl, const CodeLocation &loc, QScriptValue &v); @@ -104,7 +105,7 @@ public: bool isNonDefaultValue(const Item *item, const QString &name) const; private: - void onItemPropertyChanged(Item *item); + void onItemPropertyChanged(Item *item) override; bool evaluateProperty(QScriptValue *result, const Item *item, const QString &name, bool *propertyWasSet); diff --git a/src/lib/corelib/language/evaluatorscriptclass.cpp b/src/lib/corelib/language/evaluatorscriptclass.cpp index 6db308a7f..a5e23a131 100644 --- a/src/lib/corelib/language/evaluatorscriptclass.cpp +++ b/src/lib/corelib/language/evaluatorscriptclass.cpp @@ -381,7 +381,7 @@ QScriptClass::QueryFlags EvaluatorScriptClass::queryProperty(const QScriptValue if (debugProperties) qDebug() << "[SC] queryProperty " << object.objectId() << " " << name; - EvaluationData *const data = attachedPointer<EvaluationData>(object); + auto const data = attachedPointer<EvaluationData>(object); const QString nameString = name.toString(); if (nameString == QStringLiteral("parent")) { *id = QPTParentProperty; @@ -751,7 +751,7 @@ private: QScriptClassPropertyIterator *EvaluatorScriptClass::newIterator(const QScriptValue &object) { - EvaluationData *const data = attachedPointer<EvaluationData>(object); + auto const data = attachedPointer<EvaluationData>(object); return data ? new EvaluatorScriptClassPropertyIterator(object, data) : nullptr; } diff --git a/src/lib/corelib/language/identifiersearch.h b/src/lib/corelib/language/identifiersearch.h index e466c576a..d9769266c 100644 --- a/src/lib/corelib/language/identifiersearch.h +++ b/src/lib/corelib/language/identifiersearch.h @@ -57,8 +57,8 @@ public: void add(const QString &name, bool *found); private: - bool preVisit(QbsQmlJS::AST::Node *); - bool visit(QbsQmlJS::AST::IdentifierExpression *e); + bool preVisit(QbsQmlJS::AST::Node *) override; + bool visit(QbsQmlJS::AST::IdentifierExpression *e) override; QMap<QString, bool *> m_requests; int m_numberOfFoundIds; diff --git a/src/lib/corelib/language/item.h b/src/lib/corelib/language/item.h index 6756e576d..2dafaa774 100644 --- a/src/lib/corelib/language/item.h +++ b/src/lib/corelib/language/item.h @@ -97,7 +97,7 @@ public: Item *outerItem() const { return m_outerItem; } Item *parent() const { return m_parent; } const FileContextPtr &file() const { return m_file; } - QList<Item *> children() const { return m_children; } + const QList<Item *> &children() const { return m_children; } Item *child(ItemType type, bool checkForMultiple = true) const; const PropertyMap &properties() const { return m_properties; } const PropertyDeclarationMap &propertyDeclarations() const { return m_propertyDeclarations; } @@ -134,6 +134,7 @@ public: void setOuterItem(Item *item) { m_outerItem = item; } void setChildren(const QList<Item *> &children) { m_children = children; } void setParent(Item *item) { m_parent = item; } + void childrenReserve(int size) { m_children.reserve(size); } static void addChild(Item *parent, Item *child); static void removeChild(Item *parent, Item *child); void dump() const; diff --git a/src/lib/corelib/language/itemreaderastvisitor.cpp b/src/lib/corelib/language/itemreaderastvisitor.cpp index 5b042ffb4..2ea306138 100644 --- a/src/lib/corelib/language/itemreaderastvisitor.cpp +++ b/src/lib/corelib/language/itemreaderastvisitor.cpp @@ -75,12 +75,12 @@ ItemReaderASTVisitor::ItemReaderASTVisitor(ItemReaderVisitorState &visitorState, { } -bool ItemReaderASTVisitor::visit(AST::UiImportList *uiImportList) +bool ItemReaderASTVisitor::visit(AST::UiProgram *uiProgram) { ASTImportsHandler importsHandler(m_visitorState, m_logger, m_file); - importsHandler.handleImports(uiImportList); + importsHandler.handleImports(uiProgram->imports); m_typeNameToFile = importsHandler.typeNameFileMap(); - return false; + return true; } static ItemValuePtr findItemProperty(const Item *container, const Item *item) diff --git a/src/lib/corelib/language/itemreaderastvisitor.h b/src/lib/corelib/language/itemreaderastvisitor.h index 6bd39ceb2..4a0bedc91 100644 --- a/src/lib/corelib/language/itemreaderastvisitor.h +++ b/src/lib/corelib/language/itemreaderastvisitor.h @@ -67,7 +67,7 @@ public: Item *rootItem() const { return m_item; } private: - bool visit(QbsQmlJS::AST::UiImportList *uiImportList) override; + bool visit(QbsQmlJS::AST::UiProgram *uiProgram) override; bool visit(QbsQmlJS::AST::UiObjectDefinition *ast) override; bool visit(QbsQmlJS::AST::UiPublicMember *ast) override; bool visit(QbsQmlJS::AST::UiScriptBinding *ast) override; diff --git a/src/lib/corelib/language/itemtype.h b/src/lib/corelib/language/itemtype.h index c0e76c94b..324a1fb87 100644 --- a/src/lib/corelib/language/itemtype.h +++ b/src/lib/corelib/language/itemtype.h @@ -53,6 +53,7 @@ enum class ItemType { Export, FileTagger, Group, + JobLimit, Module, Parameter, Parameters, diff --git a/src/lib/corelib/language/language.cpp b/src/lib/corelib/language/language.cpp index 5351ba80c..d8a5d9162 100644 --- a/src/lib/corelib/language/language.cpp +++ b/src/lib/corelib/language/language.cpp @@ -279,12 +279,6 @@ QString Rule::toString() const + inputTagsSorted.join(QLatin1Char(',')) + QLatin1Char(']'); } -bool Rule::acceptsAsInput(Artifact *artifact) const -{ - return artifact->fileTags().intersects(inputs) - && !artifact->fileTags().intersects(excludedInputs); -} - FileTags Rule::staticOutputFileTags() const { FileTags result; @@ -394,36 +388,6 @@ void ResolvedProduct::store(PersistentPool &pool) serializationOp<PersistentPool::Store>(pool); } -void ResolvedProduct::registerArtifactWithChangedInputs(Artifact *artifact) -{ - QBS_CHECK(buildData); - QBS_CHECK(artifact->product == this); - QBS_CHECK(artifact->transformer); - if (artifact->transformer->rule->multiplex) { - // Reapplication of rules only makes sense for multiplex rules (e.g. linker). - buildData->addArtifactWithChangedInputsForRule(artifact->transformer->rule, artifact); - } -} - -void ResolvedProduct::unregisterArtifactWithChangedInputs(Artifact *artifact) -{ - QBS_CHECK(buildData); - QBS_CHECK(artifact->product == this); - QBS_CHECK(artifact->transformer); - buildData->removeArtifactWithChangedInputsForRule(artifact->transformer->rule, artifact); -} - -void ResolvedProduct::unmarkForReapplication(const RuleConstPtr &rule) -{ - QBS_CHECK(buildData); - buildData->removeAllArtifactsWithChangedInputsForRule(rule); -} - -bool ResolvedProduct::isMarkedForReapplication(const RuleConstPtr &rule) const -{ - return buildData->ruleHasArtifactWithChangedInputs(rule); -} - ArtifactSet ResolvedProduct::lookupArtifactsByFileTag(const FileTag &tag) const { QBS_CHECK(buildData); @@ -444,8 +408,8 @@ ArtifactSet ResolvedProduct::targetArtifacts() const QBS_CHECK(buildData); ArtifactSet taSet; for (Artifact * const a : buildData->rootArtifacts()) { - if (a->fileTags().intersects(fileTags)) - taSet << a; + QBS_CHECK(a->fileTags().intersects(fileTags)); + taSet << a; } return taSet; } @@ -879,12 +843,6 @@ bool operator==(const SourceArtifactInternal &sa1, const SourceArtifactInternal && *sa1.properties == *sa2.properties; } -bool sourceArtifactSetsAreEqual(const std::vector<SourceArtifactPtr> &l1, - const std::vector<SourceArtifactPtr> &l2) -{ - return listsAreEqual(l1, l2); -} - bool operator==(const Rule &r1, const Rule &r2) { if (r1.artifacts.size() != r2.artifacts.size()) diff --git a/src/lib/corelib/language/language.h b/src/lib/corelib/language/language.h index 571681241..f9d69efff 100644 --- a/src/lib/corelib/language/language.h +++ b/src/lib/corelib/language/language.h @@ -49,6 +49,7 @@ #include <buildgraph/forward_decls.h> #include <tools/codelocation.h> #include <tools/filetime.h> +#include <tools/joblimits.h> #include <tools/persistence.h> #include <tools/set.h> #include <tools/weakpointer.h> @@ -245,9 +246,6 @@ inline bool operator!=(const SourceArtifactInternal &sa1, const SourceArtifactIn return !(sa1 == sa2); } -bool sourceArtifactSetsAreEqual(const std::vector<SourceArtifactPtr> &l1, - const std::vector<SourceArtifactPtr> &l2); - class SourceWildCards { public: @@ -427,7 +425,6 @@ public: static QStringList argumentNamesForPrepare(); QString toString() const; - bool acceptsAsInput(Artifact *artifact) const; FileTags staticOutputFileTags() const; FileTags collectedOutputFileTags() const; bool isDynamic() const; @@ -575,6 +572,7 @@ public: std::vector<ResolvedProductPtr> dependencies; QHash<ResolvedProductConstPtr, QVariantMap> dependencyParameters; std::vector<FileTaggerConstPtr> fileTaggers; + JobLimits jobLimits; std::vector<ResolvedModulePtr> modules; QHash<ResolvedModuleConstPtr, QVariantMap> moduleParameters; std::vector<ResolvedScannerConstPtr> scanners; @@ -594,10 +592,6 @@ public: std::vector<SourceArtifactPtr> allEnabledFiles() const; FileTags fileTagsForFileName(const QString &fileName) const; - void registerArtifactWithChangedInputs(Artifact *artifact); - void unregisterArtifactWithChangedInputs(Artifact *artifact); - void unmarkForReapplication(const RuleConstPtr &rule); - bool isMarkedForReapplication(const RuleConstPtr &rule) const; ArtifactSet lookupArtifactsByFileTag(const FileTag &tag) const; ArtifactSet lookupArtifactsByFileTags(const FileTags &tags) const; ArtifactSet targetArtifacts() const; @@ -636,7 +630,8 @@ private: missingSourceFiles, location, productProperties, moduleProperties, rules, dependencies, dependencyParameters, fileTaggers, modules, moduleParameters, scanners, groups, - artifactProperties, probes, exportedModule, buildData); + artifactProperties, probes, exportedModule, buildData, + jobLimits); } QHash<QString, QString> m_executablePathCache; diff --git a/src/lib/corelib/language/moduleloader.cpp b/src/lib/corelib/language/moduleloader.cpp index 583354c4d..ff5fd2f84 100644 --- a/src/lib/corelib/language/moduleloader.cpp +++ b/src/lib/corelib/language/moduleloader.cpp @@ -166,7 +166,7 @@ public: for (ProductContext * const rootProduct : rootProducts) traverse(rootProduct); if (m_sortedProducts.size() < allProducts.size()) { - for (auto * const product : qAsConst(allProducts)) { + for (auto const product : qAsConst(allProducts)) { QList<ModuleLoader::ProductContext *> path; findCycle(product, path); } @@ -175,7 +175,7 @@ public: } // No product at position i has dependencies to a product at position j > i. - QList<ProductContext *> sortedProducts() + const QList<ProductContext *> &sortedProducts() const { return m_sortedProducts; } @@ -201,7 +201,7 @@ private: throw error; } path << product; - for (auto * const dep : m_dependencyMap.value(product)) + for (auto const dep : m_dependencyMap.value(product)) findCycle(dep, path); path.removeLast(); } @@ -3370,6 +3370,8 @@ void ModuleLoader::instantiateModule(ProductContext *productContext, Item *expor void ModuleLoader::createChildInstances(Item *instance, Item *prototype, QHash<Item *, Item *> *prototypeInstanceMap) const { + instance->childrenReserve(instance->children().size() + prototype->children().size()); + for (Item * const childPrototype : prototype->children()) { Item *childInstance = Item::create(m_pool, childPrototype->type()); prototypeInstanceMap->insert(childPrototype, childInstance); @@ -3867,8 +3869,7 @@ void ModuleLoader::copyGroupsFromModuleToProduct(const ProductContext &productCo const Item::Module &module, const Item *modulePrototype) { - const auto children = modulePrototype->children(); - for (Item * const child : children) { + for (Item * const child : modulePrototype->children()) { if (child->type() == ItemType::Group) { Item * const clonedGroup = child->clone(); clonedGroup->setScope(productContext.scope); diff --git a/src/lib/corelib/language/moduleloader.h b/src/lib/corelib/language/moduleloader.h index 7be9a0fcc..7b9e0bede 100644 --- a/src/lib/corelib/language/moduleloader.h +++ b/src/lib/corelib/language/moduleloader.h @@ -324,7 +324,7 @@ private: void resolveProbe(ProductContext *productContext, Item *parent, Item *probe); void checkCancelation() const; bool checkItemCondition(Item *item, Item *itemToDisable = nullptr); - QStringList readExtraSearchPaths(Item *item, bool *wasSet = 0); + QStringList readExtraSearchPaths(Item *item, bool *wasSet = nullptr); void copyProperties(const Item *sourceProject, Item *targetProject); Item *wrapInProjectIfNecessary(Item *item); static QString findExistingModulePath(const QString &searchPath, diff --git a/src/lib/corelib/language/preparescriptobserver.cpp b/src/lib/corelib/language/preparescriptobserver.cpp index eca0ccf29..632cbfb51 100644 --- a/src/lib/corelib/language/preparescriptobserver.cpp +++ b/src/lib/corelib/language/preparescriptobserver.cpp @@ -42,7 +42,11 @@ #include "property.h" #include "scriptengine.h" +#include <buildgraph/artifact.h> +#include <language/language.h> +#include <tools/scripttools.h> #include <tools/stlutils.h> +#include <tools/stringconstants.h> #include <QtScript/qscriptvalue.h> @@ -80,6 +84,13 @@ void PrepareScriptObserver::onPropertyRead(const QScriptValue &object, const QSt Property(it->second.first, it->second.second, name, value.toVariant(), Property::PropertyInParameters)); } + if (name == StringConstants::fileTagsProperty() && m_artifactIds.contains(objectId)) { + const Artifact * const artifact = attachedPointer<Artifact>(object); + QBS_CHECK(artifact); + const Property p(artifact->product->uniqueName(), QString(), name, value.toVariant(), + Property::PropertyInArtifact); + engine()->addPropertyRequestedFromArtifact(artifact, p); + } } } // namespace Internal diff --git a/src/lib/corelib/language/preparescriptobserver.h b/src/lib/corelib/language/preparescriptobserver.h index e70652eb4..36e395efc 100644 --- a/src/lib/corelib/language/preparescriptobserver.h +++ b/src/lib/corelib/language/preparescriptobserver.h @@ -68,6 +68,7 @@ public: m_exportsObjectIds.insert(std::make_pair(exportsId, product)); } + void addArtifactId(qint64 artifactId) { m_artifactIds.insert(artifactId); } bool addImportId(qint64 importId) { return m_importIds.insert(importId).second; } void clearImportIds() { m_importIds.clear(); } void addParameterObjectId(qint64 id, const QString &productName, const QString &depName, @@ -79,12 +80,14 @@ public: } private: - void onPropertyRead(const QScriptValue &object, const QString &name, const QScriptValue &value); + void onPropertyRead(const QScriptValue &object, const QString &name, + const QScriptValue &value) override; std::unordered_map<qint64, QString> m_projectObjectIds; std::unordered_map<qint64, std::pair<QString, QString>> m_parameterObjects; std::unordered_map<qint64, const ResolvedProduct *> m_exportsObjectIds; Set<qint64> m_importIds; + Set<qint64> m_artifactIds; }; } // namespace Internal diff --git a/src/lib/corelib/language/projectresolver.cpp b/src/lib/corelib/language/projectresolver.cpp index cf0e6b706..7deb964b3 100644 --- a/src/lib/corelib/language/projectresolver.cpp +++ b/src/lib/corelib/language/projectresolver.cpp @@ -56,6 +56,7 @@ #include <logging/translator.h> #include <tools/error.h> #include <tools/fileinfo.h> +#include <tools/joblimits.h> #include <tools/jsliterals.h> #include <tools/profiling.h> #include <tools/progressobserver.h> @@ -89,6 +90,7 @@ struct ProjectResolver::ProjectContext ResolvedProjectPtr project; std::vector<FileTaggerConstPtr> fileTaggers; std::vector<RulePtr> rules; + JobLimits jobLimits; ResolvedModulePtr dummyModule; }; @@ -97,7 +99,7 @@ struct ProjectResolver::ProductContext ResolvedProductPtr product; QString buildDirectory; Item *item; - typedef std::pair<ArtifactPropertiesPtr, CodeLocation> ArtifactPropertiesInfo; + typedef std::pair<ArtifactPropertiesPtr, std::vector<CodeLocation>> ArtifactPropertiesInfo; QHash<QStringList, ArtifactPropertiesInfo> artifactPropertiesPerFilter; ProjectResolver::FileLocations sourceArtifactLocations; GroupConstPtr currentGroup; @@ -106,6 +108,7 @@ struct ProjectResolver::ProductContext struct ProjectResolver::ModuleContext { ResolvedModulePtr module; + JobLimits jobLimits; }; class CancelException { }; @@ -333,6 +336,7 @@ void ProjectResolver::resolveProjectFully(Item *item, ProjectResolver::ProjectCo { ItemType::Product, &ProjectResolver::resolveProduct }, { ItemType::Probe, &ProjectResolver::ignoreItem }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::PropertyOptions, &ProjectResolver::ignoreItem } }; @@ -494,6 +498,7 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon { ItemType::Depends, &ProjectResolver::ignoreItem }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Group, &ProjectResolver::resolveGroup }, { ItemType::Product, &ProjectResolver::resolveShadowProduct }, { ItemType::Export, &ProjectResolver::resolveExport }, @@ -504,6 +509,11 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon for (Item * const child : qAsConst(subItems)) callItemFunction(mapping, child, projectContext); + for (const ProjectContext *p = projectContext; p; p = p->parentContext) { + JobLimits tempLimits = p->jobLimits; + product->jobLimits = tempLimits.update(product->jobLimits); + } + resolveModules(item, projectContext); for (const FileTag &t : qAsConst(product->fileTags)) @@ -512,12 +522,19 @@ void ProjectResolver::resolveProductFully(Item *item, ProjectContext *projectCon void ProjectResolver::resolveModules(const Item *item, ProjectContext *projectContext) { + JobLimits jobLimits; for (const Item::Module &m : item->modules()) - resolveModule(m.name, m.item, m.isProduct, m.parameters, projectContext); + resolveModule(m.name, m.item, m.isProduct, m.parameters, jobLimits, projectContext); + for (int i = 0; i < jobLimits.count(); ++i) { + const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i); + if (m_productContext->product->jobLimits.getLimit(moduleJobLimit.pool()) == -1) + m_productContext->product->jobLimits.setJobLimit(moduleJobLimit); + } } void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, ProjectContext *projectContext) + const QVariantMap ¶meters, JobLimits &jobLimits, + ProjectContext *projectContext) { checkCancelation(); if (!item->isPresentModule()) @@ -550,6 +567,7 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b { ItemType::Group, &ProjectResolver::ignoreItem }, { ItemType::Rule, &ProjectResolver::resolveRule }, { ItemType::FileTagger, &ProjectResolver::resolveFileTagger }, + { ItemType::JobLimit, &ProjectResolver::resolveJobLimit }, { ItemType::Scanner, &ProjectResolver::resolveScanner }, { ItemType::PropertyOptions, &ProjectResolver::ignoreItem }, { ItemType::Depends, &ProjectResolver::ignoreItem }, @@ -559,6 +577,12 @@ void ProjectResolver::resolveModule(const QualifiedId &moduleName, Item *item, b }; for (Item *child : item->children()) callItemFunction(mapping, child, projectContext); + for (int i = 0; i < moduleContext.jobLimits.count(); ++i) { + const JobLimit &newJobLimit = moduleContext.jobLimits.jobLimitAt(i); + const int oldLimit = jobLimits.getLimit(newJobLimit.pool()); + if (oldLimit == -1 || oldLimit > newJobLimit.limit()) + jobLimits.setJobLimit(newJobLimit); + } m_moduleContext = oldModuleContext; } @@ -692,19 +716,30 @@ void ProjectResolver::resolveGroup(Item *item, ProjectContext *projectContext) void ProjectResolver::resolveGroupFully(Item *item, ProjectResolver::ProjectContext *projectContext, bool isEnabled) { - PropertyMapPtr moduleProperties = m_productContext->currentGroup - ? m_productContext->currentGroup->properties - : m_productContext->product->moduleProperties; - const QVariantMap newModuleProperties - = resolveAdditionalModuleProperties(item, moduleProperties->value()); - if (!newModuleProperties.empty()) { - moduleProperties = PropertyMapInternal::create(); - moduleProperties->setValue(newModuleProperties); - } - AccumulatingTimer groupTimer(m_setupParams.logElapsedTime() ? &m_elapsedTimeGroups : nullptr); + const auto getGroupPropertyMap = [this, item](const ArtifactProperties *existingProps) { + PropertyMapPtr moduleProperties; + bool newPropertyMapRequired = false; + if (existingProps) + moduleProperties = existingProps->propertyMap(); + if (!moduleProperties) { + newPropertyMapRequired = true; + moduleProperties = m_productContext->currentGroup + ? m_productContext->currentGroup->properties + : m_productContext->product->moduleProperties; + } + const QVariantMap newModuleProperties + = resolveAdditionalModuleProperties(item, moduleProperties->value()); + if (!newModuleProperties.empty()) { + if (newPropertyMapRequired) + moduleProperties = PropertyMapInternal::create(); + moduleProperties->setValue(newModuleProperties); + } + return moduleProperties; + }; + QStringList files = m_evaluator->stringListValue(item, StringConstants::filesProperty()); bool fileTagsSet; const FileTags fileTags = m_evaluator->fileTagsValue(item, StringConstants::fileTagsProperty(), @@ -716,28 +751,30 @@ void ProjectResolver::resolveGroupFully(Item *item, ProjectResolver::ProjectCont throw ErrorInfo(Tr::tr("Group.files and Group.fileTagsFilters are exclusive."), item->location()); - ProductContext::ArtifactPropertiesInfo apinfo - = m_productContext->artifactPropertiesPerFilter.value(fileTagsFilter); + if (!isEnabled) + return; + + ProductContext::ArtifactPropertiesInfo &apinfo + = m_productContext->artifactPropertiesPerFilter[fileTagsFilter]; if (apinfo.first) { - if (apinfo.second.filePath() == item->location().filePath()) { + const auto it = std::find_if(apinfo.second.cbegin(), apinfo.second.cend(), + [item](const CodeLocation &loc) { + return item->location().filePath() == loc.filePath(); + }); + if (it != apinfo.second.cend()) { ErrorInfo error(Tr::tr("Conflicting fileTagsFilter in Group items.")); - error.append(Tr::tr("First item"), apinfo.second); + error.append(Tr::tr("First item"), *it); error.append(Tr::tr("Second item"), item->location()); throw error; } - - // Discard any Group with the same fileTagsFilter that was defined in a base file. - removeAll(m_productContext->product->artifactProperties, apinfo.first); + } else { + apinfo.first = ArtifactProperties::create(); + apinfo.first->setFileTagsFilter(FileTags::fromStringList(fileTagsFilter)); + m_productContext->product->artifactProperties.push_back(apinfo.first); } - if (!isEnabled) - return; - ArtifactPropertiesPtr aprops = ArtifactProperties::create(); - aprops->setFileTagsFilter(FileTags::fromStringList(fileTagsFilter)); - aprops->setExtraFileTags(fileTags); - aprops->setPropertyMapInternal(moduleProperties); - m_productContext->product->artifactProperties.push_back(aprops); - m_productContext->artifactPropertiesPerFilter.insert(fileTagsFilter, - ProductContext::ArtifactPropertiesInfo(aprops, item->location())); + apinfo.second.push_back(item->location()); + apinfo.first->setPropertyMapInternal(getGroupPropertyMap(apinfo.first.get())); + apinfo.first->addExtraFileTags(fileTags); return; } QStringList patterns; @@ -757,7 +794,7 @@ void ProjectResolver::resolveGroupFully(Item *item, ProjectResolver::ProjectCont } group->location = item->location(); group->enabled = isEnabled; - group->properties = moduleProperties; + group->properties = getGroupPropertyMap(nullptr); group->fileTags = fileTags; group->overrideTags = m_evaluator->boolValue(item, StringConstants::overrideTagsProperty()); if (group->overrideTags && fileTagsSet) { @@ -822,19 +859,26 @@ void ProjectResolver::resolveGroupFully(Item *item, ProjectResolver::ProjectCont resolveGroup(childItem, projectContext); } -void ProjectResolver::adaptExportedPropertyValues() +void ProjectResolver::adaptExportedPropertyValues(const Item *shadowProductItem) { ExportedModule &m = m_productContext->product->exportedModule; const QVariantList prefixList = m.propertyValues.take( StringConstants::prefixMappingProperty()).toList(); - if (prefixList.empty()) - return; + const QString shadowProductName = m_evaluator->stringValue( + shadowProductItem, StringConstants::nameProperty()); + const QString shadowProductBuildDir = m_evaluator->stringValue( + shadowProductItem, StringConstants::buildDirectoryProperty()); QVariantMap prefixMap; for (const QVariant &v : prefixList) { const QVariantMap o = v.toMap(); prefixMap.insert(o.value(QLatin1String("prefix")).toString(), o.value(QLatin1String("replacement")).toString()); } + const auto valueRefersToImportingProduct + = [shadowProductName, shadowProductBuildDir](const QString &value) { + return value.toLower().contains(shadowProductName.toLower()) + || value.contains(shadowProductBuildDir); + }; static const auto stringMapper = [](const QVariantMap &mappings, const QString &value) -> QString { for (auto it = mappings.cbegin(); it != mappings.cend(); ++it) { @@ -843,17 +887,19 @@ void ProjectResolver::adaptExportedPropertyValues() } return value; }; - static const auto stringListMapper = [](const QVariantMap &mappings, const QStringList &value) - -> QStringList { + const auto stringListMapper = [&valueRefersToImportingProduct]( + const QVariantMap &mappings, const QStringList &value) -> QStringList { QStringList result; result.reserve(value.size()); for (const QString &s : value) { - result.push_back(stringMapper(mappings, s)); + if (!valueRefersToImportingProduct(s)) + result.push_back(stringMapper(mappings, s)); } return result; }; - static const std::function<QVariant(const QVariantMap &, const QVariant &)> mapper - = [](const QVariantMap &mappings, const QVariant &value) -> QVariant { + const std::function<QVariant(const QVariantMap &, const QVariant &)> mapper + = [&stringListMapper, &mapper]( + const QVariantMap &mappings, const QVariant &value) -> QVariant { switch (static_cast<QMetaType::Type>(value.type())) { case QMetaType::QString: return stringMapper(mappings, value.toString()); @@ -938,7 +984,9 @@ void ProjectResolver::resolveShadowProduct(Item *item, ProjectResolver::ProjectC collectPropertiesForModuleInExportItem(dep); break; } - adaptExportedPropertyValues(); + try { + adaptExportedPropertyValues(item); + } catch (const ErrorInfo &) {} m_productExportInfo.push_back(std::make_pair(m_productContext->product, item)); } @@ -1162,19 +1210,18 @@ void ProjectResolver::resolveRule(Item *item, ProjectContext *projectContext) rule->prepareScript.initialize(scriptFunctionValue(item, StringConstants::prepareProperty())); rule->outputArtifactsScript.initialize(scriptFunctionValue( item, StringConstants::outputArtifactsProperty())); + rule->outputFileTags = m_evaluator->fileTagsValue( + item, StringConstants::outputFileTagsProperty()); if (rule->outputArtifactsScript.isValid()) { if (hasArtifactChildren) throw ErrorInfo(Tr::tr("The Rule.outputArtifacts script is not allowed in rules " "that contain Artifact items."), item->location()); - rule->outputFileTags = m_evaluator->fileTagsValue( - item, StringConstants::outputFileTagsProperty()); - if (rule->outputFileTags.empty()) - throw ErrorInfo(Tr::tr("Rule.outputFileTags must be specified if " - "Rule.outputArtifacts is specified."), - item->location()); } - + if (!hasArtifactChildren && rule->outputFileTags.empty()) { + throw ErrorInfo(Tr::tr("A rule needs to have Artifact items or a non-empty " + "outputFileTags property."), item->location()); + } rule->multiplex = m_evaluator->boolValue(item, StringConstants::multiplexProperty()); rule->alwaysRun = m_evaluator->boolValue(item, StringConstants::alwaysRunProperty()); rule->inputs = m_evaluator->fileTagsValue(item, StringConstants::inputsProperty()); @@ -1301,6 +1348,35 @@ void ProjectResolver::resolveFileTagger(Item *item, ProjectContext *projectConte fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority)); } +void ProjectResolver::resolveJobLimit(Item *item, ProjectResolver::ProjectContext *projectContext) +{ + if (!m_evaluator->boolValue(item, StringConstants::conditionProperty())) + return; + const QString jobPool = m_evaluator->stringValue(item, StringConstants::jobPoolProperty()); + if (jobPool.isEmpty()) + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.") + .arg(StringConstants::jobPoolProperty()), item->location()); + bool jobCountWasSet; + const int jobCount = m_evaluator->intValue(item, StringConstants::jobCountProperty(), -1, + &jobCountWasSet); + if (!jobCountWasSet) { + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + if (jobCount < 0) { + throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + JobLimits &jobLimits = m_moduleContext + ? m_moduleContext->jobLimits + : m_productContext ? m_productContext->product->jobLimits + : projectContext->jobLimits; + JobLimit jobLimit(jobPool, jobCount); + const int oldLimit = jobLimits.getLimit(jobPool); + if (oldLimit == -1 || oldLimit > jobCount) + jobLimits.setJobLimit(jobLimit); +} + void ProjectResolver::resolveScanner(Item *item, ProjectResolver::ProjectContext *projectContext) { checkCancelation(); diff --git a/src/lib/corelib/language/projectresolver.h b/src/lib/corelib/language/projectresolver.h index e81034b5a..777451fa6 100644 --- a/src/lib/corelib/language/projectresolver.h +++ b/src/lib/corelib/language/projectresolver.h @@ -56,6 +56,7 @@ #include <vector> namespace qbs { +class JobLimits; namespace Internal { class Evaluator; @@ -89,8 +90,8 @@ private: class ProductContextSwitcher; void checkCancelation() const; - QString verbatimValue(const ValueConstPtr &value, bool *propertyWasSet = 0) const; - QString verbatimValue(Item *item, const QString &name, bool *propertyWasSet = 0) const; + QString verbatimValue(const ValueConstPtr &value, bool *propertyWasSet = nullptr) const; + QString verbatimValue(Item *item, const QString &name, bool *propertyWasSet = nullptr) const; ScriptFunctionPtr scriptFunctionValue(Item *item, const QString &name) const; ResolvedFileContextPtr resolvedFileContext(const FileContextConstPtr &ctx) const; void ignoreItem(Item *item, ProjectContext *projectContext); @@ -102,7 +103,8 @@ private: void resolveProductFully(Item *item, ProjectContext *projectContext); void resolveModules(const Item *item, ProjectContext *projectContext); void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, ProjectContext *projectContext); + const QVariantMap ¶meters, JobLimits &jobLimits, + ProjectContext *projectContext); void gatherProductTypes(ResolvedProduct *product, Item *item); QVariantMap resolveAdditionalModuleProperties(const Item *group, const QVariantMap ¤tValues); @@ -118,6 +120,7 @@ private: const QStringList &namePrefix, QualifiedIdSet *seenBindings); void resolveFileTagger(Item *item, ProjectContext *projectContext); + void resolveJobLimit(Item *item, ProjectContext *projectContext); void resolveScanner(Item *item, ProjectContext *projectContext); void resolveProductDependencies(const ProjectContext &projectContext); void postProcess(const ResolvedProductPtr &product, ProjectContext *projectContext) const; @@ -131,7 +134,7 @@ private: QVariantMap &result, bool checkErrors); void createProductConfig(ResolvedProduct *product); ProjectContext createProjectContext(ProjectContext *parentProjectContext) const; - void adaptExportedPropertyValues(); + void adaptExportedPropertyValues(const Item *shadowProductItem); void collectExportedProductDependencies(); struct ProductDependencyInfo diff --git a/src/lib/corelib/language/property.h b/src/lib/corelib/language/property.h index bae9de74e..8ad992bdb 100644 --- a/src/lib/corelib/language/property.h +++ b/src/lib/corelib/language/property.h @@ -56,7 +56,8 @@ public: PropertyInModule, PropertyInProduct, PropertyInProject, - PropertyInParameters + PropertyInParameters, + PropertyInArtifact, }; Property() diff --git a/src/lib/corelib/language/scriptengine.cpp b/src/lib/corelib/language/scriptengine.cpp index e983ae945..fb4706b3b 100644 --- a/src/lib/corelib/language/scriptengine.cpp +++ b/src/lib/corelib/language/scriptengine.cpp @@ -371,6 +371,11 @@ void ScriptEngine::setEnvironment(const QProcessEnvironment &env) void ScriptEngine::importFile(const QString &filePath, QScriptValue &targetObject) { AccumulatingTimer importTimer(m_elapsedTimeImporting != -1 ? &m_elapsedTimeImporting : nullptr); + QScriptValue &evaluationResult = m_jsFileCache[filePath]; + if (evaluationResult.isValid()) { + ScriptImporter::copyProperties(evaluationResult, targetObject); + return; + } QFile file(filePath); if (Q_UNLIKELY(!file.open(QFile::ReadOnly))) throw ErrorInfo(tr("Cannot open '%1'.").arg(filePath)); @@ -379,7 +384,7 @@ void ScriptEngine::importFile(const QString &filePath, QScriptValue &targetObjec const QString sourceCode = stream.readAll(); file.close(); m_currentDirPathStack.push(FileInfo::path(filePath)); - m_scriptImporter->importSourceCode(sourceCode, filePath, targetObject); + evaluationResult = m_scriptImporter->importSourceCode(sourceCode, filePath, targetObject); m_currentDirPathStack.pop(); } @@ -763,7 +768,7 @@ void ScriptEngine::extendJavaScriptBuiltins() } void ScriptEngine::installFunction(const QString &name, int length, QScriptValue *functionValue, - FunctionSignature f, QScriptValue *targetObject = 0) + FunctionSignature f, QScriptValue *targetObject = nullptr) { if (!functionValue->isValid()) *functionValue = newFunction(f, length); diff --git a/src/lib/corelib/language/scriptengine.h b/src/lib/corelib/language/scriptengine.h index eff4cb48f..226cf16a9 100644 --- a/src/lib/corelib/language/scriptengine.h +++ b/src/lib/corelib/language/scriptengine.h @@ -164,6 +164,10 @@ public: void addImportRequestedInScript(qint64 importValueId); std::vector<QString> importedFilesUsedInScript() const; + void setUsesIo() { m_usesIo = true; } + void clearUsesIo() { m_usesIo = false; } + bool usesIo() const { return m_usesIo; } + void enableProfiling(bool enable); void setPropertyCacheEnabled(bool enable) { m_propertyCacheEnabled = enable; } @@ -314,6 +318,7 @@ private: QScriptClass *m_productPropertyScriptClass = nullptr; QScriptClass *m_artifactsScriptClass = nullptr; QHash<JsImport, QScriptValue> m_jsImportCache; + std::unordered_map<QString, QScriptValue> m_jsFileCache; bool m_propertyCacheEnabled; bool m_active; QHash<PropertyCacheKey, QVariant> m_propertyCache; @@ -336,6 +341,7 @@ private: QScriptValue m_consoleObject; QScriptValue m_cancelationError; qint64 m_elapsedTimeImporting = -1; + bool m_usesIo = false; EvalContext m_evalContext; std::vector<ResourceAcquiringScriptObject *> m_resourceAcquiringScriptObjects; const std::unique_ptr<PrepareScriptObserver> m_observer; diff --git a/src/lib/corelib/language/scriptimporter.cpp b/src/lib/corelib/language/scriptimporter.cpp index 6249e360e..c5dc887b7 100644 --- a/src/lib/corelib/language/scriptimporter.cpp +++ b/src/lib/corelib/language/scriptimporter.cpp @@ -120,7 +120,7 @@ ScriptImporter::ScriptImporter(ScriptEngine *scriptEngine) { } -void ScriptImporter::importSourceCode(const QString &sourceCode, const QString &filePath, +QScriptValue ScriptImporter::importSourceCode(const QString &sourceCode, const QString &filePath, QScriptValue &targetObject) { Q_ASSERT(targetObject.isObject()); @@ -146,6 +146,7 @@ void ScriptImporter::importSourceCode(const QString &sourceCode, const QString & QScriptValue result = m_engine->evaluate(code, filePath, 0); throwOnEvaluationError(m_engine, result, [&filePath] () { return CodeLocation(filePath, 0); }); copyProperties(result, targetObject); + return result; } void ScriptImporter::copyProperties(const QScriptValue &src, QScriptValue &dst) diff --git a/src/lib/corelib/language/scriptimporter.h b/src/lib/corelib/language/scriptimporter.h index 0f37038ff..8cff09382 100644 --- a/src/lib/corelib/language/scriptimporter.h +++ b/src/lib/corelib/language/scriptimporter.h @@ -53,11 +53,11 @@ class ScriptImporter { public: ScriptImporter(ScriptEngine *scriptEngine); - void importSourceCode(const QString &sourceCode, const QString &filePath, QScriptValue &targetObject); + QScriptValue importSourceCode(const QString &sourceCode, const QString &filePath, QScriptValue &targetObject); -private: static void copyProperties(const QScriptValue &src, QScriptValue &dst); +private: ScriptEngine *m_engine; QHash<QString, QString> m_sourceCodeCache; }; diff --git a/src/lib/corelib/language/value.h b/src/lib/corelib/language/value.h index 6e7eca2f8..fde8e79e4 100644 --- a/src/lib/corelib/language/value.h +++ b/src/lib/corelib/language/value.h @@ -117,8 +117,8 @@ public: static JSSourceValuePtr QBS_AUTOTEST_EXPORT create(bool createdByPropertiesBlock = false); ~JSSourceValue(); - void apply(ValueHandler *handler) { handler->handle(this); } - ValuePtr clone() const; + void apply(ValueHandler *handler) override { handler->handle(this); } + ValuePtr clone() const override; void setSourceCode(const QStringRef &sourceCode) { m_sourceCode = sourceCode; } const QStringRef &sourceCode() const { return m_sourceCode; } @@ -127,7 +127,7 @@ public: void setLocation(int line, int column); int line() const { return m_line; } int column() const { return m_column; } - CodeLocation location() const; + CodeLocation location() const override; void setFile(const FileContextPtr &file) { m_file = file; } const FileContextPtr &file() const { return m_file; } @@ -175,7 +175,7 @@ public: void addAlternative(const Alternative &alternative) { m_alternatives.push_back(alternative); } void clearAlternatives(); - void setDefiningItem(Item *item); + void setDefiningItem(Item *item) override; private: QStringRef m_sourceCode; @@ -211,8 +211,8 @@ class VariantValue : public Value public: static VariantValuePtr create(const QVariant &v = QVariant()); - void apply(ValueHandler *handler) { handler->handle(this); } - ValuePtr clone() const; + void apply(ValueHandler *handler) override { handler->handle(this); } + ValuePtr clone() const override; const QVariant &value() const { return m_value; } diff --git a/src/lib/corelib/tools/buildoptions.cpp b/src/lib/corelib/tools/buildoptions.cpp index ac8e39867..5507e0842 100644 --- a/src/lib/corelib/tools/buildoptions.cpp +++ b/src/lib/corelib/tools/buildoptions.cpp @@ -58,6 +58,8 @@ public: QStringList changedFiles; QStringList filesToConsider; QStringList activeFileTags; + JobLimits jobLimits; + QString settingsDir; int maxJobCount; bool dryRun; bool keepGoing; @@ -68,6 +70,7 @@ public: bool install; bool removeExistingInstallation; bool onlyExecuteRules; + bool jobLimitsFromProjectTakePrecedence = false; }; } // namespace Internal @@ -189,6 +192,44 @@ void BuildOptions::setMaxJobCount(int jobCount) } /*! + * \brief The base directory for qbs settings. + * This value is used to locate profiles and preferences. + */ +QString BuildOptions::settingsDirectory() const +{ + return d->settingsDir; +} + +/*! + * \brief Sets the base directory for qbs settings. + * \param settingsBaseDir Will be used to locate profiles and preferences. + */ +void BuildOptions::setSettingsDirectory(const QString &settingsBaseDir) +{ + d->settingsDir = settingsBaseDir; +} + +JobLimits BuildOptions::jobLimits() const +{ + return d->jobLimits; +} + +void BuildOptions::setJobLimits(const JobLimits &jobLimits) +{ + d->jobLimits = jobLimits; +} + +bool BuildOptions::projectJobLimitsTakePrecedence() const +{ + return d->jobLimitsFromProjectTakePrecedence; +} + +void BuildOptions::setProjectJobLimitsTakePrecedence(bool toggle) +{ + d->jobLimitsFromProjectTakePrecedence = toggle; +} + +/*! * \brief Returns true iff qbs will not actually execute any commands, but just show what * would happen. * The default is false. diff --git a/src/lib/corelib/tools/buildoptions.h b/src/lib/corelib/tools/buildoptions.h index 630a6aa22..cea89d0ea 100644 --- a/src/lib/corelib/tools/buildoptions.h +++ b/src/lib/corelib/tools/buildoptions.h @@ -42,6 +42,7 @@ #include "qbs_export.h" #include "commandechomode.h" +#include "joblimits.h" #include <QtCore/qshareddata.h> @@ -73,6 +74,15 @@ public: int maxJobCount() const; void setMaxJobCount(int jobCount); + QString settingsDirectory() const; + void setSettingsDirectory(const QString &settingsBaseDir); + + JobLimits jobLimits() const; + void setJobLimits(const JobLimits &jobLimits); + + bool projectJobLimitsTakePrecedence() const; + void setProjectJobLimitsTakePrecedence(bool toggle); + bool dryRun() const; void setDryRun(bool dryRun); diff --git a/src/lib/corelib/tools/fileinfo.cpp b/src/lib/corelib/tools/fileinfo.cpp index 3adacc883..15a8e8783 100644 --- a/src/lib/corelib/tools/fileinfo.cpp +++ b/src/lib/corelib/tools/fileinfo.cpp @@ -295,15 +295,10 @@ bool FileInfo::fileExists(const QFileInfo &fi) #define z(x) reinterpret_cast<WIN32_FILE_ATTRIBUTE_DATA*>(const_cast<FileInfo::InternalStatType*>(&x)) -template<bool> struct CompileTimeAssert; -template<> struct CompileTimeAssert<true> {}; - FileInfo::FileInfo(const QString &fileName) { - static CompileTimeAssert< - sizeof(FileInfo::InternalStatType) == sizeof(WIN32_FILE_ATTRIBUTE_DATA) - > internal_type_has_wrong_size; - Q_UNUSED(internal_type_has_wrong_size); + static_assert(sizeof(FileInfo::InternalStatType) == sizeof(WIN32_FILE_ATTRIBUTE_DATA), + "FileInfo::InternalStatType has wrong size."); QString filePath = fileName; @@ -368,7 +363,7 @@ FileInfo::FileInfo(const QString &fileName) bool FileInfo::exists() const { - return m_stat.st_mtime != 0; + return m_stat.st_mode != 0; } FileTime FileInfo::lastModified() const diff --git a/src/lib/corelib/tools/filetime.cpp b/src/lib/corelib/tools/filetime.cpp index 14680ef6e..263950d9c 100644 --- a/src/lib/corelib/tools/filetime.cpp +++ b/src/lib/corelib/tools/filetime.cpp @@ -50,11 +50,6 @@ namespace qbs { namespace Internal { -#ifdef Q_OS_WIN -template<bool> struct CompileTimeAssert; -template<> struct CompileTimeAssert<true> {}; -#endif - #ifdef APPLE_CUSTOM_CLOCK_GETTIME #include <sys/time.h> @@ -77,8 +72,8 @@ int clock_gettime(int /*clk_id*/, struct timespec *t) FileTime::FileTime() { #ifdef Q_OS_WIN - static CompileTimeAssert<sizeof(FileTime::InternalType) == sizeof(FILETIME)> internal_type_has_wrong_size; - Q_UNUSED(internal_type_has_wrong_size); + static_assert(sizeof(FileTime::InternalType) == sizeof(FILETIME), + "FileTime::InternalType has wrong size."); m_fileTime = 0; #elif HAS_CLOCK_GETTIME m_fileTime = {0, 0}; @@ -98,8 +93,8 @@ FileTime::FileTime(const FileTime::InternalType &ft) : m_fileTime(ft) int FileTime::compare(const FileTime &other) const { #ifdef Q_OS_WIN - const FILETIME *const t1 = reinterpret_cast<const FILETIME *>(&m_fileTime); - const FILETIME *const t2 = reinterpret_cast<const FILETIME *>(&other.m_fileTime); + auto const t1 = reinterpret_cast<const FILETIME *>(&m_fileTime); + auto const t2 = reinterpret_cast<const FILETIME *>(&other.m_fileTime); return CompareFileTime(t1, t2); #elif HAS_CLOCK_GETTIME if (m_fileTime.tv_sec < other.m_fileTime.tv_sec) @@ -140,7 +135,7 @@ FileTime FileTime::currentTime() FileTime result; SYSTEMTIME st; GetSystemTime(&st); - FILETIME *const ft = reinterpret_cast<FILETIME *>(&result.m_fileTime); + auto const ft = reinterpret_cast<FILETIME *>(&result.m_fileTime); SystemTimeToFileTime(&st, ft); return result; #elif defined APPLE_CUSTOM_CLOCK_GETTIME @@ -173,7 +168,7 @@ FileTime FileTime::oldestTime() 0 }; FileTime result; - FILETIME *const ft = reinterpret_cast<FILETIME *>(&result.m_fileTime); + auto const ft = reinterpret_cast<FILETIME *>(&result.m_fileTime); SystemTimeToFileTime(&st, ft); return result; #elif HAS_CLOCK_GETTIME @@ -195,7 +190,7 @@ double FileTime::asDouble() const QString FileTime::toString() const { #ifdef Q_OS_WIN - const FILETIME *const ft = reinterpret_cast<const FILETIME *>(&m_fileTime); + auto const ft = reinterpret_cast<const FILETIME *>(&m_fileTime); SYSTEMTIME stUTC, stLocal; FileTimeToSystemTime(ft, &stUTC); SystemTimeToTzSpecificLocalTime(NULL, &stUTC, &stLocal); diff --git a/src/lib/corelib/tools/joblimits.cpp b/src/lib/corelib/tools/joblimits.cpp new file mode 100644 index 000000000..3b1fde83d --- /dev/null +++ b/src/lib/corelib/tools/joblimits.cpp @@ -0,0 +1,185 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "joblimits.h" + +#include <tools/persistence.h> + +#include <utility> +#include <vector> + +namespace qbs { +namespace Internal { + +static int transformLimit(int limitFromUser) +{ + return limitFromUser == 0 + ? std::numeric_limits<int>::max() + : limitFromUser < -1 ? -1 + : limitFromUser; +} + +class JobLimitPrivate : public QSharedData +{ +public: + JobLimitPrivate(const QString &pool, int limit) + : jobLimit(std::make_pair(pool, transformLimit(limit))) + { + } + template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) + { + pool.serializationOp<opType>(jobLimit); + } + std::pair<QString, int> jobLimit; +}; + +class JobLimitsPrivate : public QSharedData +{ +public: + template<PersistentPool::OpType opType> void serializationOp(PersistentPool &pool) + { + pool.serializationOp<opType>(jobLimits); + } + std::vector<JobLimit> jobLimits; +}; + +} // namespace Internal + +JobLimit::JobLimit() : JobLimit(QString(), -1) +{ +} +JobLimit::JobLimit(const QString &pool, int limit) : d(new Internal::JobLimitPrivate(pool, limit)) +{ +} +JobLimit::JobLimit(const JobLimit &other) : d(other.d) { } +JobLimit &JobLimit::operator=(const JobLimit &other) +{ + d = other.d; + return *this; +} +JobLimit::~JobLimit() {} +QString JobLimit::pool() const { return d->jobLimit.first; } +int JobLimit::limit() const { return d->jobLimit.second; } + +void JobLimit::load(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Load>(pool); +} + +void JobLimit::store(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Store>(pool); +} + +JobLimits::JobLimits() : d(new Internal::JobLimitsPrivate) { } +JobLimits::JobLimits(const JobLimits &other) : d(other.d) { } +JobLimits &JobLimits::operator=(const JobLimits &other) +{ + d = other.d; + return *this; +} +JobLimits::~JobLimits() {} + +void JobLimits::setJobLimit(const JobLimit &limit) +{ + for (std::size_t i = 0; i < d->jobLimits.size(); ++i) { + JobLimit ¤tLimit = d->jobLimits.at(i); + if (currentLimit.pool() == limit.pool()) { + if (currentLimit.limit() != limit.limit()) + currentLimit = limit; + return; + } + } + d->jobLimits.push_back(limit); +} + +void JobLimits::setJobLimit(const QString &pool, int limit) +{ + setJobLimit(JobLimit(pool, limit)); +} + +int JobLimits::getLimit(const QString &pool) const +{ + for (const JobLimit &l : d->jobLimits) { + if (l.pool() == pool) + return l.limit(); + } + return -1; +} + +bool JobLimits::isEmpty() const +{ + return d->jobLimits.empty(); +} + +int JobLimits::count() const +{ + return d->jobLimits.size(); +} + +JobLimit JobLimits::jobLimitAt(int i) const +{ + return d->jobLimits.at(i); +} + +JobLimits &JobLimits::update(const JobLimits &other) +{ + if (isEmpty()) { + *this = other; + } else { + for (int i = 0; i < other.count(); ++i) { + const JobLimit &l = other.jobLimitAt(i); + if (l.limit() != -1) + setJobLimit(l); + } + } + return *this; +} + +void JobLimits::load(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Load>(pool); +} + +void JobLimits::store(Internal::PersistentPool &pool) +{ + d->serializationOp<Internal::PersistentPool::Store>(pool); +} + +} // namespace qbs diff --git a/src/lib/corelib/tools/joblimits.h b/src/lib/corelib/tools/joblimits.h new file mode 100644 index 000000000..de95f5513 --- /dev/null +++ b/src/lib/corelib/tools/joblimits.h @@ -0,0 +1,102 @@ +/**************************************************************************** +** +** Copyright (C) 2018 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QBS_JOB_LIMITS_H +#define QBS_JOB_LIMITS_H + +#include "qbs_export.h" + +#include <QtCore/qshareddata.h> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +namespace qbs { +namespace Internal { +class JobLimitPrivate; +class JobLimitsPrivate; +class PersistentPool; +} + +class QBS_EXPORT JobLimit +{ +public: + JobLimit(); + JobLimit(const QString &pool, int limit); + JobLimit(const JobLimit &other); + JobLimit &operator=(const JobLimit &other); + ~JobLimit(); + + QString pool() const; + int limit() const; + + void load(Internal::PersistentPool &pool); + void store(Internal::PersistentPool &pool); +private: + QSharedDataPointer<Internal::JobLimitPrivate> d; +}; + +class QBS_EXPORT JobLimits +{ +public: + JobLimits(); + JobLimits(const JobLimits &other); + JobLimits &operator=(const JobLimits &other); + ~JobLimits(); + + void setJobLimit(const JobLimit &limit); + void setJobLimit(const QString &pool, int limit); + int getLimit(const QString &pool) const; + bool hasLimit(const QString &pool) const { return getLimit(pool) != -1; } + bool isEmpty() const; + + int count() const; + JobLimit jobLimitAt(int i) const; + + JobLimits &update(const JobLimits &other); + + void load(Internal::PersistentPool &pool); + void store(Internal::PersistentPool &pool); +private: + QSharedDataPointer<Internal::JobLimitsPrivate> d; +}; + +} // namespace qbs + +#endif // include guard diff --git a/src/lib/corelib/tools/persistence.cpp b/src/lib/corelib/tools/persistence.cpp index 996c8415d..ec412cf3b 100644 --- a/src/lib/corelib/tools/persistence.cpp +++ b/src/lib/corelib/tools/persistence.cpp @@ -42,14 +42,13 @@ #include "fileinfo.h" #include <logging/translator.h> #include <tools/error.h> -#include <tools/qbsassert.h> #include <QtCore/qdir.h> namespace qbs { namespace Internal { -static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-120"; +static const char QBS_PERSISTENCE_MAGIC[] = "QBSPERSISTENCE-124"; NoBuildGraphError::NoBuildGraphError(const QString &filePath) : ErrorInfo(Tr::tr("Build graph not found for configuration '%1'. Expected location was '%2'.") @@ -82,7 +81,7 @@ void PersistentPool::load(const QString &filePath) QByteArray magic; m_stream >> magic; if (magic != QBS_PERSISTENCE_MAGIC) { - m_stream.setDevice(0); + m_stream.setDevice(nullptr); throw ErrorInfo(Tr::tr("Cannot use stored build graph at '%1': Incompatible file format. " "Expected magic token '%2', got '%3'.") .arg(filePath, QString::fromLatin1(QBS_PERSISTENCE_MAGIC), @@ -121,6 +120,8 @@ void PersistentPool::setupWriteStream(const QString &filePath) m_stream << QByteArray(qstrlen(QBS_PERSISTENCE_MAGIC), 0) << m_headData.projectConfig; m_lastStoredObjectId = 0; m_lastStoredStringId = 0; + m_lastStoredEnvId = 0; + m_lastStoredStringListId = 0; } void PersistentPool::finalizeWriteStream() @@ -153,7 +154,7 @@ void PersistentPool::storeVariant(const QVariant &variant) m_stream << type; switch (type) { case QMetaType::QString: - storeString(variant.toString()); + store(variant.toString()); break; case QMetaType::QStringList: store(variant.toStringList()); @@ -175,7 +176,7 @@ QVariant PersistentPool::loadVariant() QVariant value; switch (type) { case QMetaType::QString: - value = idLoadString(); + value = load<QString>(); break; case QMetaType::QStringList: value = load<QStringList>(); @@ -200,50 +201,48 @@ void PersistentPool::clear() m_inverseStringStorage.clear(); } -const int StringNotFoundId = -1; -const int NullStringId = -2; - -void PersistentPool::storeString(const QString &t) +void PersistentPool::doLoadValue(QString &s) { - if (t.isNull()) { - m_stream << NullStringId; - return; - } - - int id = m_inverseStringStorage.value(t, StringNotFoundId); - if (id < 0) { - id = m_lastStoredStringId++; - m_inverseStringStorage.insert(t, id); - m_stream << id << t; - } else { - m_stream << id; - } + m_stream >> s; } -QString PersistentPool::loadString(int id) +void PersistentPool::doLoadValue(QStringList &l) { - if (id == NullStringId) - return QString(); + int size; + m_stream >> size; + for (int i = 0; i < size; ++i) + l << load<QString>(); +} - QBS_CHECK(id >= 0); +void PersistentPool::doLoadValue(QProcessEnvironment &env) +{ + const QStringList keys = load<QStringList>(); + for (const QString &key : keys) + env.insert(key, load<QString>()); +} - if (id >= static_cast<int>(m_stringStorage.size())) { - QString s; - m_stream >> s; - m_stringStorage.resize(id + 1); - m_stringStorage[id] = s; - return s; - } +void PersistentPool::doStoreValue(const QString &s) +{ + m_stream << s; +} - return m_stringStorage.at(id); +void PersistentPool::doStoreValue(const QStringList &l) +{ + m_stream << l.size(); + for (const QString &s : l) + store(s); } -QString PersistentPool::idLoadString() +void PersistentPool::doStoreValue(const QProcessEnvironment &env) { - int id; - m_stream >> id; - return loadString(id); + const QStringList &keys = env.keys(); + store(keys); + for (const QString &key : keys) + store(env.value(key)); } +const PersistentPool::PersistentObjectId PersistentPool::ValueNotFoundId; +const PersistentPool::PersistentObjectId PersistentPool::EmptyValueId; + } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/tools/persistence.h b/src/lib/corelib/tools/persistence.h index 2687d6120..e8d938ad3 100644 --- a/src/lib/corelib/tools/persistence.h +++ b/src/lib/corelib/tools/persistence.h @@ -42,6 +42,8 @@ #include "error.h" #include <logging/logger.h> +#include <tools/qbsassert.h> +#include <tools/qttools.h> #include <QtCore/qdatastream.h> #include <QtCore/qflags.h> @@ -137,20 +139,34 @@ private: template <typename T> T *idLoad(); template <class T> std::shared_ptr<T> idLoadS(); + template <typename T> T idLoadValue(); + + void doLoadValue(QString &s); + void doLoadValue(QStringList &l); + void doLoadValue(QProcessEnvironment &env); template<typename T> void storeSharedObject(const T *object); void storeVariant(const QVariant &variant); QVariant loadVariant(); - void storeString(const QString &t); - QString loadString(int id); - QString idLoadString(); + template <typename T> void idStoreValue(const T &value); + + void doStoreValue(const QString &s); + void doStoreValue(const QStringList &l); + void doStoreValue(const QProcessEnvironment &env); + + template<typename T> std::vector<T> &idStorage(); + template<typename T> QHash<T, PersistentObjectId> &idMap(); + template<typename T> PersistentObjectId &lastStoredId(); // Recursion termination void store() {} void load() {} + static const PersistentObjectId ValueNotFoundId = -1; + static const PersistentObjectId EmptyValueId = -2; + QDataStream m_stream; HeadData m_headData; std::vector<void *> m_loadedRaw; @@ -161,6 +177,12 @@ private: std::vector<QString> m_stringStorage; QHash<QString, int> m_inverseStringStorage; PersistentObjectId m_lastStoredStringId; + std::vector<QProcessEnvironment> m_envStorage; + QHash<QProcessEnvironment, int> m_inverseEnvStorage; + PersistentObjectId m_lastStoredEnvId; + std::vector<QStringList> m_stringListStorage; + QHash<QStringList, int> m_inverseStringListStorage; + PersistentObjectId m_lastStoredStringListId; Logger &m_logger; template<typename T, typename Enable> @@ -209,6 +231,42 @@ template <typename T> inline T *PersistentPool::idLoad() return t; } +template<> inline std::vector<QString> &PersistentPool::idStorage() { return m_stringStorage; } +template<> inline QHash<QString, PersistentPool::PersistentObjectId> &PersistentPool::idMap() +{ + return m_inverseStringStorage; +} +template<> inline PersistentPool::PersistentObjectId &PersistentPool::lastStoredId<QString>() +{ + return m_lastStoredStringId; +} +template<> inline std::vector<QStringList> &PersistentPool::idStorage() +{ + return m_stringListStorage; +} +template<> inline QHash<QStringList, PersistentPool::PersistentObjectId> &PersistentPool::idMap() +{ + return m_inverseStringListStorage; +} +template<> inline PersistentPool::PersistentObjectId &PersistentPool::lastStoredId<QStringList>() +{ + return m_lastStoredStringListId; +} +template<> inline std::vector<QProcessEnvironment> &PersistentPool::idStorage() +{ + return m_envStorage; +} +template<> inline QHash<QProcessEnvironment, PersistentPool::PersistentObjectId> +&PersistentPool::idMap() +{ + return m_inverseEnvStorage; +} +template<> inline PersistentPool::PersistentObjectId +&PersistentPool::lastStoredId<QProcessEnvironment>() +{ + return m_lastStoredEnvId; +} + template <class T> inline std::shared_ptr<T> PersistentPool::idLoadS() { PersistentObjectId id; @@ -227,6 +285,41 @@ template <class T> inline std::shared_ptr<T> PersistentPool::idLoadS() return t; } +template<typename T> inline T PersistentPool::idLoadValue() +{ + int id; + m_stream >> id; + if (id == EmptyValueId) + return T(); + QBS_CHECK(id >= 0); + if (id >= static_cast<int>(idStorage<T>().size())) { + T value; + doLoadValue(value); + idStorage<T>().resize(id + 1); + idStorage<T>()[id] = value; + return value; + } + return idStorage<T>().at(id); +} + +template<typename T> +void PersistentPool::idStoreValue(const T &value) +{ + if (value.isEmpty()) { + m_stream << EmptyValueId; + return; + } + int id = idMap<T>().value(value, ValueNotFoundId); + if (id < 0) { + id = lastStoredId<T>()++; + idMap<T>().insert(value, id); + m_stream << id; + doStoreValue(value); + } else { + m_stream << id; + } +} + // We need a helper class template, because we require partial specialization for some of // the aggregate types, which is not possible with function templates. // The generic implementation assumes that T is of class type and has load() and store() @@ -320,10 +413,11 @@ template<typename T> struct PPHelper<T *> static void load(T* &value, PersistentPool *pool) { value = pool->idLoad<T>(); } }; -template<> struct PPHelper<QString> +template<typename T> struct PPHelper<T, std::enable_if_t<std::is_same<T, QString>::value + || std::is_same<T, QStringList>::value || std::is_same<T, QProcessEnvironment>::value>> { - static void store(const QString &s, PersistentPool *pool) { pool->storeString(s); } - static void load(QString &s, PersistentPool *pool) { s = pool->idLoadString(); } + static void store(const T &v, PersistentPool *pool) { pool->idStoreValue(v); } + static void load(T &v, PersistentPool *pool) { v = pool->idLoadValue<T>(); } }; template<> struct PPHelper<QVariant> @@ -335,30 +429,9 @@ template<> struct PPHelper<QVariant> template<> struct PPHelper<QRegExp> { static void store(const QRegExp &re, PersistentPool *pool) { pool->store(re.pattern()); } - static void load(QRegExp &re, PersistentPool *pool) { re.setPattern(pool->idLoadString()); } + static void load(QRegExp &re, PersistentPool *pool) { re.setPattern(pool->load<QString>()); } }; -template<> struct PPHelper<QProcessEnvironment> -{ - static void store(const QProcessEnvironment &env, PersistentPool *pool) - { - const QStringList &keys = env.keys(); - pool->store(keys.size()); - for (const QString &key : keys) { - pool->store(key); - pool->store(env.value(key)); - } - } - static void load(QProcessEnvironment &env, PersistentPool *pool) - { - const int count = pool->load<int>(); - for (int i = 0; i < count; ++i) { - const auto &key = pool->load<QString>(); - const auto &value = pool->load<QString>(); - env.insert(key, value); - } - } -}; template<typename T, typename U> struct PPHelper<std::pair<T, U>> { static void store(const std::pair<T, U> &pair, PersistentPool *pool) @@ -387,7 +460,6 @@ template<typename T> struct PPHelper<QFlags<T>> }; template<typename T> struct IsSimpleContainer : std::false_type { }; -template<> struct IsSimpleContainer<QStringList> : std::true_type { }; template<typename T> struct IsSimpleContainer<QList<T>> : std::true_type { }; template<typename T> struct IsSimpleContainer<std::vector<T>> : std::true_type { }; diff --git a/src/lib/corelib/tools/preferences.cpp b/src/lib/corelib/tools/preferences.cpp index 66803a0f5..12af4e9c7 100644 --- a/src/lib/corelib/tools/preferences.cpp +++ b/src/lib/corelib/tools/preferences.cpp @@ -124,6 +124,32 @@ QStringList Preferences::pluginPaths(const QString &baseDir) const return pathList(QLatin1String("pluginsPath"), baseDir + QLatin1String("/qbs/plugins")); } +/*! + * \brief Returns the per-pool job limits. + */ +JobLimits Preferences::jobLimits() const +{ + const QString prefix = QLatin1String("preferences.jobLimit"); + JobLimits limits; + for (const QString &key : m_settings->allKeysWithPrefix(prefix, Settings::allScopes())) { + limits.setJobLimit(key, m_settings->value(prefix + QLatin1Char('.') + key, + Settings::allScopes()).toInt()); + } + const QString fullPrefix = prefix + QLatin1Char('.'); + if (!m_profile.isEmpty()) { + Profile p(m_profile, m_settings, m_profileContents); + for (const QString &key : p.allKeys(Profile::KeySelectionRecursive)) { + if (!key.startsWith(fullPrefix)) + continue; + const QString jobPool = key.mid(fullPrefix.size()); + const int limit = p.value(key).toInt(); + if (limit >= 0) + limits.setJobLimit(jobPool, limit); + } + } + return limits; +} + QVariant Preferences::getPreference(const QString &key, const QVariant &defaultValue) const { static const QString keyPrefix = QLatin1String("preferences"); diff --git a/src/lib/corelib/tools/preferences.h b/src/lib/corelib/tools/preferences.h index 07f0edcd7..661b39d7f 100644 --- a/src/lib/corelib/tools/preferences.h +++ b/src/lib/corelib/tools/preferences.h @@ -42,6 +42,7 @@ #include "qbs_export.h" #include "commandechomode.h" +#include "joblimits.h" #include "settings.h" #include <QtCore/qstringlist.h> @@ -63,6 +64,7 @@ public: CommandEchoMode defaultEchoMode() const; QStringList searchPaths(const QString &baseDir = QString()) const; QStringList pluginPaths(const QString &baseDir = QString()) const; + JobLimits jobLimits() const; private: QVariant getPreference(const QString &key, const QVariant &defaultValue = QVariant()) const; diff --git a/src/lib/corelib/tools/profile.h b/src/lib/corelib/tools/profile.h index a4c6a91f1..2ccc99def 100644 --- a/src/lib/corelib/tools/profile.h +++ b/src/lib/corelib/tools/profile.h @@ -59,7 +59,7 @@ public: bool exists() const; QVariant value(const QString &key, const QVariant &defaultValue = QVariant(), - ErrorInfo *error = 0) const; + ErrorInfo *error = nullptr) const; void setValue(const QString &key, const QVariant &value); void remove(const QString &key); @@ -72,7 +72,7 @@ public: void removeProfile(); enum KeySelection { KeySelectionRecursive, KeySelectionNonRecursive }; - QStringList allKeys(KeySelection selection, ErrorInfo *error = 0) const; + QStringList allKeys(KeySelection selection, ErrorInfo *error = nullptr) const; static QString cleanName(const QString &name); diff --git a/src/lib/corelib/tools/qbspluginmanager.cpp b/src/lib/corelib/tools/qbspluginmanager.cpp index a8d22f458..d0be73891 100644 --- a/src/lib/corelib/tools/qbspluginmanager.cpp +++ b/src/lib/corelib/tools/qbspluginmanager.cpp @@ -80,8 +80,7 @@ QbsPluginManager::~QbsPluginManager() unloadStaticPlugins(); for (QLibrary * const lib : qAsConst(d->libs)) { - QbsPluginUnloadFunction unload = reinterpret_cast<QbsPluginUnloadFunction>( - lib->resolve("QbsPluginUnload")); + auto unload = reinterpret_cast<QbsPluginUnloadFunction>(lib->resolve("QbsPluginUnload")); if (unload) unload(); lib->unload(); @@ -146,8 +145,7 @@ void QbsPluginManager::loadPlugins(const std::vector<std::string> &pluginPaths, continue; } - QbsPluginLoadFunction load = reinterpret_cast<QbsPluginLoadFunction>( - lib->resolve("QbsPluginLoad")); + auto load = reinterpret_cast<QbsPluginLoadFunction>(lib->resolve("QbsPluginLoad")); if (load) { load(); qCDebug(lcPluginManager) << "plugin" << QDir::toNativeSeparators(fileName) diff --git a/src/lib/corelib/tools/qttools.cpp b/src/lib/corelib/tools/qttools.cpp index 4e20a7f45..ffd336d56 100644 --- a/src/lib/corelib/tools/qttools.cpp +++ b/src/lib/corelib/tools/qttools.cpp @@ -39,7 +39,10 @@ #include "qttools.h" +#include <QtCore/qprocess.h> + QT_BEGIN_NAMESPACE + uint qHash(const QStringList &list) { uint s = 0; @@ -47,4 +50,10 @@ uint qHash(const QStringList &list) s ^= qHash(n) + 0x9e3779b9 + (s << 6) + (s >> 2); return s; } + +uint qHash(const QProcessEnvironment &env) +{ + return qHash(env.toStringList()); +} + QT_END_NAMESPACE diff --git a/src/lib/corelib/tools/qttools.h b/src/lib/corelib/tools/qttools.h index 50b2829d1..2252c12d3 100644 --- a/src/lib/corelib/tools/qttools.h +++ b/src/lib/corelib/tools/qttools.h @@ -45,6 +45,10 @@ #include <functional> +QT_BEGIN_NAMESPACE +class QProcessEnvironment; +QT_END_NAMESPACE + namespace std { template<> struct hash<QString> { std::size_t operator()(const QString &s) const { return qHash(s); } @@ -53,6 +57,7 @@ template<> struct hash<QString> { QT_BEGIN_NAMESPACE uint qHash(const QStringList &list); +uint qHash(const QProcessEnvironment &env); QT_END_NAMESPACE #endif // QBSQTTOOLS_H diff --git a/src/lib/corelib/tools/settingsmodel.h b/src/lib/corelib/tools/settingsmodel.h index 6f9631585..27936ba52 100644 --- a/src/lib/corelib/tools/settingsmodel.h +++ b/src/lib/corelib/tools/settingsmodel.h @@ -68,14 +68,16 @@ public: void addNewKey(const QModelIndex &parent); void removeKey(const QModelIndex &index); - Qt::ItemFlags flags(const QModelIndex &index) const; - QVariant headerData(int section, Qt::Orientation orientation, int role = Qt::DisplayRole) const; - int columnCount(const QModelIndex &parent = QModelIndex()) const; - int rowCount(const QModelIndex &parent = QModelIndex()) const; - QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const; - bool setData(const QModelIndex &index, const QVariant &value, int role = Qt::EditRole); - QModelIndex index(int row, int column, const QModelIndex &parent = QModelIndex()) const; - QModelIndex parent(const QModelIndex &child) const; + Qt::ItemFlags flags(const QModelIndex &index) const override; + QVariant headerData(int section, Qt::Orientation orientation, + int role = Qt::DisplayRole) const override; + int columnCount(const QModelIndex &parent = QModelIndex()) const override; + int rowCount(const QModelIndex &parent = QModelIndex()) const override; + QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; + bool setData(const QModelIndex &index, const QVariant &value, int role = Qt::EditRole) override; + QModelIndex index(int row, int column, + const QModelIndex &parent = QModelIndex()) const override; + QModelIndex parent(const QModelIndex &child) const override; private: class SettingsModelPrivate; diff --git a/src/lib/corelib/tools/setupprojectparameters.h b/src/lib/corelib/tools/setupprojectparameters.h index fd73d0748..fe7e3d487 100644 --- a/src/lib/corelib/tools/setupprojectparameters.h +++ b/src/lib/corelib/tools/setupprojectparameters.h @@ -97,7 +97,9 @@ public: void setOverriddenValues(const QVariantMap &values); QVariantMap overriddenValuesTree() const; - static QVariantMap expandedBuildConfiguration(const Profile &profile, const QString &configurationName, ErrorInfo *errorInfo = 0); + static QVariantMap expandedBuildConfiguration(const Profile &profile, + const QString &configurationName, + ErrorInfo *errorInfo = nullptr); ErrorInfo expandBuildConfiguration(); QVariantMap buildConfiguration() const; QVariantMap buildConfigurationTree() const; diff --git a/src/lib/corelib/tools/stringconstants.h b/src/lib/corelib/tools/stringconstants.h index f2666e070..6fcf3002b 100644 --- a/src/lib/corelib/tools/stringconstants.h +++ b/src/lib/corelib/tools/stringconstants.h @@ -104,6 +104,8 @@ public: QBS_STRING_CONSTANT(installPrefixProperty, "installPrefix") QBS_STRING_CONSTANT(installDirProperty, "installDir") QBS_STRING_CONSTANT(installSourceBaseProperty, "installSourceBase") + QBS_STRING_CONSTANT(jobCountProperty, "jobCount") + QBS_STRING_CONSTANT(jobPoolProperty, "jobPool") QBS_STRING_CONSTANT(lengthProperty, "length") QBS_STRING_CONSTANT(limitToSubProjectProperty, "limitToSubProject") QBS_STRING_CONSTANT(minimumQbsVersionProperty, "minimumQbsVersion") diff --git a/src/lib/corelib/tools/tools.pri b/src/lib/corelib/tools/tools.pri index bb3a55f12..f9c6be9a5 100644 --- a/src/lib/corelib/tools/tools.pri +++ b/src/lib/corelib/tools/tools.pri @@ -21,6 +21,7 @@ HEADERS += \ $$PWD/generateoptions.h \ $$PWD/id.h \ $$PWD/iosutils.h \ + $$PWD/joblimits.h \ $$PWD/jsliterals.h \ $$PWD/launcherinterface.h \ $$PWD/launcherpackets.h \ @@ -75,6 +76,7 @@ SOURCES += \ $$PWD/filetime.cpp \ $$PWD/generateoptions.cpp \ $$PWD/id.cpp \ + $$PWD/joblimits.cpp \ $$PWD/jsliterals.cpp \ $$PWD/launcherinterface.cpp \ $$PWD/launcherpackets.cpp \ @@ -124,6 +126,7 @@ osx { $$PWD/error.h \ $$PWD/generateoptions.h \ $$PWD/installoptions.h \ + $$PWD/joblimits.h \ $$PWD/preferences.h \ $$PWD/processresult.h \ $$PWD/profile.h \ diff --git a/src/lib/qtprofilesetup/qtmoduleinfo.cpp b/src/lib/qtprofilesetup/qtmoduleinfo.cpp index 56827a458..c225bde67 100644 --- a/src/lib/qtprofilesetup/qtmoduleinfo.cpp +++ b/src/lib/qtprofilesetup/qtmoduleinfo.cpp @@ -768,7 +768,9 @@ QList<QtModuleInfo> allQt5Modules(const Profile &profile, const QtEnvironment &q } else if (key.endsWith(".TYPE")) { moduleInfo.pluginData.type = QString::fromLatin1(value); } else if (key.endsWith(".EXTENDS")) { - moduleInfo.pluginData.extends = QString::fromLatin1(value); + moduleInfo.pluginData.extends = makeList(value); + if (moduleInfo.pluginData.extends.removeOne(QLatin1String("-"))) + moduleInfo.pluginData.autoLoad = false; } else if (key.endsWith(".CLASS_NAME")) { moduleInfo.pluginData.className = QString::fromLatin1(value); } diff --git a/src/lib/qtprofilesetup/qtmoduleinfo.h b/src/lib/qtprofilesetup/qtmoduleinfo.h index 318b5608f..078e08a19 100644 --- a/src/lib/qtprofilesetup/qtmoduleinfo.h +++ b/src/lib/qtprofilesetup/qtmoduleinfo.h @@ -95,8 +95,9 @@ public: struct PluginData { QString type; - QString extends; + QStringList extends; QString className; + bool autoLoad = true; } pluginData; private: diff --git a/src/lib/qtprofilesetup/qtprofilesetup.cpp b/src/lib/qtprofilesetup/qtprofilesetup.cpp index 1ee4aa3c6..f97b12f2c 100644 --- a/src/lib/qtprofilesetup/qtprofilesetup.cpp +++ b/src/lib/qtprofilesetup/qtprofilesetup.cpp @@ -418,10 +418,12 @@ static std::pair<int, int> findVariable(const QByteArray &content, int start) static QByteArray libraryFileTag(const QtEnvironment &env, const QtModuleInfo &module) { QByteArray result; - if (module.isStaticLibrary) + if (module.isStaticLibrary) { result = "staticlibrary"; - else - result = isMsvcQt(env) ? "dynamiclibrary_import" : "dynamiclibrary"; + } else { + result = isMsvcQt(env) || env.mkspecName.startsWith(QLatin1String("win32-g++")) + ? "dynamiclibrary_import" : "dynamiclibrary"; + } return result; } @@ -464,6 +466,7 @@ static void replaceSpecialValues(QByteArray *content, const Profile &profile, dict.insert("libFilePathRelease", utf8JSLiteral(module.libFilePathRelease)); dict.insert("libNameForLinkerDebug", utf8JSLiteral(module.libNameForLinker(qtEnvironment, true))); + dict.insert("pluginTypes", utf8JSLiteral(module.supportedPluginTypes)); dict.insert("libNameForLinkerRelease", utf8JSLiteral(module.libNameForLinker(qtEnvironment, false))); dict.insert("entryPointLibsDebug", utf8JSLiteral(qtEnvironment.entryPointLibsDebug)); @@ -515,15 +518,21 @@ static void replaceSpecialValues(QByteArray *content, const Profile &profile, additionalContent += "\n "; additionalContent += "isStaticLibrary: true"; } - if (module.isPlugin) + if (module.isPlugin) { dict.insert("className", utf8JSLiteral(module.pluginData.className)); + dict.insert("extends", utf8JSLiteral(module.pluginData.extends)); + } if (module.hasLibrary && !module.isFramework(qtEnvironment)) { if (!additionalContent.isEmpty()) additionalContent += "\n"; const QByteArray indent(4, ' '); - additionalContent += "Group {\n" - + indent + indent + "files: [Qt[\"" + module.qbsName.toUtf8() + "\"]" - + ".libFilePath]\n" + additionalContent += "Group {\n"; + if (module.isPlugin) { + additionalContent += indent + indent + + "condition: Qt[\"" + module.qbsName.toUtf8() + "\"].enableLinking\n"; + } + additionalContent += indent + indent + "files: [Qt[\"" + module.qbsName.toUtf8() + "\"]" + + ".libFilePath]\n" + indent + indent + "filesAreTargets: true\n" + indent + indent + "fileTags: [\"" + libraryFileTag(qtEnvironment, module) + "\"]\n" @@ -542,7 +551,8 @@ static void replaceSpecialValues(QByteArray *content, const Profile &profile, static void copyTemplateFile(const QString &fileName, const QString &targetDirectory, const Profile &profile, const QtEnvironment &qtEnv, QStringList *allFiles, - const QtModuleInfo *module = 0) + const QtModuleInfo *module = nullptr, const QVariantMap &pluginMap = QVariantMap(), + const QStringList &nonEssentialPlugins = QStringList()) { if (!QDir::root().mkpath(targetDirectory)) { throw ErrorInfo(Internal::Tr::tr("Setting up Qt profile '%1' failed: " @@ -555,8 +565,12 @@ static void copyTemplateFile(const QString &fileName, const QString &targetDirec "Cannot open '%1' (%2).").arg(sourceFile.fileName(), sourceFile.errorString())); } QByteArray newContent = sourceFile.readAll(); - if (module) + if (module) { replaceSpecialValues(&newContent, profile, *module, qtEnv); + } else { + newContent.replace("@allPluginsByType@", '(' + utf8JSLiteral(pluginMap) + ')'); + newContent.replace("@nonEssentialPlugins@", utf8JSLiteral(nonEssentialPlugins)); + } sourceFile.close(); const QString targetPath = targetDirectory + QLatin1Char('/') + fileName; allFiles->push_back(QFileInfo(targetPath).absoluteFilePath()); @@ -580,6 +594,16 @@ static void createModules(Profile &profile, Settings *settings, const QList<QtModuleInfo> modules = qtEnvironment.qtMajorVersion < 5 ? allQt4Modules(qtEnvironment) : allQt5Modules(profile, qtEnvironment); + QVariantMap pluginsByType; + QStringList nonEssentialPlugins; + for (const QtModuleInfo &m : modules) { + if (m.isPlugin) { + QVariant &v = pluginsByType[m.pluginData.type]; + v = v.toStringList() << m.name; + if (!m.pluginData.autoLoad) + nonEssentialPlugins << m.name; + } + } const QString profileBaseDir = QString::fromLatin1("%1/profiles/%2") .arg(QFileInfo(settings->fileName()).dir().absolutePath(), profile.name()); @@ -589,6 +613,9 @@ static void createModules(Profile &profile, Settings *settings, &allFiles); copyTemplateFile(QLatin1String("QtPlugin.qbs"), qbsQtModuleBaseDir, profile, qtEnvironment, &allFiles); + copyTemplateFile(QLatin1String("plugin_support.qbs"), + qbsQtModuleBaseDir + QLatin1String("/plugin_support"), profile, qtEnvironment, + &allFiles, nullptr, pluginsByType, nonEssentialPlugins); for (const QtModuleInfo &module : modules) { const QString qbsQtModuleDir = qbsQtModuleBaseDir + QLatin1Char('/') + module.qbsName; QString moduleTemplateFileName; @@ -629,6 +656,13 @@ static void createModules(Profile &profile, Settings *settings, copyTemplateFile(moduleTemplateFileName, qbsQtModuleDir, profile, qtEnvironment, &allFiles, &module); } + + // Note that it's not strictly necessary to copy this one, as it has no variable content. + // But we do it anyway for consistency (and it has no impact on the project files this way). + copyTemplateFile(QLatin1String("android_support.qbs"), + qbsQtModuleBaseDir + QLatin1String("/android_support"), profile, qtEnvironment, + &allFiles); + QDirIterator dit(qbsQtModuleBaseDir, QDirIterator::Subdirectories); while (dit.hasNext()) { dit.next(); diff --git a/src/lib/qtprofilesetup/templates.qrc b/src/lib/qtprofilesetup/templates.qrc index 3c6922862..35bddf8d7 100644 --- a/src/lib/qtprofilesetup/templates.qrc +++ b/src/lib/qtprofilesetup/templates.qrc @@ -16,5 +16,7 @@ <file>templates/qmlcache.qbs</file> <file>templates/quick.js</file> <file>templates/quick.qbs</file> + <file>templates/plugin_support.qbs</file> + <file>templates/android_support.qbs</file> </qresource> </RCC> diff --git a/src/lib/qtprofilesetup/templates/QtModule.qbs b/src/lib/qtprofilesetup/templates/QtModule.qbs index fcc8797cb..04820ee29 100644 --- a/src/lib/qtprofilesetup/templates/QtModule.qbs +++ b/src/lib/qtprofilesetup/templates/QtModule.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import qbs.FileInfo Module { @@ -14,6 +13,25 @@ Module { Depends { name: "cpp" } Depends { name: "Qt.core" } + Depends { name: "Qt.plugin_support" } + property stringList pluginTypes + Qt.plugin_support.pluginTypes: pluginTypes + Depends { + condition: Qt.core.staticBuild && !isPlugin + name: "Qt"; + submodules: { + // We have to pull in all plugins here, because dependency resolving happens + // before module merging, and we don't know yet if someone set + // Qt.pluginSupport.pluginsByType in the product. + // The real filtering is done later by the plugin module files themselves. + var list = []; + var allPlugins = Qt.plugin_support.allPluginsByType; + for (var i = 0; i < (pluginTypes || []).length; ++i) + Array.prototype.push.apply(list, allPlugins[pluginTypes[i]]) + return list; + } + } + property string qtModuleName property path binPath: Qt.core.binPath property path incPath: Qt.core.incPath @@ -54,9 +72,10 @@ Module { ? frameworkPathsDebug: frameworkPathsRelease cpp.linkerFlags: Qt.core.qtBuildVariant === "debug" ? linkerFlagsDebug : linkerFlagsRelease + property bool enableLinking: qtModuleName != undefined && hasLibrary Properties { - condition: qtModuleName != undefined && hasLibrary + condition: enableLinking cpp.staticLibraries: staticLibs cpp.dynamicLibraries: dynamicLibs cpp.frameworks: mFrameworks.concat(!isStaticLibrary && Qt.core.frameworkBuild diff --git a/src/lib/qtprofilesetup/templates/QtPlugin.qbs b/src/lib/qtprofilesetup/templates/QtPlugin.qbs index dceb8ef6e..23a6795f3 100644 --- a/src/lib/qtprofilesetup/templates/QtPlugin.qbs +++ b/src/lib/qtprofilesetup/templates/QtPlugin.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import qbs.FileInfo import qbs.TextFile @@ -6,9 +5,27 @@ QtModule { isPlugin: true property string className + property stringList extendsModules + + enableLinking: { + if (!base) + return false; + if (!isStaticLibrary) + return false; + if (!(Qt.plugin_support.enabledPlugins || []).contains(qtModuleName)) + return false; + if (!extendsModules || extendsModules.length === 0) + return true; + for (var i = 0; i < extendsModules.length; ++i) { + var moduleName = extendsModules[i]; + if (product.Qt[moduleName] && product.Qt[moduleName].present) + return true; + } + return false; + } Rule { - condition: isStaticLibrary + condition: enableLinking multiplex: true Artifact { filePath: product.targetName + "_qt_plugin_import_" diff --git a/src/lib/qtprofilesetup/templates/android_support.qbs b/src/lib/qtprofilesetup/templates/android_support.qbs new file mode 100644 index 000000000..79276a494 --- /dev/null +++ b/src/lib/qtprofilesetup/templates/android_support.qbs @@ -0,0 +1,291 @@ +import qbs.File +import qbs.FileInfo +import qbs.ModUtils +import qbs.TextFile +import qbs.Utilities + +Module { + property bool useMinistro: false + property string qmlRootDir: product.sourceDirectory + property stringList extraPrefixDirs + property stringList deploymentDependencies // qmake: ANDROID_DEPLOYMENT_DEPENDENCIES + property stringList extraPlugins // qmake: ANDROID_EXTRA_PLUGINS + property bool verboseAndroidDeployQt: false + + property string _androidDeployQtFilePath: FileInfo.joinPaths(_qtInstallDir, "bin", + "androiddeployqt") + property string _qtInstallDir + property bool _enableSdkSupport: product.type && product.type.contains("android.apk") + && !consoleApplication + property bool _enableNdkSupport: !product.aggregate || product.multiplexConfigurationId + property string _templatesBaseDir: FileInfo.joinPaths(_qtInstallDir, "src", "android") + property string _deployQtOutDir: FileInfo.joinPaths(product.buildDirectory, "deployqt_out") + + Depends { name: "Android.sdk"; condition: _enableSdkSupport } + Depends { name: "Android.ndk"; condition: _enableNdkSupport } + Depends { name: "java"; condition: _enableSdkSupport } + + Properties { + condition: _enableNdkSupport && qbs.toolchain.contains("clang") + Android.ndk.appStl: "c++_shared" + } + Properties { + condition: _enableNdkSupport && !qbs.toolchain.contains("clang") + Android.ndk.appStl: "gnustl_shared" + } + Properties { + condition: _enableSdkSupport + Android.sdk.customManifestProcessing: true + java._tagJniHeaders: false // prevent rule cycle + } + + Rule { + condition: _enableSdkSupport + multiplex: true + property stringList inputTags: "android.nativelibrary" + inputsFromDependencies: inputTags + inputs: product.aggregate ? [] : inputTags + Artifact { + filePath: "androiddeployqt.json" + fileTags: "qt_androiddeployqt_input" + } + prepare: { + var cmd = new JavaScriptCommand(); + cmd.description = "creating " + output.fileName; + cmd.sourceCode = function() { + var theBinary; + var nativeLibs = inputs["android.nativelibrary"]; + if (nativeLibs.length === 1) { + theBinary = nativeLibs[0]; + } else { + for (i = 0; i < nativeLibs.length; ++i) { + var candidate = nativeLibs[i]; + if (!candidate.fileName.contains(candidate.product.targetName)) + continue; + if (!theBinary) { + theBinary = candidate; + continue; + } + if (theBinary.product.name === product.name + && candidate.product.name !== product.name) { + continue; // We already have a better match. + } + if (candidate.product.name === product.name + && theBinary.product.name !== product.name) { + theBinary = candidate; // The new candidate is a better match. + continue; + } + throw "Qt applications for Android support only one native binary " + + "per package.\n" + + "In particular, you cannot build a Qt app for more than " + + "one architecture at the same time."; + } + } + var f = new TextFile(output.filePath, TextFile.WriteOnly); + f.writeLine("{"); + f.writeLine('"description": "This file was generated by qbs to be read by ' + + 'androiddeployqt and should not be modified by hand.",'); + f.writeLine('"qt": "' + product.Qt.android_support._qtInstallDir + '",'); + f.writeLine('"sdk": "' + product.Android.sdk.sdkDir + '",'); + f.writeLine('"sdkBuildToolsRevision": "' + product.Android.sdk.buildToolsVersion + + '",'); + f.writeLine('"ndk": "' + product.Android.sdk.ndkDir + '",'); + var toolPrefix = theBinary.cpp.toolchainTriple; + var toolchainPrefix = toolPrefix.startsWith("i686-") ? "x86" : toolPrefix; + f.writeLine('"toolchain-prefix": "' + toolchainPrefix + '",'); + f.writeLine('"tool-prefix": "' + toolPrefix + '",'); + f.writeLine('"toolchain-version": "' + theBinary.Android.ndk.toolchainVersion + + '",'); + f.writeLine('"ndk-host": "' + theBinary.Android.ndk.hostArch + '",'); + f.writeLine('"target-architecture": "' + theBinary.Android.ndk.abi + '",'); + f.writeLine('"qml-root-path": "' + product.Qt.android_support.qmlRootDir + '",'); + var deploymentDeps = product.Qt.android_support.deploymentDependencies; + if (deploymentDeps && deploymentDeps.length > 0) + f.writeLine('"deployment-dependencies": "' + deploymentDeps.join() + '",'); + var extraPlugins = product.Qt.android_support.extraPlugins; + if (extraPlugins && extraPlugins.length > 0) + f.writeLine('"android-extra-plugins": "' + extraPlugins.join() + '",'); + var prefixDirs = product.Qt.android_support.extraPrefixDirs; + if (prefixDirs && prefixDirs.length > 0) + f.writeLine('"extraPrefixDirs": ' + JSON.stringify(prefixDirs) + ','); + if (Array.isArray(product.qmlImportPaths) && product.qmlImportPaths.length > 0) + f.writeLine('"qml-import-paths": "' + product.qmlImportPaths.join(',') + '",'); + f.writeLine('"application-binary": "' + theBinary.filePath + '"'); + f.writeLine("}"); + f.close(); + }; + return cmd; + } + } + + // We use the manifest template from the Qt installation if and only if the project + // does not provide a manifest file. + Rule { + condition: _enableSdkSupport + multiplex: true + requiresInputs: false + inputs: "android.manifest" + excludedInputs: "qt.android_manifest" + outputFileTags: ["android.manifest", "qt.android_manifest"] + outputArtifacts: { + if (inputs["android.manifest"]) + return []; + return [{ + filePath: "qt_manifest/AndroidManifest.xml", + fileTags: ["android.manifest", "qt.android_manifest"] + }]; + } + prepare: { + var cmd = new JavaScriptCommand(); + cmd.description = "copying Qt Android manifest template"; + cmd.sourceCode = function() { + File.copy(FileInfo.joinPaths(product.Qt.android_support._templatesBaseDir, + "templates", "AndroidManifest.xml"), output.filePath); + }; + return cmd; + } + } + + Rule { + condition: _enableSdkSupport + multiplex: true + inputs: ["qt_androiddeployqt_input", "android.manifest_processed"] + outputFileTags: [ + "android.manifest_final", "android.resources", "android.assets", "bundled_jar", + "android.deployqt_list", + ] + outputArtifacts: { + var artifacts = [ + { + filePath: "AndroidManifest.xml", + fileTags: "android.manifest_final" + }, + { + filePath: product.Qt.android_support._deployQtOutDir + "/res/values/libs.xml", + fileTags: "android.resources" + }, + { + filePath: product.Qt.android_support._deployQtOutDir + + "/res/values/strings.xml", + fileTags: "android.resources" + }, + { + filePath: product.Qt.android_support._deployQtOutDir + "/assets/.dummy", + fileTags: "android.assets" + }, + { + filePath: "deployqt.list", + fileTags: "android.deployqt_list" + }, + + ]; + if (!product.Qt.android_support.useMinistro) { + artifacts.push({ + filePath: FileInfo.joinPaths(product.java.classFilesDir, "QtAndroid.jar"), + fileTags: ["bundled_jar"] + }); + } + return artifacts; + } + prepare: { + var copyCmd = new JavaScriptCommand(); + copyCmd.description = "copying Qt resource templates"; + copyCmd.sourceCode = function() { + File.copy(inputs["android.manifest_processed"][0].filePath, + product.Qt.android_support._deployQtOutDir + "/AndroidManifest.xml"); + File.copy(product.Qt.android_support._templatesBaseDir + "/java/res", + product.Qt.android_support._deployQtOutDir + "/res"); + File.copy(product.Qt.android_support._templatesBaseDir + + "/templates/res/values/libs.xml", + product.Qt.android_support._deployQtOutDir + "/res/values/libs.xml"); + try { + File.remove(FileInfo.path(outputs["android.assets"][0].filePath)); + } catch (e) { + } + }; + var androidDeployQtArgs = [ + "--output", product.Qt.android_support._deployQtOutDir, + "--input", inputs["qt_androiddeployqt_input"][0].filePath, "--aux-mode", + "--deployment", product.Qt.android_support.useMinistro ? "ministro" : "bundled", + "--android-platform", product.Android.sdk.platform, + ]; + if (product.Qt.android_support.verboseAndroidDeployQt) + args.push("--verbose"); + var androidDeployQtCmd = new Command( + product.Qt.android_support._androidDeployQtFilePath, androidDeployQtArgs); + androidDeployQtCmd.description = "running androiddeployqt"; + + // We do not want androiddeployqt to write directly into our APK base dir, so + // we ran it on an isolated directory and now we move stuff over. + // We remember the files for which we do that, so if the next invocation + // of androiddeployqt creates fewer files, the other ones are removed from + // the APK base dir. + var moveCmd = new JavaScriptCommand(); + moveCmd.description = "processing androiddeployqt outout"; + moveCmd.sourceCode = function() { + File.move(product.Qt.android_support._deployQtOutDir + "/AndroidManifest.xml", + outputs["android.manifest_final"][0].filePath); + var libsDir = product.Qt.android_support._deployQtOutDir + "/libs"; + var libDir = product.Android.sdk.apkContentsDir + "/lib"; + var listFilePath = outputs["android.deployqt_list"][0].filePath; + var oldLibs = []; + try { + var listFile = new TextFile(listFilePath, TextFile.ReadOnly); + var listFileLine = listFile.readLine(); + while (listFileLine) { + oldLibs.push(listFileLine); + listFileLine = listFile.readLine(); + } + listFile.close(); + } catch (e) { + } + listFile = new TextFile(listFilePath, TextFile.WriteOnly); + var newLibs = []; + var moveLibFiles = function(prefix) { + var fullSrcPrefix = FileInfo.joinPaths(libsDir, prefix); + var files = File.directoryEntries(fullSrcPrefix, File.Files); + for (var i = 0; i < files.length; ++i) { + var file = files[i]; + var srcFilePath = FileInfo.joinPaths(fullSrcPrefix, file); + var targetFilePath; + if (file.endsWith(".jar")) + targetFilePath = FileInfo.joinPaths(product.java.classFilesDir, file); + else + targetFilePath = FileInfo.joinPaths(libDir, prefix, file); + File.move(srcFilePath, targetFilePath); + listFile.writeLine(targetFilePath); + newLibs.push(targetFilePath); + } + var dirs = File.directoryEntries(fullSrcPrefix, + File.Dirs | File.NoDotAndDotDot); + for (i = 0; i < dirs.length; ++i) + moveLibFiles(FileInfo.joinPaths(prefix, dirs[i])); + }; + moveLibFiles(""); + listFile.close(); + for (i = 0; i < oldLibs.length; ++i) { + if (!newLibs.contains(oldLibs[i])) + File.remove(oldLibs[i]); + } + }; + return [copyCmd, androidDeployQtCmd, moveCmd]; + } + } + + Group { + condition: Qt.android_support._enableSdkSupport + name: "helper sources from qt" + prefix: Qt.android_support._templatesBaseDir + "/java/" + Android.sdk.aidlSearchPaths: prefix + "src" + files: [ + "**/*.java", + "**/*.aidl", + ] + } + + validate: { + if (Utilities.versionCompare(version, "5.12") < 0) + throw ModUtils.ModuleError("Cannot use Qt " + version + " with Android. " + + "Version 5.12 or later is required."); + } +} diff --git a/src/lib/qtprofilesetup/templates/core.qbs b/src/lib/qtprofilesetup/templates/core.qbs index 928c69cda..7abd7aa02 100644 --- a/src/lib/qtprofilesetup/templates/core.qbs +++ b/src/lib/qtprofilesetup/templates/core.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import qbs.FileInfo import qbs.ModUtils import qbs.TextFile @@ -19,6 +18,13 @@ Module { Depends { name: "cpp" } + Depends { name: "Qt.android_support"; condition: qbs.targetOS.contains("android") } + Properties { + condition: qbs.targetOS.contains("android") + Qt.android_support._qtInstallDir: FileInfo.path(binPath) + Qt.android_support.version: version + } + version: @version@ property stringList architectures: @archs@ property string targetPlatform: @targetPlatform@ @@ -172,7 +178,6 @@ Module { cpp.cxxFlags: { var flags = []; if (qbs.toolchain.contains('msvc')) { - flags.push('/Zm200'); if (versionMajor < 5) flags.push('/Zc:wchar_t-'); } @@ -266,49 +271,27 @@ Module { } property bool combineMocOutput: cpp.combineCxxSources + property bool enableBigResources: false Rule { - name: "QtCoreMocRule" + name: "QtCoreMocRuleCpp" property string cppInput: cpp.combineCxxSources ? "cpp.combine" : "cpp" property string objcppInput: cpp.combineObjcxxSources ? "objcpp.combine" : "objcpp" - inputs: [objcppInput, cppInput, "hpp"] - auxiliaryInputs: { - var auxInputs = ["qt_plugin_metadata"]; - if (cpp.combineCxxSources) - auxInputs.push("cpp"); - if (cpp.combineObjcxxSources) - auxInputs.push("objcpp"); - return auxInputs; - } - excludedInputs: ["unmocable"] + inputs: [objcppInput, cppInput] + auxiliaryInputs: "qt_plugin_metadata" + excludedInputs: "unmocable" + outputFileTags: ["hpp", "unmocable"] + outputArtifacts: Moc.outputArtifacts.apply(Moc, arguments) + prepare: Moc.commands.apply(Moc, arguments) + } + Rule { + name: "QtCoreMocRuleHpp" + inputs: "hpp" + auxiliaryInputs: ["qt_plugin_metadata", "cpp", "objcpp"]; + excludedInputs: "unmocable" outputFileTags: ["hpp", "cpp", "moc_cpp", "unmocable"] - outputArtifacts: { - if (input.fileTags.contains("unmocable")) - return []; - var mocinfo = QtMocScanner.apply(input); - if (!mocinfo.hasQObjectMacro) - return []; - var artifact = { fileTags: ["unmocable"] }; - if (input.fileTags.contains("hpp")) { - artifact.filePath = input.Qt.core.generatedHeadersDir - + "/moc_" + input.completeBaseName + ".cpp"; - } else { - artifact.filePath = input.Qt.core.generatedHeadersDir - + '/' + input.completeBaseName + ".moc"; - } - var amalgamate = input.Qt.core.combineMocOutput; - artifact.fileTags.push(mocinfo.mustCompile ? (amalgamate ? "moc_cpp" : "cpp") : "hpp"); - if (mocinfo.hasPluginMetaDataMacro) - artifact.explicitlyDependsOn = ["qt_plugin_metadata"]; - return [artifact]; - } - prepare: { - var cmd = new Command(Moc.fullPath(product), - Moc.args(product, input, output.filePath)); - cmd.description = 'moc ' + input.fileName; - cmd.highlight = 'codegen'; - return cmd; - } + outputArtifacts: Moc.outputArtifacts.apply(Moc, arguments) + prepare: Moc.commands.apply(Moc, arguments) } Rule { @@ -387,17 +370,60 @@ Module { Rule { inputs: ["qrc"] + outputFileTags: ["cpp", "cpp_intermediate_object"] + outputArtifacts: { + var artifact = { + filePath: "qrc_" + input.completeBaseName + ".cpp", + fileTags: ["cpp"] + }; + if (input.Qt.core.enableBigResources) + artifact.fileTags.push("cpp_intermediate_object"); + return [artifact]; + } + prepare: { + var args = [input.filePath, + "-name", FileInfo.completeBaseName(input.filePath), + "-o", output.filePath]; + if (input.Qt.core.enableBigResources) + args.push("-pass", "1"); + var cmd = new Command(product.Qt.core.binPath + '/rcc', args); + cmd.description = "rcc " + + (input.Qt.core.enableBigResources ? "(pass 1) " : "") + + input.fileName; + cmd.highlight = 'codegen'; + return cmd; + } + } + Rule { + inputs: ["intermediate_obj"] Artifact { - filePath: "qrc_" + input.completeBaseName + ".cpp" - fileTags: ["cpp"] + filePath: input.completeBaseName + ".2.o" + fileTags: ["obj"] } prepare: { + function findChild(artifact, predicate) { + var children = artifact.children; + var len = children.length; + for (var i = 0; i < len; ++i) { + var child = children[i]; + if (predicate(child)) + return child; + child = findChild(child, predicate); + if (child) + return child; + } + return undefined; + } + var qrcArtifact = findChild(input, function(c) { return c.fileTags.contains("qrc"); }); + var cppArtifact = findChild(input, function(c) { return c.fileTags.contains("cpp"); }); var cmd = new Command(product.Qt.core.binPath + '/rcc', - [input.filePath, '-name', - FileInfo.completeBaseName(input.filePath), - '-o', output.filePath]); - cmd.description = 'rcc ' + input.fileName; + [qrcArtifact.filePath, + "-temp", input.filePath, + "-name", FileInfo.completeBaseName(input.filePath), + "-o", output.filePath, + "-pass", "2"]); + cmd.description = "rcc (pass 2) " + qrcArtifact.fileName; cmd.highlight = 'codegen'; return cmd; } diff --git a/src/lib/qtprofilesetup/templates/dbus.qbs b/src/lib/qtprofilesetup/templates/dbus.qbs index 2bbcf1985..9245a63e3 100644 --- a/src/lib/qtprofilesetup/templates/dbus.qbs +++ b/src/lib/qtprofilesetup/templates/dbus.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import qbs.FileInfo import qbs.ModUtils import "../QtModule.qbs" as QtModule @@ -63,6 +62,7 @@ QtModule { libNameForLinkerRelease: @libNameForLinkerRelease@ libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ + pluginTypes: @pluginTypes@ cpp.defines: @defines@ cpp.includePaths: @includes@ diff --git a/src/lib/qtprofilesetup/templates/gui.qbs b/src/lib/qtprofilesetup/templates/gui.qbs index 81b3bc2fb..eb69e0cad 100644 --- a/src/lib/qtprofilesetup/templates/gui.qbs +++ b/src/lib/qtprofilesetup/templates/gui.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import qbs.FileInfo import qbs.ModUtils import '../QtModule.qbs' as QtModule @@ -49,6 +48,7 @@ QtModule { libNameForLinkerRelease: @libNameForLinkerRelease@ libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ + pluginTypes: @pluginTypes@ cpp.defines: @defines@ cpp.includePaths: @includes@ diff --git a/src/lib/qtprofilesetup/templates/moc.js b/src/lib/qtprofilesetup/templates/moc.js index a9fddbc8a..aa67766b9 100644 --- a/src/lib/qtprofilesetup/templates/moc.js +++ b/src/lib/qtprofilesetup/templates/moc.js @@ -71,3 +71,32 @@ function fullPath(product) { return product.Qt.core.binPath + '/' + product.Qt.core.mocName; } + +function outputArtifacts(project, product, inputs, input) +{ + var mocInfo = QtMocScanner.apply(input); + if (!mocInfo.hasQObjectMacro) + return []; + var artifact = { fileTags: ["unmocable"] }; + if (mocInfo.hasPluginMetaDataMacro) + artifact.explicitlyDependsOn = ["qt_plugin_metadata"]; + if (input.fileTags.contains("hpp")) { + artifact.filePath = input.Qt.core.generatedHeadersDir + + "/moc_" + input.completeBaseName + ".cpp"; + var amalgamate = input.Qt.core.combineMocOutput; + artifact.fileTags.push(mocInfo.mustCompile ? (amalgamate ? "moc_cpp" : "cpp") : "hpp"); + } else { + artifact.filePath = input.Qt.core.generatedHeadersDir + '/' + + input.completeBaseName + ".moc"; + artifact.fileTags.push("hpp"); + } + return [artifact]; +} + +function commands(project, product, inputs, outputs, input, output) +{ + var cmd = new Command(fullPath(product), args(product, input, output.filePath)); + cmd.description = 'moc ' + input.fileName; + cmd.highlight = 'codegen'; + return cmd; +} diff --git a/src/lib/qtprofilesetup/templates/module.qbs b/src/lib/qtprofilesetup/templates/module.qbs index d16307e32..60ad106e5 100644 --- a/src/lib/qtprofilesetup/templates/module.qbs +++ b/src/lib/qtprofilesetup/templates/module.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import '../QtModule.qbs' as QtModule QtModule { @@ -22,6 +21,7 @@ QtModule { libNameForLinkerRelease: @libNameForLinkerRelease@ libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ + pluginTypes: @pluginTypes@ cpp.defines: @defines@ cpp.includePaths: @includes@ cpp.libraryPaths: @libraryPaths@ diff --git a/src/lib/qtprofilesetup/templates/plugin.qbs b/src/lib/qtprofilesetup/templates/plugin.qbs index 378d8b8fe..e73e2a4d9 100644 --- a/src/lib/qtprofilesetup/templates/plugin.qbs +++ b/src/lib/qtprofilesetup/templates/plugin.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import '../QtPlugin.qbs' as QtPlugin QtPlugin { @@ -23,5 +22,6 @@ QtPlugin { libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ cpp.libraryPaths: @libraryPaths@ + extendsModules: @extends@ @additionalContent@ } diff --git a/src/lib/qtprofilesetup/templates/plugin_support.qbs b/src/lib/qtprofilesetup/templates/plugin_support.qbs new file mode 100644 index 000000000..13d95c383 --- /dev/null +++ b/src/lib/qtprofilesetup/templates/plugin_support.qbs @@ -0,0 +1,75 @@ +Module { + // Set by user. + property varList pluginsByType + + // Set by Qt modules. + property stringList pluginTypes + + // Set by setup-qt. + readonly property var allPluginsByType: @allPluginsByType@ + readonly property stringList nonEssentialPlugins: @nonEssentialPlugins@ + + // Derived. + readonly property var defaultPluginsByType: { + var map = {}; + for (var i = 0; i < (pluginTypes || []).length; ++i) { + var pType = pluginTypes[i]; + map[pType] = (allPluginsByType[pType] || []).filter(function(p) { + return !nonEssentialPlugins.contains(p); }); + } + return map; + } + readonly property var effectivePluginsByType: { + var ppt = pluginsByType || []; + var eppt = {}; + for (var i = 0; i < ppt.length; ++i) { + var listEntry = ppt[i]; + for (var pluginType in listEntry) { + var newValue = listEntry[pluginType]; + if (!newValue) + newValue = []; + else if (typeof newValue == "string") + newValue = [newValue]; + if (!Array.isArray(newValue)) + throw "Invalid value '" + newValue + "' in Qt.plugin_support.pluginsByType"; + eppt[pluginType] = (eppt[pluginType] || []).uniqueConcat(newValue); + } + } + var dppt = defaultPluginsByType; + for (var pluginType in dppt) { + if (!eppt[pluginType]) + eppt[pluginType] = dppt[pluginType]; + } + return eppt; + } + readonly property stringList enabledPlugins: { + var list = []; + var eppt = effectivePluginsByType; + for (var t in eppt) + Array.prototype.push.apply(list, eppt[t]); + return list; + } + + validate: { + var ppt = pluginsByType; + if (!ppt) + return; + var appt = allPluginsByType; + for (var i = 0; i < ppt.length; ++i) { + for (var pluginType in ppt[i]) { + var requestedPlugins = ppt[i][pluginType]; + if (!requestedPlugins) + continue; + var availablePlugins = appt[pluginType] || []; + if (typeof requestedPlugins === "string") + requestedPlugins = [requestedPlugins]; + for (var j = 0; j < requestedPlugins.length; ++j) { + if (!availablePlugins.contains(requestedPlugins[j])) { + throw "Plugin '" + requestedPlugins[j] + "' of type '" + pluginType + + "' was requested, but is not available."; + } + } + } + } + } +} diff --git a/src/lib/qtprofilesetup/templates/qml.qbs b/src/lib/qtprofilesetup/templates/qml.qbs index 3047873af..51bb5faa2 100644 --- a/src/lib/qtprofilesetup/templates/qml.qbs +++ b/src/lib/qtprofilesetup/templates/qml.qbs @@ -1,4 +1,3 @@ -import qbs import qbs.TextFile import '../QtModule.qbs' as QtModule import "qml.js" as Qml @@ -43,6 +42,7 @@ QtModule { libNameForLinkerRelease: @libNameForLinkerRelease@ libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ + pluginTypes: @pluginTypes@ cpp.defines: @defines@ cpp.includePaths: @includes@ cpp.libraryPaths: @libraryPaths@ diff --git a/src/lib/qtprofilesetup/templates/quick.qbs b/src/lib/qtprofilesetup/templates/quick.qbs index 745f29cc2..67394cabb 100644 --- a/src/lib/qtprofilesetup/templates/quick.qbs +++ b/src/lib/qtprofilesetup/templates/quick.qbs @@ -28,7 +28,6 @@ ** ****************************************************************************/ -import qbs import qbs.File import qbs.FileInfo import qbs.Process @@ -58,6 +57,7 @@ QtModule { libNameForLinkerRelease: @libNameForLinkerRelease@ libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ + pluginTypes: @pluginTypes@ cpp.defines: @defines@ cpp.includePaths: @includes@ cpp.libraryPaths: @libraryPaths@ diff --git a/src/lib/qtprofilesetup/templates/scxml.qbs b/src/lib/qtprofilesetup/templates/scxml.qbs index 0581ff8d3..b02a49246 100644 --- a/src/lib/qtprofilesetup/templates/scxml.qbs +++ b/src/lib/qtprofilesetup/templates/scxml.qbs @@ -1,4 +1,3 @@ -import qbs 1.0 import qbs.FileInfo import qbs.Utilities import "../QtModule.qbs" as QtModule @@ -22,7 +21,7 @@ QtModule { } Artifact { filePath: input.baseName + ".cpp" - fileTags: ["cpp"] + fileTags: ["cpp", "unmocable"] } prepare: { @@ -70,6 +69,7 @@ QtModule { libNameForLinkerRelease: @libNameForLinkerRelease@ libFilePathDebug: @libFilePathDebug@ libFilePathRelease: @libFilePathRelease@ + pluginTypes: @pluginTypes@ cpp.defines: @defines@ cpp.includePaths: @includes@ |