diff options
Diffstat (limited to 'src/lib/corelib/loader')
38 files changed, 11300 insertions, 0 deletions
diff --git a/src/lib/corelib/loader/astimportshandler.cpp b/src/lib/corelib/loader/astimportshandler.cpp new file mode 100644 index 000000000..c0281ca24 --- /dev/null +++ b/src/lib/corelib/loader/astimportshandler.cpp @@ -0,0 +1,293 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "astimportshandler.h" + +#include "itemreadervisitorstate.h" + +#include <jsextensions/jsextensions.h> +#include <language/asttools.h> +#include <language/builtindeclarations.h> +#include <language/filecontext.h> +#include <logging/logger.h> +#include <logging/translator.h> +#include <parser/qmljsast_p.h> +#include <tools/error.h> +#include <tools/fileinfo.h> +#include <tools/qttools.h> +#include <tools/stringconstants.h> +#include <tools/version.h> + +#include <QtCore/qdiriterator.h> + +namespace qbs { +namespace Internal { + +ASTImportsHandler::ASTImportsHandler(ItemReaderVisitorState &visitorState, Logger &logger, + const FileContextPtr &file) + : m_visitorState(visitorState) + , m_logger(logger) + , m_file(file) + , m_directory(FileInfo::path(m_file->filePath())) +{ +} + +void ASTImportsHandler::handleImports(const QbsQmlJS::AST::UiImportList *uiImportList) +{ + const auto searchPaths = m_file->searchPaths(); + for (const QString &searchPath : searchPaths) + collectPrototypes(searchPath + QStringLiteral("/imports"), QString()); + + // files in the same directory are available as prototypes + collectPrototypes(m_directory, QString()); + + bool baseImported = false; + for (const auto *it = uiImportList; it; it = it->next) + handleImport(it->import, &baseImported); + if (!baseImported) { + QStringRef qbsref(&StringConstants::qbsModule()); + QbsQmlJS::AST::UiQualifiedId qbsURI(qbsref); + qbsURI.finish(); + QbsQmlJS::AST::UiImport imp(&qbsURI); + handleImport(&imp, &baseImported); + } + + for (auto it = m_jsImports.constBegin(); it != m_jsImports.constEnd(); ++it) + m_file->addJsImport(it.value()); +} + +void ASTImportsHandler::handleImport(const QbsQmlJS::AST::UiImport *import, bool *baseImported) +{ + QStringList importUri; + bool isBase = false; + if (import->importUri) { + importUri = toStringList(import->importUri); + isBase = (importUri.size() == 1 && importUri.front() == StringConstants::qbsModule()) + || (importUri.size() == 2 && importUri.front() == StringConstants::qbsModule() + && importUri.last() == StringConstants::baseVar()); + if (isBase) { + *baseImported = true; + checkImportVersion(import->versionToken); + } else if (import->versionToken.length) { + m_logger.printWarning(ErrorInfo(Tr::tr("Superfluous version specification."), + toCodeLocation(m_file->filePath(), import->versionToken))); + } + } + + QString as; + if (isBase) { + if (Q_UNLIKELY(!import->importId.isNull())) { + throw ErrorInfo(Tr::tr("Import of qbs.base must have no 'as <Name>'"), + toCodeLocation(m_file->filePath(), import->importIdToken)); + } + } else { + if (importUri.size() == 2 && importUri.front() == StringConstants::qbsModule()) { + const QString extensionName = importUri.last(); + if (JsExtensions::hasExtension(extensionName)) { + if (Q_UNLIKELY(!import->importId.isNull())) { + throw ErrorInfo(Tr::tr("Import of built-in extension '%1' " + "must not have 'as' specifier.").arg(extensionName), + toCodeLocation(m_file->filePath(), import->asToken)); + } + if (Q_UNLIKELY(m_file->jsExtensions().contains(extensionName))) { + m_logger.printWarning(ErrorInfo(Tr::tr("Built-in extension '%1' already " + "imported.").arg(extensionName), + toCodeLocation(m_file->filePath(), + import->importToken))); + } else { + m_file->addJsExtension(extensionName); + } + return; + } + } + + if (import->importId.isNull()) { + if (!import->fileName.isNull()) { + throw ErrorInfo(Tr::tr("File imports require 'as <Name>'"), + toCodeLocation(m_file->filePath(), import->importToken)); + } + if (importUri.empty()) { + throw ErrorInfo(Tr::tr("Invalid import URI."), + toCodeLocation(m_file->filePath(), import->importToken)); + } + as = importUri.last(); + } else { + as = import->importId.toString(); + } + + if (Q_UNLIKELY(JsExtensions::hasExtension(as))) + throw ErrorInfo(Tr::tr("Cannot reuse the name of built-in extension '%1'.").arg(as), + toCodeLocation(m_file->filePath(), import->importIdToken)); + if (Q_UNLIKELY(!m_importAsNames.insert(as).second)) { + throw ErrorInfo(Tr::tr("Cannot import into the same name more than once."), + toCodeLocation(m_file->filePath(), import->importIdToken)); + } + } + + if (!import->fileName.isNull()) { + QString filePath = FileInfo::resolvePath(m_directory, import->fileName.toString()); + + QFileInfo fi(filePath); + if (Q_UNLIKELY(!fi.exists())) + throw ErrorInfo(Tr::tr("Cannot find imported file %0.") + .arg(QDir::toNativeSeparators(filePath)), + CodeLocation(m_file->filePath(), import->fileNameToken.startLine, + import->fileNameToken.startColumn)); + filePath = fi.canonicalFilePath(); + if (fi.isDir()) { + collectPrototypesAndJsCollections(filePath, as, + toCodeLocation(m_file->filePath(), import->fileNameToken)); + } else { + if (filePath.endsWith(QStringLiteral(".js"), Qt::CaseInsensitive)) { + JsImport &jsImport = m_jsImports[as]; + jsImport.scopeName = as; + jsImport.filePaths.push_back(filePath); + jsImport.location + = toCodeLocation(m_file->filePath(), import->firstSourceLocation()); + } else if (filePath.endsWith(QStringLiteral(".qbs"), Qt::CaseInsensitive)) { + m_typeNameToFile.insert(QStringList(as), filePath); + } else { + throw ErrorInfo(Tr::tr("Can only import .qbs and .js files"), + CodeLocation(m_file->filePath(), import->fileNameToken.startLine, + import->fileNameToken.startColumn)); + } + } + } else if (!importUri.empty()) { + const QString importPath = isBase + ? QStringLiteral("qbs/base") : importUri.join(QDir::separator()); + bool found = m_typeNameToFile.contains(importUri); + if (!found) { + const auto searchPaths = m_file->searchPaths(); + for (const QString &searchPath : searchPaths) { + const QFileInfo fi(FileInfo::resolvePath( + FileInfo::resolvePath(searchPath, + StringConstants::importsDir()), + importPath)); + if (fi.isDir()) { + // ### versioning, qbsdir file, etc. + const QString &resultPath = fi.absoluteFilePath(); + collectPrototypesAndJsCollections(resultPath, as, + toCodeLocation(m_file->filePath(), import->fileNameToken)); + found = true; + break; + } + } + } + if (Q_UNLIKELY(!found)) { + throw ErrorInfo(Tr::tr("import %1 not found") + .arg(importUri.join(QLatin1Char('.'))), + toCodeLocation(m_file->filePath(), import->fileNameToken)); + } + } +} + +Version ASTImportsHandler::readImportVersion(const QString &str, const CodeLocation &location) +{ + const Version v = Version::fromString(str); + if (Q_UNLIKELY(!v.isValid())) + throw ErrorInfo(Tr::tr("Cannot parse version number in import statement."), location); + if (Q_UNLIKELY(v.patchLevel() != 0)) { + throw ErrorInfo(Tr::tr("Version number in import statement cannot have more than " + "two components."), location); + } + return v; +} + +bool ASTImportsHandler::addPrototype(const QString &fileName, const QString &filePath, + const QString &as, bool needsCheck) +{ + if (needsCheck && fileName.size() <= 4) + return false; + + const QString componentName = fileName.left(fileName.size() - 4); + // ### validate componentName + + if (needsCheck && !componentName.at(0).isUpper()) + return false; + + QStringList prototypeName; + if (!as.isEmpty()) + prototypeName.push_back(as); + prototypeName.push_back(componentName); + if (!m_typeNameToFile.contains(prototypeName)) + m_typeNameToFile.insert(prototypeName, filePath); + return true; +} + +void ASTImportsHandler::checkImportVersion(const QbsQmlJS::AST::SourceLocation &versionToken) const +{ + if (!versionToken.length) + return; + const QString importVersionString + = m_file->content().mid(versionToken.offset, versionToken.length); + const Version importVersion = readImportVersion(importVersionString, + toCodeLocation(m_file->filePath(), versionToken)); + if (Q_UNLIKELY(importVersion != BuiltinDeclarations::instance().languageVersion())) + throw ErrorInfo(Tr::tr("Incompatible qbs language version %1. This is version %2.").arg( + importVersionString, + BuiltinDeclarations::instance().languageVersion().toString()), + toCodeLocation(m_file->filePath(), versionToken)); + +} + +void ASTImportsHandler::collectPrototypes(const QString &path, const QString &as) +{ + QStringList fileNames; // Yes, file *names*. + m_visitorState.findDirectoryEntries(path, &fileNames); + for (const QString &fileName : std::as_const(fileNames)) + addPrototype(fileName, path + QLatin1Char('/') + fileName, as, false); +} + +void ASTImportsHandler::collectPrototypesAndJsCollections(const QString &path, const QString &as, + const CodeLocation &location) +{ + collectPrototypes(path, as); + QDirIterator dirIter(path, StringConstants::jsFileWildcards()); + while (dirIter.hasNext()) { + dirIter.next(); + JsImport &jsImport = m_jsImports[as]; + if (jsImport.scopeName.isNull()) { + jsImport.scopeName = as; + jsImport.location = location; + } + jsImport.filePaths.push_back(dirIter.filePath()); + } +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/astimportshandler.h b/src/lib/corelib/loader/astimportshandler.h new file mode 100644 index 000000000..582e1c698 --- /dev/null +++ b/src/lib/corelib/loader/astimportshandler.h @@ -0,0 +1,94 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QBS_ASTIMPORTSHANDLER_H +#define QBS_ASTIMPORTSHANDLER_H + +#include <language/forward_decls.h> + +#include <parser/qmljsastfwd_p.h> +#include <tools/set.h> + +#include <QtCore/qhash.h> +#include <QtCore/qstringlist.h> + +namespace qbs { +class CodeLocation; +class Version; + +namespace Internal { +class ItemReaderVisitorState; +class JsImport; +class Logger; + +class ASTImportsHandler +{ +public: + ASTImportsHandler(ItemReaderVisitorState &visitorState, Logger &logger, + const FileContextPtr &file); + + void handleImports(const QbsQmlJS::AST::UiImportList *uiImportList); + + QHash<QStringList, QString> typeNameFileMap() const { return m_typeNameToFile; } + +private: + static Version readImportVersion(const QString &str, const CodeLocation &location); + + bool addPrototype(const QString &fileName, const QString &filePath, const QString &as, + bool needsCheck); + void checkImportVersion(const QbsQmlJS::AST::SourceLocation &versionToken) const; + void collectPrototypes(const QString &path, const QString &as); + void collectPrototypesAndJsCollections(const QString &path, const QString &as, + const CodeLocation &location); + void handleImport(const QbsQmlJS::AST::UiImport *import, bool *baseImported); + + ItemReaderVisitorState &m_visitorState; + Logger &m_logger; + const FileContextPtr &m_file; + const QString m_directory; + QHash<QStringList, QString> m_typeNameToFile; + Set<QString> m_importAsNames; + + using JsImportsHash = QHash<QString, JsImport>; + JsImportsHash m_jsImports; +}; + +} // namespace Internal +} // namespace qbs + +#endif // Include guard diff --git a/src/lib/corelib/loader/astpropertiesitemhandler.cpp b/src/lib/corelib/loader/astpropertiesitemhandler.cpp new file mode 100644 index 000000000..8183e6b79 --- /dev/null +++ b/src/lib/corelib/loader/astpropertiesitemhandler.cpp @@ -0,0 +1,195 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "astpropertiesitemhandler.h" + +#include <language/item.h> +#include <language/value.h> + +#include <logging/translator.h> +#include <tools/error.h> +#include <tools/qbsassert.h> +#include <tools/stringconstants.h> + +namespace qbs { +namespace Internal { + +ASTPropertiesItemHandler::ASTPropertiesItemHandler(Item *parentItem, ItemPool &itemPool) + : m_parentItem(parentItem), m_itemPool(itemPool) +{ +} + +void ASTPropertiesItemHandler::handlePropertiesItems() +{ + // TODO: Simply forbid Properties items to have child items and get rid of this check. + if (m_parentItem->type() != ItemType::Properties) + setupAlternatives(); +} + +void ASTPropertiesItemHandler::setupAlternatives() +{ + auto it = m_parentItem->m_children.begin(); + while (it != m_parentItem->m_children.end()) { + Item * const child = *it; + bool remove = false; + if (child->type() == ItemType::Properties) { + handlePropertiesBlock(child); + remove = m_parentItem->type() != ItemType::Export; + } + if (remove) + it = m_parentItem->m_children.erase(it); + else + ++it; + } +} + +class PropertiesBlockConverter +{ +public: + PropertiesBlockConverter(const JSSourceValue::AltProperty &condition, + const JSSourceValue::AltProperty &overrideListProperties, + Item *propertiesBlockContainer, const Item *propertiesBlock, + ItemPool &pool) + : m_propertiesBlockContainer(propertiesBlockContainer) + , m_propertiesBlock(propertiesBlock) + , m_itemPool(pool) + { + m_alternative.condition = condition; + m_alternative.overrideListProperties = overrideListProperties; + } + + void apply() + { + doApply(m_propertiesBlockContainer, m_propertiesBlock); + } + +private: + JSSourceValue::Alternative m_alternative; + Item * const m_propertiesBlockContainer; + const Item * const m_propertiesBlock; + ItemPool &m_itemPool; + + void doApply(Item *outer, const Item *inner) + { + for (auto it = inner->properties().constBegin(); + it != inner->properties().constEnd(); ++it) { + if (inner == m_propertiesBlock + && (it.key() == StringConstants::conditionProperty() + || it.key() == StringConstants::overrideListPropertiesProperty())) { + continue; + } + if (it.value()->type() == Value::ItemValueType) { + Item * const innerVal = std::static_pointer_cast<ItemValue>(it.value())->item(); + ItemValuePtr outerVal = outer->itemProperty(it.key(), m_itemPool); + if (!outerVal) { + outerVal = ItemValue::create(Item::create(&m_itemPool, innerVal->type()), + true); + outer->setProperty(it.key(), outerVal); + } + doApply(outerVal->item(), innerVal); + } else if (it.value()->type() == Value::JSSourceValueType) { + const ValuePtr outerVal = outer->property(it.key()); + if (Q_UNLIKELY(outerVal && outerVal->type() != Value::JSSourceValueType)) { + throw ErrorInfo(Tr::tr("Incompatible value type in unconditional value at %1.") + .arg(outerVal->location().toString())); + } + doApply(it.key(), outer, std::static_pointer_cast<JSSourceValue>(outerVal), + std::static_pointer_cast<JSSourceValue>(it.value())); + } else { + QBS_CHECK(!"Unexpected value type in conditional value."); + } + } + } + + void doApply(const QString &propertyName, Item *item, JSSourceValuePtr value, + const JSSourceValuePtr &conditionalValue) + { + if (!value) { + value = JSSourceValue::create(true); + value->setFile(conditionalValue->file()); + item->setProperty(propertyName, value); + value->setSourceCode(StringConstants::baseVar()); + value->setSourceUsesBase(); + } + m_alternative.value = conditionalValue; + value->addAlternative(m_alternative); + } +}; + +static JSSourceValue::AltProperty getPropertyData(const Item *propertiesItem, const QString &name) +{ + const ValuePtr value = propertiesItem->property(name); + if (!value) { + if (name == StringConstants::conditionProperty()) { + throw ErrorInfo(Tr::tr("Properties.condition must be provided."), + propertiesItem->location()); + } + return {StringConstants::falseValue(), propertiesItem->location()}; + } + if (Q_UNLIKELY(value->type() != Value::JSSourceValueType)) { + throw ErrorInfo(Tr::tr("Properties.%1 must be a value binding.").arg(name), + propertiesItem->location()); + } + if (name == StringConstants::overrideListPropertiesProperty()) { + const Item *parent = propertiesItem->parent(); + while (parent) { + if (parent->type() == ItemType::Product) + break; + parent = parent->parent(); + } + if (!parent) { + throw ErrorInfo(Tr::tr("Properties.overrideListProperties can only be set " + "in a Product item.")); + } + + } + const JSSourceValuePtr srcval = std::static_pointer_cast<JSSourceValue>(value); + return {srcval->sourceCodeForEvaluation(), srcval->location()}; +} + +void ASTPropertiesItemHandler::handlePropertiesBlock(const Item *propertiesItem) +{ + const auto condition = getPropertyData(propertiesItem, StringConstants::conditionProperty()); + const auto overrideListProperties = getPropertyData(propertiesItem, + StringConstants::overrideListPropertiesProperty()); + PropertiesBlockConverter(condition, overrideListProperties, m_parentItem, + propertiesItem, m_itemPool).apply(); +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/astpropertiesitemhandler.h b/src/lib/corelib/loader/astpropertiesitemhandler.h new file mode 100644 index 000000000..804abb8a1 --- /dev/null +++ b/src/lib/corelib/loader/astpropertiesitemhandler.h @@ -0,0 +1,65 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QBS_ASTPROPERTIESITEMHANDLER_H +#define QBS_ASTPROPERTIESITEMHANDLER_H + +namespace qbs { +namespace Internal { +class Item; +class ItemPool; + +class ASTPropertiesItemHandler +{ +public: + ASTPropertiesItemHandler(Item *parentItem, ItemPool &itemPool); + + void handlePropertiesItems(); + +private: + void setupAlternatives(); + void handlePropertiesBlock(const Item *propertiesItem); + + Item * const m_parentItem; + ItemPool &m_itemPool; +}; + +} // namespace Internal +} // namespace qbs + +#endif // Include guard. diff --git a/src/lib/corelib/loader/dependenciesresolver.cpp b/src/lib/corelib/loader/dependenciesresolver.cpp new file mode 100644 index 000000000..059592e9c --- /dev/null +++ b/src/lib/corelib/loader/dependenciesresolver.cpp @@ -0,0 +1,1175 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "dependenciesresolver.h" + +#include "itemreader.h" +#include "loaderutils.h" +#include "moduleinstantiator.h" +#include "moduleloader.h" + +#include <language/scriptengine.h> +#include <language/evaluator.h> +#include <language/item.h> +#include <language/itempool.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/fileinfo.h> +#include <tools/preferences.h> +#include <tools/profiling.h> +#include <tools/settings.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <optional> +#include <queue> +#include <unordered_map> + +namespace qbs::Internal { +namespace { +enum class HandleDependency { Use, Ignore, Defer }; + +class LoadModuleResult +{ +public: + Item *moduleItem = nullptr; + ProductContext *product = nullptr; + HandleDependency handleDependency = HandleDependency::Use; +}; + +// Corresponds completely to a Depends item. +// May result in more than one module, due to "multiplexing" properties such as subModules etc. +// May also result in no module at all, e.g. if productTypes does not match anything. +class EvaluatedDependsItem +{ +public: + Item *item = nullptr; + QualifiedId name; + QStringList subModules; + FileTags productTypes; + QStringList multiplexIds; + std::optional<QStringList> profiles; + VersionRange versionRange; + QVariantMap parameters; + bool limitToSubProject = false; + bool requiredLocally = true; + bool requiredGlobally = true; +}; + +// As opposed to EvaluatedDependsItem, one of these corresponds exactly to one module +// to be loaded. Such an attempt might still fail, though, which may or may not result +// in an error, depending on the value of Depends.required and other circumstances. +class FullyResolvedDependsItem +{ +public: + FullyResolvedDependsItem(ProductContext *product, const EvaluatedDependsItem &dependency); + FullyResolvedDependsItem(const EvaluatedDependsItem &dependency, QualifiedId name, + QString profile, QString multiplexId); + FullyResolvedDependsItem() = default; + static FullyResolvedDependsItem makeBaseDependency(); + + QString id() const; + CodeLocation location() const; + QString displayName() const; + + // If product is non-null, we already know which product the dependency targets. + // This happens either if Depends.productTypes was set, or if we tried to load the + // dependency before and already identified the product, but could not complete the + // procedure because said product had itself not been handled yet. + ProductContext *product = nullptr; + + Item *item = nullptr; + QualifiedId name; + QString profile; + QString multiplexId; + VersionRange versionRange; + QVariantMap parameters; + bool limitToSubProject = false; + bool requiredLocally = true; + bool requiredGlobally = true; + bool checkProduct = true; +}; + +class DependenciesResolvingState +{ +public: + Item *loadingItem = nullptr; + FullyResolvedDependsItem loadingItemOrigin; + std::queue<Item *> pendingDependsItems; + std::optional<EvaluatedDependsItem> currentDependsItem; + std::queue<FullyResolvedDependsItem> pendingResolvedDependencies; + bool requiredByLoadingItem = true; +}; + +class DependenciesContextImpl : public DependenciesContext +{ +public: + DependenciesContextImpl(ProductContext &product, LoaderState &loaderState); + + std::list<DependenciesResolvingState> stateStack; + +private: + std::pair<ProductDependency, ProductContext *> pendingDependency() const override; + + void setSearchPathsForProduct(LoaderState &loaderState); + + ProductContext &m_product; +}; + +class DependenciesResolver +{ +public: + DependenciesResolver(LoaderState &loaderState, ProductContext &product, Deferral deferral) + : m_loaderState(loaderState), m_product(product), m_deferral(deferral) {} + + void resolve(); + LoadModuleResult loadModule(Item *loadingItem, const FullyResolvedDependsItem &dependency); + +private: + void evaluateNextDependsItem(); + HandleDependency handleResolvedDependencies(); + std::pair<Item::Module *, Item *> findExistingModule(const FullyResolvedDependsItem &dependency, + Item *item); + void updateModule(Item::Module &module, const FullyResolvedDependsItem &dependency); + int dependsChainLength(); + ProductContext *findMatchingProduct(const FullyResolvedDependsItem &dependency); + Item *findMatchingModule(const FullyResolvedDependsItem &dependency); + std::pair<bool, HandleDependency> checkProductDependency( + const FullyResolvedDependsItem &depSpec, const ProductContext &dep); + void checkModule(const FullyResolvedDependsItem &dependency, Item *moduleItem, + ProductContext *productDep); + void checkForModuleNamePrefixCollision(const FullyResolvedDependsItem &dependency); + Item::Module createModule(const FullyResolvedDependsItem &dependency, Item *item, + ProductContext *productDep); + void adjustDependsItemForMultiplexing(Item *dependsItem); + std::optional<EvaluatedDependsItem> evaluateDependsItem(Item *item); + std::queue<FullyResolvedDependsItem> multiplexDependency( + const EvaluatedDependsItem &dependency); + QVariantMap extractParameters(Item *dependsItem) const; + void forwardParameterDeclarations(const Item *dependsItem, const Item::Modules &modules); + void forwardParameterDeclarations(const QualifiedId &moduleName, Item *item, + const Item::Modules &modules); + std::list<DependenciesResolvingState> &stateStack(); + + LoaderState &m_loaderState; + ProductContext &m_product; + Deferral m_deferral; +}; + +static bool haveSameSubProject(const ProductContext &p1, const ProductContext &p2); +static QVariantMap safeToVariant(JSContext *ctx, const JSValue &v); + +} // namespace + +void resolveDependencies(ProductContext &product, Deferral deferral, LoaderState &loaderState) +{ + DependenciesResolver(loaderState, product, deferral).resolve(); +} + +Item *loadBaseModule(ProductContext &product, Item *item, LoaderState &loaderState) +{ + const auto baseDependency = FullyResolvedDependsItem::makeBaseDependency(); + Item * const moduleItem = DependenciesResolver(loaderState, product, Deferral::NotAllowed) + .loadModule(item, baseDependency).moduleItem; + if (Q_UNLIKELY(!moduleItem)) + throw ErrorInfo(Tr::tr("Cannot load base qbs module.")); + return moduleItem; +} + +namespace { + +void DependenciesResolver::resolve() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.dependenciesResolving : nullptr); + + if (!m_product.dependenciesContext) { + m_product.dependenciesContext = std::make_unique<DependenciesContextImpl>( + m_product, m_loaderState); + } else { + QBS_CHECK(!m_product.dependenciesContext->dependenciesResolved); + } + SearchPathsManager searchPathsMgr(m_loaderState.itemReader(), m_product.searchPaths); + + while (!stateStack().empty()) { + auto &state = stateStack().front(); + + // If we have pending FullyResolvedDependsItems, then these are handled first. + if (handleResolvedDependencies() == HandleDependency::Defer) + return; + + // The above procedure might have pushed another state to the stack due to recursive + // dependencies (i.e. Depends items in the newly loaded module), in which case we + // continue with that one. + if (&state != &stateStack().front()) + continue; + + // If we have a pending EvaluatedDependsItem, we multiplex it and then handle + // the resulting FullyResolvedDependsItems, if there were any. + if (state.currentDependsItem) { + QBS_CHECK(state.pendingResolvedDependencies.empty()); + + // We postpone handling Depends.productTypes for as long as possible, because + // the full type of a product becomes available only after its modules have been loaded. + if (!state.currentDependsItem->productTypes.empty() && m_deferral == Deferral::Allowed) + return; + + state.pendingResolvedDependencies = multiplexDependency(*state.currentDependsItem); + state.currentDependsItem.reset(); + m_deferral = Deferral::Allowed; // We made progress. + + continue; + } + + // Here we have no resolved/evaluated Depends items of any kind, so we evaluate the next + // pending Depends item. + evaluateNextDependsItem(); + if (state.currentDependsItem) + continue; + + // No resolved or unresolved Depends items are left, so we're done with the current state. + QBS_CHECK(!state.currentDependsItem); + QBS_CHECK(state.pendingResolvedDependencies.empty()); + QBS_CHECK(state.pendingDependsItems.empty()); + + // This ensures our invariant: A sorted module list in the product + // (dependers after dependencies). + if (stateStack().size() > 1) { + QBS_CHECK(state.loadingItem->type() == ItemType::ModuleInstance); + Item::Modules &modules = m_product.item->modules(); + const auto loadingItemModule = std::find_if(modules.begin(), modules.end(), + [&](const Item::Module &m) { + return m.item == state.loadingItem; + }); + QBS_CHECK(loadingItemModule != modules.end()); + const Item::Module tempModule = *loadingItemModule; + modules.erase(loadingItemModule); + modules.push_back(tempModule); + } + stateStack().pop_front(); + } + m_product.dependenciesContext->dependenciesResolved = true; +} + +void DependenciesResolver::evaluateNextDependsItem() +{ + auto &state = stateStack().front(); + while (!state.pendingDependsItems.empty()) { + QBS_CHECK(!state.currentDependsItem); + QBS_CHECK(state.pendingResolvedDependencies.empty()); + Item * const dependsItem = state.pendingDependsItems.front(); + state.pendingDependsItems.pop(); + adjustDependsItemForMultiplexing(dependsItem); + if (auto evaluated = evaluateDependsItem(dependsItem)) { + evaluated->requiredGlobally = evaluated->requiredLocally + && state.loadingItemOrigin.requiredGlobally; + state.currentDependsItem = evaluated; + return; + } + } +} + +HandleDependency DependenciesResolver::handleResolvedDependencies() +{ + DependenciesResolvingState &state = stateStack().front(); + while (!state.pendingResolvedDependencies.empty()) { + QBS_CHECK(!state.currentDependsItem); + const FullyResolvedDependsItem dependency = state.pendingResolvedDependencies.front(); + try { + const LoadModuleResult res = loadModule(state.loadingItem, dependency); + switch (res.handleDependency) { + case HandleDependency::Defer: + QBS_CHECK(m_deferral == Deferral::Allowed); + + // Optimization: We already looked up the product, so let's not do that again + // next time. + if (res.product) + state.pendingResolvedDependencies.front().product = res.product; + + return HandleDependency::Defer; + case HandleDependency::Ignore: + // This happens if the dependency was not required or the module was already + // loaded via another path. + state.pendingResolvedDependencies.pop(); + continue; + case HandleDependency::Use: + if (dependency.name.toString() == StringConstants::qbsModule()) { + // Shortcut: No need to look for recursive dependencies in the base module. + state.pendingResolvedDependencies.pop(); + continue; + } + m_deferral = Deferral::Allowed; // We made progress. + break; + } + + QBS_CHECK(res.moduleItem); + + // Now continue with the dependencies of the just-loaded module. + std::queue<Item *> moduleDependsItems; + for (Item * const child : res.moduleItem->children()) { + if (child->type() == ItemType::Depends) + moduleDependsItems.push(child); + } + state.pendingResolvedDependencies.pop(); + stateStack().push_front( + {res.moduleItem, dependency, moduleDependsItems, {}, {}, + dependency.requiredGlobally || state.requiredByLoadingItem}); + stateStack().front().pendingResolvedDependencies.push( + FullyResolvedDependsItem::makeBaseDependency()); + break; + } catch (const ErrorInfo &e) { + if (dependency.name.toString() == StringConstants::qbsModule()) + throw e; + + // See QBS-1338 for why we do not abort handling the product. + state.pendingResolvedDependencies.pop(); + Item::Modules &modules = m_product.item->modules(); + + // Unwind. + while (stateStack().size() > 1) { + const auto loadingItemModule = std::find_if( + modules.begin(), modules.end(), [&](const Item::Module &m) { + return m.item == stateStack().front().loadingItem; + }); + for (auto it = loadingItemModule; it != modules.end(); ++it) { + createNonPresentModule(m_loaderState.itemPool(), it->name.toString(), + QLatin1String("error in Depends chain"), it->item); + } + modules.erase(loadingItemModule, modules.end()); + stateStack().pop_front(); + } + + m_product.handleError(e); + return HandleDependency::Ignore; + } + } + return HandleDependency::Ignore; +} + +// Produces an item of type ModuleInstance corresponding to the specified dependency. +// The instance's prototype item is either of type Export (if the dependency is a product) +// or of type Module (for an actual module). +// The loadingItem parameter is either the depending product or another module. The newly +// created module is added to the module list of the product item and additionally to the +// loading item's one, if it is not the product. Its name is also injected into the respective +// scopes. +LoadModuleResult DependenciesResolver::loadModule( + Item *loadingItem, const FullyResolvedDependsItem &dependency) +{ + qCDebug(lcModuleLoader) << "loadModule name:" << dependency.name.toString() + << "id:" << dependency.id(); + + QBS_CHECK(loadingItem); + + ProductContext *productDep = nullptr; + Item *moduleItem = nullptr; + + const auto addLoadContext = [&](Item::Module &module) { + module.loadContexts.emplace_back(dependency.item, + std::make_pair(dependency.parameters, + INT_MAX - dependsChainLength())); + }; + + // The module might already have been loaded for this product (directly or indirectly). + const auto &[existingModule, moduleWithSameName] + = findExistingModule(dependency, m_product.item); + if (existingModule) { + // Merge version range and required property. These will be checked again + // after probes resolving. + if (existingModule->name.toString() != StringConstants::qbsModule()) + updateModule(*existingModule, dependency); + + QBS_CHECK(existingModule->item); + moduleItem = existingModule->item; + const auto matcher = [loadingItem](const Item::Module::LoadContext &context) { + return context.loadingItem() == loadingItem; + }; + const auto it = std::find_if(existingModule->loadContexts.begin(), + existingModule->loadContexts.end(), matcher); + if (it == existingModule->loadContexts.end()) + addLoadContext(*existingModule); + else + it->parameters.first = mergeDependencyParameters(it->parameters.first, + dependency.parameters); + } else if (dependency.product) { + productDep = dependency.product; // We have already done the look-up. + } else if (productDep = findMatchingProduct(dependency); !productDep) { + moduleItem = findMatchingModule(dependency); + } + + if (productDep) { + const auto checkResult = checkProductDependency(dependency, *productDep); + + // We found a product dependency, but that product has not been handled yet, + // so stop dependency resolving for the current product and resume it later, when the + // dependency is ready. + if (checkResult.second == HandleDependency::Defer) + return {nullptr, productDep, HandleDependency::Defer}; + + if (checkResult.first) { + QBS_CHECK(productDep->mergedExportItem); + moduleItem = productDep->mergedExportItem->clone(m_loaderState.itemPool()); + moduleItem->setParent(nullptr); + + // Needed for isolated Export item evaluation. + moduleItem->setPrototype(productDep->mergedExportItem); + } else { + // The product dependency is faulty, but Depends.reqired was false. + productDep = nullptr; + } + } + + if (moduleItem) + checkModule(dependency, moduleItem, productDep); + + // Can only happen with multiplexing. + if (moduleWithSameName && moduleWithSameName != moduleItem) + QBS_CHECK(productDep); + + // The loading name is only used to ensure consistent sorting in case of equal + // value priorities; see ModulePropertyMerger. + QString loadingName; + if (loadingItem == m_product.item) { + loadingName = m_product.name; + } else if (m_product.dependenciesContext && !stateStack().empty()) { + const auto &loadingItemOrigin = stateStack().front().loadingItemOrigin; + loadingName = loadingItemOrigin.name.toString() + loadingItemOrigin.multiplexId + + loadingItemOrigin.profile; + } + instantiateModule({m_product, loadingItem, loadingName, moduleItem, moduleWithSameName, + productDep ? productDep->item : nullptr, dependency.name, dependency.id(), + bool(existingModule)}, m_loaderState); + + // At this point, a null module item is only possible for a non-required dependency. + // Note that we still needed to to the instantiation above, as that injects the module + // name into the surrounding item for the ".present" check. + if (!moduleItem) { + QBS_CHECK(!dependency.requiredGlobally); + return {nullptr, nullptr, HandleDependency::Ignore}; + } + + const auto addLocalModule = [&] { + if (loadingItem->type() == ItemType::ModuleInstance + && !findExistingModule(dependency, loadingItem).first) { + loadingItem->addModule(createModule(dependency, moduleItem, productDep)); + } + }; + + // The module has already been loaded, so we don't need to add it to the product's list of + // modules, nor do we need to handle its dependencies. The only thing we might need to + // do is to add it to the "local" list of modules of the loading item, if it isn't in there yet. + if (existingModule) { + addLocalModule(); + return {nullptr, nullptr, HandleDependency::Ignore}; + } + + qCDebug(lcModuleLoader) << "module loaded:" << dependency.name.toString(); + if (m_product.item) { + Item::Module module = createModule(dependency, moduleItem, productDep); + module.required = dependency.requiredGlobally; + addLoadContext(module); + module.maxDependsChainLength = dependsChainLength(); + m_product.item->addModule(module); + addLocalModule(); + } + return {moduleItem, nullptr, HandleDependency::Use}; +} + +std::pair<Item::Module *, Item *> DependenciesResolver::findExistingModule( + const FullyResolvedDependsItem &dependency, Item *item) +{ + if (!item) // Happens if and only if called via loadBaseModule(). + return {}; + Item *moduleWithSameName = nullptr; + for (Item::Module &m : item->modules()) { + if (m.name != dependency.name) + continue; + if (!m.product) { + QBS_CHECK(!dependency.product); + return {&m, m.item}; + } + if ((dependency.profile.isEmpty() || (m.product->profileName == dependency.profile)) + && m.product->multiplexConfigurationId == dependency.multiplexId) { + return {&m, m.item}; + } + + // This can happen if there are dependencies to several variants of a multiplexed + // product. + moduleWithSameName = m.item; + } + return {nullptr, moduleWithSameName}; +} + +void DependenciesResolver::updateModule( + Item::Module &module, const FullyResolvedDependsItem &dependency) +{ + forwardParameterDeclarations(dependency.item, m_product.item->modules()); + module.versionRange.narrowDown(dependency.versionRange); + module.required |= dependency.requiredGlobally; + if (dependsChainLength() > module.maxDependsChainLength) + module.maxDependsChainLength = dependsChainLength(); +} + +int DependenciesResolver::dependsChainLength() +{ + return m_product.dependenciesContext ? stateStack().size() : 1; +} + +ProductContext *DependenciesResolver::findMatchingProduct( + const FullyResolvedDependsItem &dependency) +{ + const auto constraint = [this, &dependency](ProductContext &product) { + if (product.multiplexConfigurationId != dependency.multiplexId) + return false; + if (!dependency.profile.isEmpty() && dependency.profile != product.profileName) + return false; + if (dependency.limitToSubProject && !haveSameSubProject(m_product, product)) + return false; + return true; + + }; + return m_product.project->topLevelProject->productWithNameAndConstraint( + dependency.name.toString(), constraint); +} + +Item *DependenciesResolver::findMatchingModule( + const FullyResolvedDependsItem &dependency) +{ + // If we can tell that this is supposed to be a product dependency, we can skip + // the module look-up. + if (!dependency.profile.isEmpty() || !dependency.multiplexId.isEmpty()) { + if (dependency.requiredGlobally) { + if (!dependency.profile.isEmpty()) { + throw ErrorInfo(Tr::tr("Product '%1' depends on '%2', which does not exist " + "for the requested profile '%3'.") + .arg(m_product.displayName(), dependency.displayName(), + dependency.profile), + m_product.item->location()); + } + throw ErrorInfo(Tr::tr("Product '%1' depends on '%2', which does not exist.") + .arg(m_product.displayName(), dependency.displayName()), + m_product.item->location()); + } + return nullptr; + } + + if (Item *moduleItem = searchAndLoadModuleFile( + m_loaderState, m_product, dependency.location(), dependency.name)) { + QBS_CHECK(moduleItem->type() == ItemType::Module); + Item * const proto = moduleItem; + ModuleItemLocker locker(*moduleItem); + moduleItem = moduleItem->clone(m_loaderState.itemPool()); + moduleItem->setPrototype(proto); // For parameter declarations. + return moduleItem; + } + + if (!dependency.requiredGlobally) { + return createNonPresentModule(m_loaderState.itemPool(), dependency.name.toString(), + QStringLiteral("not found"), nullptr); + } + + throw ErrorInfo(Tr::tr("Dependency '%1' not found for product '%2'.") + .arg(dependency.name.toString(), m_product.displayName()), + dependency.location()); +} + +std::pair<bool, HandleDependency> DependenciesResolver::checkProductDependency( + const FullyResolvedDependsItem &depSpec, const ProductContext &dep) +{ + // Optimization: If we already checked the product earlier and then deferred, we don't + // need to check it again. + if (!depSpec.checkProduct) + return {true, HandleDependency::Use}; + + if (&dep == &m_product) { + throw ErrorInfo(Tr::tr("Dependency '%1' refers to itself.").arg(depSpec.name.toString()), + depSpec.location()); + } + + if (m_product.project->topLevelProject->isProductQueuedForHandling(dep)) { + if (m_deferral == Deferral::Allowed) + return {false, HandleDependency::Defer}; + ErrorInfo e; + e.append(Tr::tr("Cyclic dependencies detected:")); + e.append(Tr::tr("First product is '%1'.") + .arg(m_product.displayName()), m_product.item->location()); + e.append(Tr::tr("Second product is '%1'.") + .arg(dep.displayName()), dep.item->location()); + e.append(Tr::tr("Requested here."), depSpec.location()); + throw e; + } + + // This covers both the case of user-disabled products and products with errors. + // The latter are force-disabled in ProductContext::handleError(). + if (m_product.project->topLevelProject->isDisabledItem(dep.item)) { + if (depSpec.requiredGlobally) { + ErrorInfo e; + e.append(Tr::tr("Product '%1' depends on '%2',") + .arg(m_product.displayName(), dep.displayName()), + m_product.item->location()); + e.append(Tr::tr("but product '%1' is disabled.").arg(dep.displayName()), + dep.item->location()); + throw e; + } + return {false, HandleDependency::Ignore}; + } + return {true, HandleDependency::Use}; +} + +void DependenciesResolver::checkModule( + const FullyResolvedDependsItem &dependency, Item *moduleItem, ProductContext *productDep) +{ + // When loading a pseudo or temporary qbs module in early setup via loadBaseModule(), + // there is no proper state yet. + if (!m_product.dependenciesContext) + return; + + for (auto it = stateStack().begin(); it != stateStack().end(); ++it) { + Item *itemToCheck = moduleItem; + if (it->loadingItem != itemToCheck) { + if (!productDep) + continue; + itemToCheck = productDep->item; + } + if (it->loadingItem != itemToCheck) + continue; + ErrorInfo e; + e.append(Tr::tr("Cyclic dependencies detected:")); + while (true) { + e.append(it->loadingItemOrigin.name.toString(), + it->loadingItemOrigin.location()); + if (it->loadingItem->type() == ItemType::ModuleInstance) { + createNonPresentModule(m_loaderState.itemPool(), + it->loadingItemOrigin.name.toString(), + QLatin1String("cyclic dependency"), it->loadingItem); + } + if (it == stateStack().begin()) + break; + --it; + } + e.append(dependency.name.toString(), dependency.location()); + throw e; + } + checkForModuleNamePrefixCollision(dependency); +} + +void DependenciesResolver::adjustDependsItemForMultiplexing(Item *dependsItem) +{ + if (m_product.name.startsWith(StringConstants::shadowProductPrefix())) + return; + + Evaluator &evaluator = m_loaderState.evaluator(); + const QString name = evaluator.stringValue(dependsItem, StringConstants::nameProperty()); + const bool productIsMultiplexed = !m_product.multiplexConfigurationId.isEmpty(); + if (name == m_product.name) { + QBS_CHECK(!productIsMultiplexed); // This product must be an aggregator. + return; + } + + bool hasNonMultiplexedDependency = false; + const std::vector<ProductContext *> multiplexedDependencies = m_product.project + ->topLevelProject->productsWithNameAndConstraint(name, [&hasNonMultiplexedDependency] + (const ProductContext &product) { + if (product.multiplexConfigurationId.isEmpty()) { + hasNonMultiplexedDependency = true; + return false; + } + return true; + }); + const bool hasMultiplexedDependencies = !multiplexedDependencies.empty(); + + // These are the allowed cases: + // (1) Normal dependency with no multiplexing whatsoever. + // (2) Both product and dependency are multiplexed. + // (2a) The profiles property is not set, we want to depend on the best + // matching variant. + // (2b) The profiles property is set, we want to depend on all variants + // with a matching profile. + // (3) The product is not multiplexed, but the dependency is. + // (3a) The profiles property is not set, the dependency has an aggregator. + // We want to depend on the aggregator. + // (3b) The profiles property is not set, the dependency does not have an + // aggregator. We want to depend on all the multiplexed variants. + // (3c) The profiles property is set, we want to depend on all variants + // with a matching profile regardless of whether an aggregator exists or not. + // (4) The product is multiplexed, but the dependency is not. We don't have to adapt + // any Depends items. + // (1) and (4) + if (!hasMultiplexedDependencies) + return; + + bool profilesPropertyIsSet; + const QStringList profiles = evaluator.stringListValue( + dependsItem, StringConstants::profilesProperty(), &profilesPropertyIsSet); + + // (3a) + if (!productIsMultiplexed && hasNonMultiplexedDependency && !profilesPropertyIsSet) + return; + + static const auto multiplexConfigurationIntersects = [](const QVariantMap &lhs, + const QVariantMap &rhs) { + QBS_CHECK(!lhs.isEmpty() && !rhs.isEmpty()); + for (auto lhsProperty = lhs.constBegin(); lhsProperty != lhs.constEnd(); lhsProperty++) { + const auto rhsProperty = rhs.find(lhsProperty.key()); + const bool isCommonProperty = rhsProperty != rhs.constEnd(); + if (isCommonProperty && !qVariantsEqual(lhsProperty.value(), rhsProperty.value())) + return false; + } + return true; + }; + + QStringList multiplexIds; + const auto productMultiplexConfig = m_loaderState.topLevelProject().multiplexConfiguration( + m_product.multiplexConfigurationId); + + for (const ProductContext *dependency : multiplexedDependencies) { + if (productIsMultiplexed && !profilesPropertyIsSet) { // 2a + if (dependency->multiplexConfigurationId == m_product.multiplexConfigurationId) { + const ValuePtr &multiplexId = m_product.item->property( + StringConstants::multiplexConfigurationIdProperty()); + dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), + multiplexId); + return; + + } + // Otherwise collect partial matches and decide later + const auto dependencyMultiplexConfig = m_loaderState.topLevelProject() + .multiplexConfiguration(dependency->multiplexConfigurationId); + + if (multiplexConfigurationIntersects(dependencyMultiplexConfig, productMultiplexConfig)) + multiplexIds << dependency->multiplexConfigurationId; + } else { + // (2b), (3b) or (3c) + const bool profileMatch = !profilesPropertyIsSet || profiles.empty() + || profiles.contains(dependency->profileName); + if (profileMatch) + multiplexIds << dependency->multiplexConfigurationId; + } + } + if (multiplexIds.empty()) { + const QString productName = m_product.displayName(); + throw ErrorInfo(Tr::tr("Dependency from product '%1' to product '%2' not fulfilled. " + "There are no eligible multiplex candidates.").arg(productName, + name), + dependsItem->location()); + } + + // In case of (2a), at most 1 match is allowed + if (productIsMultiplexed && !profilesPropertyIsSet && multiplexIds.size() > 1) { + QStringList candidateNames; + for (const auto &id : std::as_const(multiplexIds)) + candidateNames << fullProductDisplayName(name, id); + throw ErrorInfo( + Tr::tr("Dependency from product '%1' to product '%2' is ambiguous. " + "Eligible multiplex candidates: %3.").arg( + m_product.displayName(), name, candidateNames.join(QLatin1String(", "))), + dependsItem->location()); + } + + dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), + VariantValue::create(multiplexIds)); + +} + +std::optional<EvaluatedDependsItem> DependenciesResolver::evaluateDependsItem(Item *item) +{ + Evaluator &evaluator = m_loaderState.evaluator(); + if (!m_product.project->topLevelProject->checkItemCondition(item, evaluator)) { + qCDebug(lcModuleLoader) << "Depends item disabled, ignoring."; + return {}; + } + + const QString name = evaluator.stringValue(item, StringConstants::nameProperty()); + if (name == StringConstants::qbsModule()) // Redundant + return {}; + + bool submodulesPropertySet; + const QStringList submodules = evaluator.stringListValue( + item, StringConstants::submodulesProperty(), &submodulesPropertySet); + if (submodules.empty() && submodulesPropertySet) { + qCDebug(lcModuleLoader) << "Ignoring Depends item with empty submodules list."; + return {}; + } + if (Q_UNLIKELY(submodules.size() > 1 && !item->id().isEmpty())) { + QString msg = Tr::tr("A Depends item with more than one module cannot have an id."); + throw ErrorInfo(msg, item->location()); + } + bool productTypesWasSet; + const QStringList productTypes = evaluator.stringListValue( + item, StringConstants::productTypesProperty(), &productTypesWasSet); + if (!name.isEmpty() && !productTypes.isEmpty()) { + throw ErrorInfo(Tr::tr("The name and productTypes are mutually exclusive " + "in a Depends item."), item->location()); + } + if (productTypes.isEmpty() && productTypesWasSet) { + qCDebug(lcModuleLoader) << "Ignoring Depends item with empty productTypes list."; + return {}; + } + if (name.isEmpty() && !productTypesWasSet) { + throw ErrorInfo(Tr::tr("Either name or productTypes must be set in a Depends item"), + item->location()); + } + + bool profilesPropertyWasSet = false; + std::optional<QStringList> profiles; + bool required = true; + if (productTypes.isEmpty()) { + const QStringList profileList = evaluator.stringListValue( + item, StringConstants::profilesProperty(), &profilesPropertyWasSet); + if (profilesPropertyWasSet) + profiles.emplace(profileList); + required = evaluator.boolValue(item, StringConstants::requiredProperty()); + } + const Version minVersion = Version::fromString( + evaluator.stringValue(item, StringConstants::versionAtLeastProperty())); + const Version maxVersion = Version::fromString( + evaluator.stringValue(item, StringConstants::versionBelowProperty())); + const bool limitToSubProject = evaluator.boolValue( + item, StringConstants::limitToSubProjectProperty()); + const QStringList multiplexIds = evaluator.stringListValue( + item, StringConstants::multiplexConfigurationIdsProperty()); + adjustParametersScopes(item, item); + forwardParameterDeclarations(item, m_product.item->modules()); + const QVariantMap parameters = extractParameters(item); + + const FileTags productTypeTags = FileTags::fromStringList(productTypes); + if (!productTypeTags.empty()) + m_product.bulkDependencies.emplace_back(productTypeTags, item->location()); + return EvaluatedDependsItem{ + item, + QualifiedId::fromString(name), + submodules, + productTypeTags, + multiplexIds, + profiles, + {minVersion, maxVersion}, + parameters, + limitToSubProject, + required}; +} + +// Potentially multiplexes a dependency along Depends.productTypes, Depends.subModules and +// Depends.profiles, as well as internally set up multiplexing axes. +// Each entry in the resulting queue corresponds to exactly one product or module to pull in. +std::queue<FullyResolvedDependsItem> +DependenciesResolver::multiplexDependency(const EvaluatedDependsItem &dependency) +{ + std::queue<FullyResolvedDependsItem> dependencies; + if (!dependency.productTypes.empty()) { + const auto constraint = [&](const ProductContext &product) { + return &product != &m_product && product.name != m_product.name + && (!dependency.limitToSubProject || haveSameSubProject(m_product, product)); + }; + const std::vector<ProductContext *> matchingProducts = m_product.project->topLevelProject + ->productsWithTypeAndConstraint(dependency.productTypes, constraint); + if (matchingProducts.empty()) { + qCDebug(lcModuleLoader) << "Depends.productTypes does not match anything." + << dependency.item->location(); + return {}; + } + for (ProductContext * const match : matchingProducts) + dependencies.emplace(match, dependency); + return dependencies; + } + + const QStringList profiles = dependency.profiles && !dependency.profiles->isEmpty() + ? *dependency.profiles : QStringList(QString()); + const QStringList multiplexIds = !dependency.multiplexIds.isEmpty() + ? dependency.multiplexIds : QStringList(QString()); + const QStringList subModules = !dependency.subModules.isEmpty() + ? dependency.subModules : QStringList(QString()); + for (const QString &profile : profiles) { + for (const QString &multiplexId : multiplexIds) { + for (const QString &subModule : subModules) { + QualifiedId name = dependency.name; + if (!subModule.isEmpty()) + name << subModule.split(QLatin1Char('.')); + dependencies.emplace(dependency, name, profile, multiplexId); + } + } + } + return dependencies; +} + +QVariantMap DependenciesResolver::extractParameters(Item *dependsItem) const +{ + try { + QVariantMap result; + const auto &properties = dependsItem->properties(); + Evaluator &evaluator = m_loaderState.evaluator(); + for (auto it = properties.begin(); it != properties.end(); ++it) { + if (it.value()->type() != Value::ItemValueType) + continue; + const JSValue sv = evaluator.scriptValue( + std::static_pointer_cast<ItemValue>(it.value())->item()); + result.insert(it.key(), safeToVariant(evaluator.engine()->context(), sv)); + } + return result; + } catch (const ErrorInfo &exception) { + auto ei = exception; + ei.prepend(Tr::tr("Error in dependency parameter."), dependsItem->location()); + throw ei; + } +} + +void DependenciesResolver::forwardParameterDeclarations(const Item *dependsItem, + const Item::Modules &modules) +{ + for (auto it = dependsItem->properties().begin(); it != dependsItem->properties().end(); ++it) { + if (it.value()->type() != Value::ItemValueType) + continue; + forwardParameterDeclarations(it.key(), + std::static_pointer_cast<ItemValue>(it.value())->item(), + modules); + } +} + +void DependenciesResolver::forwardParameterDeclarations( + const QualifiedId &moduleName, Item *item, const Item::Modules &modules) +{ + auto it = std::find_if(modules.begin(), modules.end(), [&moduleName] (const Item::Module &m) { + return m.name == moduleName; + }); + if (it != modules.end()) { + item->setPropertyDeclarations(m_loaderState.topLevelProject().parameterDeclarations( + it->item->rootPrototype())); + } else { + for (auto it = item->properties().begin(); it != item->properties().end(); ++it) { + if (it.value()->type() != Value::ItemValueType) + continue; + forwardParameterDeclarations(QualifiedId(moduleName) << it.key(), + std::static_pointer_cast<ItemValue>(it.value())->item(), + modules); + } + } +} + +std::list<DependenciesResolvingState> &DependenciesResolver::stateStack() +{ + QBS_CHECK(m_product.dependenciesContext); + return static_cast<DependenciesContextImpl *>(m_product.dependenciesContext.get())->stateStack; +} + +void DependenciesResolver::checkForModuleNamePrefixCollision( + const FullyResolvedDependsItem &dependency) +{ + if (!m_product.item) + return; + + for (const Item::Module &m : m_product.item->modules()) { + if (m.name.length() == dependency.name.length()) + continue; + + QualifiedId shortName; + QualifiedId longName; + if (m.name.length() < dependency.name.length()) { + shortName = m.name; + longName = dependency.name; + } else { + shortName = dependency.name; + longName = m.name; + } + const auto isPrefix = [&] { + for (int i = 0; i < shortName.length(); ++i) { + if (shortName.at(i) != longName.at(i)) + return false; + } + return true; + }; + if (!isPrefix()) + continue; + + throw ErrorInfo(Tr::tr("The name of module '%1' is a prefix of the name of module '%2', " + "which is not allowed") + .arg(shortName.toString(), longName.toString()), dependency.location()); + } +} + +Item::Module DependenciesResolver::createModule( + const FullyResolvedDependsItem &dependency, Item *item, ProductContext *productDep) +{ + Item::Module m; + m.item = item; + m.product = productDep; + m.name = dependency.name; + m.required = dependency.requiredLocally; + m.versionRange = dependency.versionRange; + return m; +} + +FullyResolvedDependsItem::FullyResolvedDependsItem( + ProductContext *product, const EvaluatedDependsItem &dependency) + : product(product) + , item(dependency.item) + , name(product->name) + , versionRange(dependency.versionRange) + , parameters(dependency.parameters) + , checkProduct(false) +{} + +FullyResolvedDependsItem FullyResolvedDependsItem::makeBaseDependency() +{ + FullyResolvedDependsItem item; + item.name = StringConstants::qbsModule(); + return item; +} + +FullyResolvedDependsItem::FullyResolvedDependsItem( + const EvaluatedDependsItem &dependency, QualifiedId name, QString profile, QString multiplexId) + : item(dependency.item) + , name(std::move(name)) + , profile(std::move(profile)) + , multiplexId(std::move(multiplexId)) + , versionRange(dependency.versionRange) + , parameters(dependency.parameters) + , limitToSubProject(dependency.limitToSubProject) + , requiredLocally(dependency.requiredLocally) + , requiredGlobally(dependency.requiredGlobally) +{} + +QString FullyResolvedDependsItem::id() const +{ + if (!item) { + QBS_CHECK(name.toString() == StringConstants::qbsModule()); + return {}; + } + return item->id(); +} + +CodeLocation FullyResolvedDependsItem::location() const +{ + if (!item) { + QBS_CHECK(name.toString() == StringConstants::qbsModule()); + return {}; + } + return item->location(); +} + +QString FullyResolvedDependsItem::displayName() const +{ + return fullProductDisplayName(name.toString(), multiplexId); +} + +bool haveSameSubProject(const ProductContext &p1, const ProductContext &p2) +{ + for (const Item *otherParent = p2.item->parent(); otherParent; + otherParent = otherParent->parent()) { + if (otherParent == p1.item->parent()) + return true; + } + return false; +} + +QVariantMap safeToVariant(JSContext *ctx, const JSValue &v) +{ + QVariantMap result; + handleJsProperties(ctx, v, [&](const JSAtom &prop, const JSPropertyDescriptor &desc) { + const JSValue u = desc.value; + if (JS_IsError(ctx, u)) + throw ErrorInfo(getJsString(ctx, u)); + const QString name = getJsString(ctx, prop); + result[name] = (JS_IsObject(u) && !JS_IsArray(ctx, u) && !JS_IsRegExp(ctx, u)) + ? safeToVariant(ctx, u) : getJsVariant(ctx, u); + }); + return result; +} + +DependenciesContextImpl::DependenciesContextImpl(ProductContext &product, LoaderState &loaderState) + : m_product(product) +{ + setSearchPathsForProduct(loaderState); + + // Initialize the state with the direct Depends items of the product item. + DependenciesResolvingState newState{product.item,}; + for (Item * const child : product.item->children()) { + if (child->type() == ItemType::Depends) + newState.pendingDependsItems.push(child); + } + stateStack.push_front(std::move(newState)); + stateStack.front().pendingResolvedDependencies.push( + FullyResolvedDependsItem::makeBaseDependency()); +} + +std::pair<ProductDependency, ProductContext *> DependenciesContextImpl::pendingDependency() const +{ + QBS_CHECK(!stateStack.empty()); + if (const auto ¤tDependsItem = stateStack.front().currentDependsItem; + currentDependsItem && !currentDependsItem->productTypes.empty()) { + qCDebug(lcLoaderScheduling) << "product" << m_product.displayName() + << "to be delayed because of bulk dependency"; + return {ProductDependency::Bulk, nullptr}; + } + if (!stateStack.front().pendingResolvedDependencies.empty()) { + if (ProductContext * const dep = stateStack.front().pendingResolvedDependencies + .front().product) { + if (m_product.project->topLevelProject->isProductQueuedForHandling(*dep)) { + qCDebug(lcLoaderScheduling) << "product" << m_product.displayName() + << "to be delayed because of dependency " + "to unfinished product" << dep->displayName(); + return {ProductDependency::Single, dep}; + } else { + qCDebug(lcLoaderScheduling) << "product" << m_product.displayName() + << "to be re-scheduled, as dependency " + << dep->displayName() + << "appears to have finished in the meantime"; + return {ProductDependency::None, dep}; + } + } + } + return {ProductDependency::None, nullptr}; +} + +void DependenciesContextImpl::setSearchPathsForProduct(LoaderState &loaderState) +{ + QBS_CHECK(m_product.searchPaths.isEmpty()); + + m_product.searchPaths = loaderState.itemReader().readExtraSearchPaths(m_product.item); + Settings settings(loaderState.parameters().settingsDirectory()); + const QStringList prefsSearchPaths = Preferences(&settings, m_product.profileModuleProperties) + .searchPaths(); + const QStringList ¤tSearchPaths = loaderState.itemReader().allSearchPaths(); + for (const QString &p : prefsSearchPaths) { + if (!currentSearchPaths.contains(p) && FileInfo(p).exists()) + m_product.searchPaths << p; + } +} + +} // namespace +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/dependenciesresolver.h b/src/lib/corelib/loader/dependenciesresolver.h new file mode 100644 index 000000000..48502adb7 --- /dev/null +++ b/src/lib/corelib/loader/dependenciesresolver.h @@ -0,0 +1,57 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs::Internal { +class Item; +class LoaderState; +class ProductContext; +enum class Deferral; + +// Collects the products' dependencies and builds the list of modules from them. +// Actual loading of module files is offloaded to ModuleLoader. +void resolveDependencies(ProductContext &product, Deferral deferral, LoaderState &loaderState); + +// Note: This function is never called for regular loading of the base module into a product, +// but only for the special cases of loading the dummy base module into a project +// and temporarily providing a base module for product multiplexing. +Item *loadBaseModule(ProductContext &product, Item *item, LoaderState &loaderState); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/groupshandler.cpp b/src/lib/corelib/loader/groupshandler.cpp new file mode 100644 index 000000000..3dbfad352 --- /dev/null +++ b/src/lib/corelib/loader/groupshandler.cpp @@ -0,0 +1,300 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "groupshandler.h" + +#include "loaderutils.h" +#include "moduleinstantiator.h" + +#include <language/evaluator.h> +#include <language/item.h> +#include <language/value.h> +#include <logging/translator.h> +#include <tools/profiling.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +namespace qbs::Internal { +class GroupsHandler +{ +public: + GroupsHandler(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + + void run(); + +private: + void gatherAssignedProperties(ItemValue *iv, const QualifiedId &prefix, + QualifiedIdSet &properties); + void markModuleTargetGroups(Item *group, const Item::Module &module); + void moveGroupsFromModuleToProduct(const Item::Module &module); + void moveGroupsFromModulesToProduct(); + void propagateModulesFromParent(Item *group); + void handleGroup(Item *group); + void adjustScopesInGroupModuleInstances(Item *groupItem, const Item::Module &module); + QualifiedIdSet gatherModulePropertiesSetInGroup(const Item *group); + + ProductContext &m_product; + LoaderState &m_loaderState; +}; + +void setupGroups(ProductContext &product, LoaderState &loaderState) +{ + GroupsHandler(product, loaderState).run(); +} + +void GroupsHandler::run() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.groupsSetup : nullptr); + + moveGroupsFromModulesToProduct(); + for (Item * const child : m_product.item->children()) { + if (child->type() == ItemType::Group) + handleGroup(child); + } +} + +void GroupsHandler::gatherAssignedProperties(ItemValue *iv, const QualifiedId &prefix, + QualifiedIdSet &properties) +{ + const Item::PropertyMap &props = iv->item()->properties(); + for (auto it = props.cbegin(); it != props.cend(); ++it) { + switch (it.value()->type()) { + case Value::JSSourceValueType: + properties << (QualifiedId(prefix) << it.key()); + break; + case Value::ItemValueType: + if (iv->item()->type() == ItemType::ModulePrefix) { + gatherAssignedProperties(std::static_pointer_cast<ItemValue>(it.value()).get(), + QualifiedId(prefix) << it.key(), properties); + } + break; + default: + break; + } + } +} + +void GroupsHandler::markModuleTargetGroups(Item *group, const Item::Module &module) +{ + QBS_CHECK(group->type() == ItemType::Group); + if (m_loaderState.evaluator().boolValue(group, StringConstants::filesAreTargetsProperty())) { + group->setProperty(StringConstants::modulePropertyInternal(), + VariantValue::create(module.name.toString())); + } + for (Item * const child : group->children()) + markModuleTargetGroups(child, module); +} + +void GroupsHandler::moveGroupsFromModuleToProduct(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + for (auto it = module.item->children().begin(); it != module.item->children().end();) { + Item * const child = *it; + if (child->type() != ItemType::Group) { + ++it; + continue; + } + + Item * const scope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); + scope->setProperties(module.item->properties()); + scope->setScope(m_product.scope); + child->setScope(scope); + setScopeForDescendants(child, scope); + + Item::addChild(m_product.item, child); + markModuleTargetGroups(child, module); + it = module.item->children().erase(it); + } +} + +void GroupsHandler::moveGroupsFromModulesToProduct() +{ + for (const Item::Module &module : m_product.item->modules()) + moveGroupsFromModuleToProduct(module); +} + +// TODO: I don't completely understand this function, and I suspect we do both too much +// and too little here. In particular, I'm not sure why we should even have to do anything +// with groups that don't attach properties. +// Set aside a day or two at some point to fully grasp all the details and rewrite accordingly. +void GroupsHandler::propagateModulesFromParent(Item *group) +{ + QBS_CHECK(group->type() == ItemType::Group); + QHash<QualifiedId, Item *> moduleInstancesForGroup; + + // Step 1: "Instantiate" the product's modules for the group. + for (Item::Module m : group->parent()->modules()) { + Item * const targetItem = retrieveModuleInstanceItem(group, m.name, m_loaderState); + QBS_CHECK(targetItem->type() == ItemType::ModuleInstancePlaceholder); + targetItem->setPrototype(m.item); + + Item * const moduleScope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); + moduleScope->setFile(group->file()); + moduleScope->setProperties(m.item->scope()->properties()); // "project", "product", ids + moduleScope->setScope(group); + targetItem->setScope(moduleScope); + + targetItem->setFile(m.item->file()); + + // "parent" should point to the group/artifact parent + targetItem->setParent(group->parent()); + + targetItem->setOuterItem(m.item); + + m.item = targetItem; + group->addModule(m); + moduleInstancesForGroup.insert(m.name, targetItem); + } + + // Step 2: Make the inter-module references point to the instances created in step 1. + for (const Item::Module &module : group->modules()) { + Item::Modules adaptedModules; + const Item::Modules &oldModules = module.item->prototype()->modules(); + for (Item::Module depMod : oldModules) { + depMod.item = moduleInstancesForGroup.value(depMod.name); + adaptedModules << depMod; + if (depMod.name.front() == module.name.front()) + continue; + const ItemValuePtr &modulePrefix = group->itemProperty(depMod.name.front(), + m_loaderState.itemPool()); + QBS_CHECK(modulePrefix); + module.item->setProperty(depMod.name.front(), modulePrefix); + } + module.item->setModules(adaptedModules); + } + + const QualifiedIdSet &propsSetInGroup = gatherModulePropertiesSetInGroup(group); + if (propsSetInGroup.empty()) + return; + m_product.modulePropertiesSetInGroups.insert(std::make_pair(group, propsSetInGroup)); + + // Step 3: Adapt scopes in values. This is potentially necessary if module properties + // get assigned on the group level. + for (const Item::Module &module : group->modules()) + adjustScopesInGroupModuleInstances(group, module); +} + +void GroupsHandler::handleGroup(Item *group) +{ + propagateModulesFromParent(group); + m_loaderState.topLevelProject().checkItemCondition(group, m_loaderState.evaluator()); + for (Item * const child : group->children()) { + if (child->type() == ItemType::Group) + handleGroup(child); + } +} + +void GroupsHandler::adjustScopesInGroupModuleInstances(Item *groupItem, + const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + + Item *modulePrototype = module.item->rootPrototype(); + QBS_CHECK(modulePrototype->type() == ItemType::Module + || modulePrototype->type() == ItemType::Export); + + const Item::PropertyDeclarationMap &propDecls = modulePrototype->propertyDeclarations(); + for (const auto &decl : propDecls) { + const QString &propName = decl.name(); + + // Nothing gets inherited for module properties assigned directly in the group. + // In particular, setting a list property overwrites the value from the product's + // (or parent group's) instance completely, rather than appending to it + // (concatenation happens via outer.concat()). + ValuePtr propValue = module.item->ownProperty(propName); + if (propValue) { + propValue->setScope(module.item->scope(), {}); + continue; + } + + // Find the nearest prototype instance that has the value assigned. + // The result is either an instance of a parent group (or the parent group's + // parent group and so on) or the instance of the product or the module prototype. + // In the latter case, we don't have to do anything. + const Item *instanceWithProperty = module.item; + do { + instanceWithProperty = instanceWithProperty->prototype(); + QBS_CHECK(instanceWithProperty); + propValue = instanceWithProperty->ownProperty(propName); + } while (!propValue); + QBS_CHECK(propValue); + + if (propValue->type() != Value::JSSourceValueType) + continue; + + if (propValue->scope()) + module.item->setProperty(propName, propValue->clone(m_loaderState.itemPool())); + } + + for (const ValuePtr &prop : module.item->properties()) { + if (prop->type() != Value::JSSourceValueType) { + QBS_CHECK(!prop->next()); + continue; + } + for (ValuePtr v = prop; v; v = v->next()) { + if (!v->scope()) + continue; + for (const Item::Module &module : groupItem->modules()) { + if (v->scope() == module.item->prototype()) { + v->setScope(module.item, {}); + break; + } + } + } + } +} + +QualifiedIdSet GroupsHandler::gatherModulePropertiesSetInGroup(const Item *group) +{ + QualifiedIdSet propsSetInGroup; + const Item::PropertyMap &props = group->properties(); + for (auto it = props.cbegin(); it != props.cend(); ++it) { + if (it.value()->type() == Value::ItemValueType) { + gatherAssignedProperties(std::static_pointer_cast<ItemValue>(it.value()).get(), + QualifiedId(it.key()), propsSetInGroup); + } + } + return propsSetInGroup; +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/groupshandler.h b/src/lib/corelib/loader/groupshandler.h new file mode 100644 index 000000000..5de94ce7c --- /dev/null +++ b/src/lib/corelib/loader/groupshandler.h @@ -0,0 +1,56 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs::Internal { +class LoaderState; +class ProductContext; + +// Sets up Group items for the actual resolving stage. Responsibilities: +// - Moving Group items located in modules over to the product. +// - Identifying groups declaring target artifacts and marking them accordingly. +// - Setting up group-level module instances to ensure proper resolving of per-group module +// properties. +// - As a side effect of the above point, collecting all properties set on the Group level +// to help the ProjectResolver decide which properties need to be re-evaluated at all, +// which is an important optimization (see commit 9cd8653eef). +void setupGroups(ProductContext &product, LoaderState &loaderState); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/itemreader.cpp b/src/lib/corelib/loader/itemreader.cpp new file mode 100644 index 000000000..0638d1af5 --- /dev/null +++ b/src/lib/corelib/loader/itemreader.cpp @@ -0,0 +1,274 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "itemreader.h" + +#include "itemreadervisitorstate.h" +#include "loaderutils.h" + +#include <language/deprecationinfo.h> +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/item.h> +#include <language/value.h> +#include <logging/categories.h> +#include <tools/fileinfo.h> +#include <tools/profiling.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> +#include <tools/stlutils.h> + +#include <QtCore/qfileinfo.h> + +#include <algorithm> + +namespace qbs { +namespace Internal { + +static void makePathsCanonical(QStringList &paths) +{ + Internal::removeIf(paths, [](QString &p) { + p = QFileInfo(p).canonicalFilePath(); + return p.isEmpty(); + }); +} + +ItemReader::ItemReader(LoaderState &loaderState) : m_loaderState(loaderState) {} + +void ItemReader::init() +{ + m_visitorState = std::make_unique<ItemReaderVisitorState>( + m_loaderState.topLevelProject().itemReaderCache(), m_loaderState.logger()); + m_visitorState->setDeprecationWarningMode(m_loaderState.parameters().deprecationWarningMode()); + m_projectFilePath = m_loaderState.parameters().projectFilePath(); + setSearchPaths(m_loaderState.parameters().searchPaths()); + m_elapsedTime = m_loaderState.parameters().logElapsedTime() ? 0 : -1; +} + +ItemReader::~ItemReader() = default; + +void ItemReader::setSearchPaths(const QStringList &searchPaths) +{ + qCDebug(lcModuleLoader) << "initial search paths:" << searchPaths; + m_searchPaths = searchPaths; + makePathsCanonical(m_searchPaths); + m_allSearchPaths.clear(); +} + +void ItemReader::pushExtraSearchPaths(const QStringList &extraSearchPaths) +{ + m_extraSearchPaths.push_back(extraSearchPaths); + makePathsCanonical(m_extraSearchPaths.back()); + m_allSearchPaths.clear(); +} + +void ItemReader::popExtraSearchPaths() +{ + m_extraSearchPaths.pop_back(); + m_allSearchPaths.clear(); +} + +const std::vector<QStringList> &ItemReader::extraSearchPathsStack() const +{ + return m_extraSearchPaths; +} + +void ItemReader::setExtraSearchPathsStack(const std::vector<QStringList> &s) +{ + m_extraSearchPaths = s; + m_allSearchPaths.clear(); +} + +void ItemReader::clearExtraSearchPathsStack() +{ + m_extraSearchPaths.clear(); + m_allSearchPaths.clear(); +} + +const QStringList &ItemReader::allSearchPaths() const +{ + if (m_allSearchPaths.empty()) { + std::for_each(m_extraSearchPaths.crbegin(), m_extraSearchPaths.crend(), + [this] (const QStringList &paths) { + m_allSearchPaths += paths; + }); + m_allSearchPaths += m_searchPaths; + m_allSearchPaths.removeDuplicates(); + } + return m_allSearchPaths; +} + +Item *ItemReader::readFile(const QString &filePath) +{ + AccumulatingTimer readFileTimer(m_elapsedTime != -1 ? &m_elapsedTime : nullptr); + return m_visitorState->readFile(filePath, allSearchPaths(), &m_loaderState.itemPool()); +} + +Item *ItemReader::readFile(const QString &filePath, const CodeLocation &referencingLocation) +{ + try { + return readFile(filePath); + } catch (const ErrorInfo &e) { + if (e.hasLocation()) + throw; + throw ErrorInfo(e.toString(), referencingLocation); + } +} + +void ItemReader::handlePropertyOptions(Item *optionsItem) +{ + Evaluator &evaluator = m_loaderState.evaluator(); + const QString name = evaluator.stringValue(optionsItem, StringConstants::nameProperty()); + if (name.isEmpty()) { + throw ErrorInfo(Tr::tr("PropertyOptions item needs a name property"), + optionsItem->location()); + } + const QString description = evaluator.stringValue( + optionsItem, StringConstants::descriptionProperty()); + const auto removalVersion = Version::fromString( + evaluator.stringValue(optionsItem, StringConstants::removalVersionProperty())); + const auto allowedValues = evaluator.stringListValue( + optionsItem, StringConstants::allowedValuesProperty()); + + PropertyDeclaration decl = optionsItem->parent()->propertyDeclaration(name); + if (!decl.isValid()) { + decl.setName(name); + decl.setType(PropertyDeclaration::Variant); + } + decl.setDescription(description); + if (removalVersion.isValid()) { + DeprecationInfo di(removalVersion, description); + decl.setDeprecationInfo(di); + } + decl.setAllowedValues(allowedValues); + const ValuePtr property = optionsItem->parent()->property(name); + if (!property && !decl.isExpired()) { + throw ErrorInfo(Tr::tr("PropertyOptions item refers to non-existing property '%1'") + .arg(name), optionsItem->location()); + } + if (property && decl.isExpired()) { + ErrorInfo e(Tr::tr("Property '%1' was scheduled for removal in version %2, but " + "is still present.") + .arg(name, removalVersion.toString()), + property->location()); + e.append(Tr::tr("Removal version for '%1' specified here.").arg(name), + optionsItem->location()); + m_visitorState->logger().printWarning(e); + } + optionsItem->parent()->setPropertyDeclaration(name, decl); +} + +void ItemReader::handleAllPropertyOptionsItems(Item *item) +{ + QList<Item *> childItems = item->children(); + auto childIt = childItems.begin(); + while (childIt != childItems.end()) { + Item * const child = *childIt; + if (child->type() == ItemType::PropertyOptions) { + handlePropertyOptions(child); + childIt = childItems.erase(childIt); + } else { + handleAllPropertyOptionsItems(child); + ++childIt; + } + } + item->setChildren(childItems); +} + +Item *ItemReader::setupItemFromFile(const QString &filePath, const CodeLocation &referencingLocation) +{ + Item *item = readFile(filePath, referencingLocation); + + // This is technically not needed, because files are only set up once and then served + // from a cache. But it simplifies the checks in item.cpp if we require the locking invariant + // to always hold. + std::unique_ptr<ModuleItemLocker> locker; + if (item->type() == ItemType::Module) + locker = std::make_unique<ModuleItemLocker>(*item); + + handleAllPropertyOptionsItems(item); + return item; +} + +Item *ItemReader::wrapInProjectIfNecessary(Item *item) +{ + if (item->type() == ItemType::Project) + return item; + Item *prj = Item::create(&m_loaderState.itemPool(), ItemType::Project); + Item::addChild(prj, item); + prj->setFile(item->file()); + prj->setLocation(item->location()); + prj->setupForBuiltinType(m_loaderState.parameters().deprecationWarningMode(), + m_visitorState->logger()); + return prj; +} + +QStringList ItemReader::readExtraSearchPaths(Item *item, bool *wasSet) +{ + QStringList result; + const QStringList paths = m_loaderState.evaluator().stringListValue( + item, StringConstants::qbsSearchPathsProperty(), wasSet); + const JSSourceValueConstPtr prop = item->sourceProperty( + StringConstants::qbsSearchPathsProperty()); + + // Value can come from within a project file or as an overridden value from the user + // (e.g command line). + const QString basePath = FileInfo::path(prop ? prop->file()->filePath() + : m_projectFilePath); + for (const QString &path : paths) + result += FileInfo::resolvePath(basePath, path); + return result; +} + +SearchPathsManager::SearchPathsManager(ItemReader &itemReader, const QStringList &extraSearchPaths) + : m_itemReader(itemReader), + m_oldSize(itemReader.extraSearchPathsStack().size()) +{ + if (!extraSearchPaths.isEmpty()) + m_itemReader.pushExtraSearchPaths(extraSearchPaths); +} + +SearchPathsManager::~SearchPathsManager() +{ + while (m_itemReader.extraSearchPathsStack().size() > m_oldSize) + m_itemReader.popExtraSearchPaths(); +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/itemreader.h b/src/lib/corelib/loader/itemreader.h new file mode 100644 index 000000000..e444db16c --- /dev/null +++ b/src/lib/corelib/loader/itemreader.h @@ -0,0 +1,121 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QBS_ITEMREADER_H +#define QBS_ITEMREADER_H + +#include <logging/logger.h> +#include <tools/deprecationwarningmode.h> +#include <tools/set.h> + +#include <QtCore/qstringlist.h> + +#include <memory> + +namespace qbs { +class SetupProjectParameters; +namespace Internal { +class Evaluator; +class Item; +class ItemPool; +class ItemReaderVisitorState; +class LoaderState; + +/* + * Reads a qbs file and creates a tree of Item objects. + * + * In this stage the following steps are performed: + * - The QML/JS parser creates the AST. + * - The AST is converted to a tree of Item objects. + * + * This class is also responsible for the QMLish inheritance semantics. + */ +class ItemReader +{ +public: + ItemReader(LoaderState &loaderState); + ~ItemReader(); + void init(); + + void pushExtraSearchPaths(const QStringList &extraSearchPaths); + void popExtraSearchPaths(); + const std::vector<QStringList> &extraSearchPathsStack() const; + void setExtraSearchPathsStack(const std::vector<QStringList> &s); + void clearExtraSearchPathsStack(); + const QStringList &allSearchPaths() const; + + // Parses a file, creates an item for it, generates PropertyDeclarations from + // PropertyOptions items and removes said items from the item tree. + Item *setupItemFromFile(const QString &filePath, const CodeLocation &referencingLocation); + + Item *wrapInProjectIfNecessary(Item *item); + QStringList readExtraSearchPaths(Item *item, bool *wasSet = nullptr); + + qint64 elapsedTime() const { return m_elapsedTime; } + +private: + void setSearchPaths(const QStringList &searchPaths); + Item *readFile(const QString &filePath); + Item *readFile(const QString &filePath, const CodeLocation &referencingLocation); + void handlePropertyOptions(Item *optionsItem); + void handleAllPropertyOptionsItems(Item *item); + + LoaderState &m_loaderState; + QStringList m_searchPaths; + std::vector<QStringList> m_extraSearchPaths; + mutable QStringList m_allSearchPaths; + std::unique_ptr<ItemReaderVisitorState> m_visitorState; + QString m_projectFilePath; + qint64 m_elapsedTime = -1; +}; + +class SearchPathsManager { +public: + SearchPathsManager(ItemReader &itemReader, const QStringList &extraSearchPaths = {}); + ~SearchPathsManager(); + +private: + ItemReader &m_itemReader; + size_t m_oldSize{0}; +}; + +} // namespace Internal +} // namespace qbs + +#endif // QBS_ITEMREADER_H diff --git a/src/lib/corelib/loader/itemreaderastvisitor.cpp b/src/lib/corelib/loader/itemreaderastvisitor.cpp new file mode 100644 index 000000000..c3e6b9b89 --- /dev/null +++ b/src/lib/corelib/loader/itemreaderastvisitor.cpp @@ -0,0 +1,405 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "itemreaderastvisitor.h" + +#include "astimportshandler.h" +#include "astpropertiesitemhandler.h" +#include "itemreadervisitorstate.h" + +#include <api/languageinfo.h> +#include <jsextensions/jsextensions.h> +#include <language/asttools.h> +#include <language/builtindeclarations.h> +#include <language/filecontext.h> +#include <language/identifiersearch.h> +#include <language/item.h> +#include <language/value.h> +#include <parser/qmljsast_p.h> +#include <tools/codelocation.h> +#include <tools/error.h> +#include <tools/qbsassert.h> +#include <tools/qttools.h> +#include <tools/stringconstants.h> +#include <logging/translator.h> + +#include <algorithm> + +using namespace QbsQmlJS; + +namespace qbs { +namespace Internal { + +ItemReaderASTVisitor::ItemReaderASTVisitor(ItemReaderVisitorState &visitorState, + FileContextPtr file, ItemPool *itemPool, Logger &logger) + : m_visitorState(visitorState) + , m_file(std::move(file)) + , m_itemPool(itemPool) + , m_logger(logger) +{ +} + +ItemReaderASTVisitor::~ItemReaderASTVisitor() = default; + +bool ItemReaderASTVisitor::visit(AST::UiProgram *uiProgram) +{ + ASTImportsHandler importsHandler(m_visitorState, m_logger, m_file); + importsHandler.handleImports(uiProgram->imports); + m_typeNameToFile = importsHandler.typeNameFileMap(); + return true; +} + +static ItemValuePtr findItemProperty(const Item *container, const Item *item) +{ + ItemValuePtr itemValue; + const auto &srcprops = container->properties(); + auto it = std::find_if(srcprops.begin(), srcprops.end(), [item] (const ValuePtr &v) { + return v->type() == Value::ItemValueType + && std::static_pointer_cast<ItemValue>(v)->item() == item; + }); + if (it != srcprops.end()) + itemValue = std::static_pointer_cast<ItemValue>(it.value()); + return itemValue; +} + +bool ItemReaderASTVisitor::visit(AST::UiObjectDefinition *ast) +{ + const QString typeName = ast->qualifiedTypeNameId->name.toString(); + const CodeLocation itemLocation = toCodeLocation(ast->qualifiedTypeNameId->identifierToken); + const Item *baseItem = nullptr; + Item *mostDerivingItem = nullptr; + + Item *item = Item::create(m_itemPool, ItemType::Unknown); + item->setFile(m_file); + item->setLocation(itemLocation); + const AST::SourceLocation &endLoc = ast->lastSourceLocation(); + item->setEndPosition(CodePosition(endLoc.startLine, endLoc.startColumn)); + + // Inheritance resolving, part 1: Find out our actual type name (needed for setting + // up children and alternatives). + const QStringList fullTypeName = toStringList(ast->qualifiedTypeNameId); + const QString baseTypeFileName = m_typeNameToFile.value(fullTypeName); + ItemType itemType; + if (!baseTypeFileName.isEmpty()) { + const bool isMostDerivingItem = (m_visitorState.mostDerivingItem() == nullptr); + if (isMostDerivingItem) + m_visitorState.setMostDerivingItem(item); + mostDerivingItem = m_visitorState.mostDerivingItem(); + baseItem = m_visitorState.readFile(baseTypeFileName, m_file->searchPaths(), m_itemPool); + if (isMostDerivingItem) + m_visitorState.setMostDerivingItem(nullptr); + QBS_CHECK(baseItem->type() <= ItemType::LastActualItem); + itemType = baseItem->type(); + } else { + if (fullTypeName.size() > 1) { + throw ErrorInfo(Tr::tr("Invalid item '%1'. Did you mean to set a module property?") + .arg(fullTypeName.join(QLatin1Char('.'))), itemLocation); + } + itemType = BuiltinDeclarations::instance().typeForName(typeName, itemLocation); + checkDeprecationStatus(itemType, typeName, itemLocation); + if (itemType == ItemType::Properties && m_item && m_item->type() == ItemType::SubProject) + itemType = ItemType::PropertiesInSubProject; + } + + item->m_type = itemType; + + if (m_item) { + Item::addChild(m_item, item); // Add this item to the children of the parent item. + } else { + m_item = item; // This is the root item. + if (itemType == ItemType::Module) { + QBS_CHECK(!m_moduleItemLocker); + m_moduleItemLocker = std::make_unique<ModuleItemLocker>(*m_item); + } + } + + if (ast->initializer) { + Item *mdi = m_visitorState.mostDerivingItem(); + m_visitorState.setMostDerivingItem(nullptr); + qSwap(m_item, item); + const ItemType oldInstanceItemType = m_instanceItemType; + if (itemType == ItemType::Parameters || itemType == ItemType::Depends) + m_instanceItemType = ItemType::ModuleParameters; + ast->initializer->accept(this); + m_instanceItemType = oldInstanceItemType; + qSwap(m_item, item); + m_visitorState.setMostDerivingItem(mdi); + } + + ASTPropertiesItemHandler(item, *m_itemPool).handlePropertiesItems(); + + // Inheritance resolving, part 2 (depends on alternatives having been set up). + if (baseItem) { + inheritItem(item, baseItem); + if (baseItem->file()->idScope()) { + // Make ids from the derived file visible in the base file. + // ### Do we want to turn off this feature? It's QMLish but kind of strange. + item->file()->ensureIdScope(m_itemPool); + baseItem->file()->idScope()->setPrototype(item->file()->idScope()); + + // Replace the base item with the most deriving item. + ItemValuePtr baseItemIdValue = findItemProperty(baseItem->file()->idScope(), baseItem); + if (baseItemIdValue) + baseItemIdValue->setItem(mostDerivingItem); + } + } else { + // Only the item at the top of the inheritance chain is a built-in item. + // We cannot do this in "part 1", because then the visitor would complain about duplicate + // bindings. + item->setupForBuiltinType(m_visitorState.deprecationWarningMode(), m_logger); + } + + return false; +} + +void ItemReaderASTVisitor::checkDuplicateBinding(Item *item, const QStringList &bindingName, + const AST::SourceLocation &sourceLocation) +{ + if (Q_UNLIKELY(item->hasOwnProperty(bindingName.last()))) { + QString msg = Tr::tr("Duplicate binding for '%1'"); + throw ErrorInfo(msg.arg(bindingName.join(QLatin1Char('.'))), + toCodeLocation(sourceLocation)); + } +} + +bool ItemReaderASTVisitor::visit(AST::UiPublicMember *ast) +{ + PropertyDeclaration p; + if (Q_UNLIKELY(ast->name.isEmpty())) + throw ErrorInfo(Tr::tr("public member without name")); + if (Q_UNLIKELY(ast->memberType.isEmpty())) + throw ErrorInfo(Tr::tr("public member without type")); + if (Q_UNLIKELY(ast->type == AST::UiPublicMember::Signal)) + throw ErrorInfo(Tr::tr("public member with signal type not supported")); + p.setName(ast->name.toString()); + p.setType(PropertyDeclaration::propertyTypeFromString(ast->memberType.toString())); + if (p.type() == PropertyDeclaration::UnknownType) { + throw ErrorInfo(Tr::tr("Unknown type '%1' in property declaration.") + .arg(ast->memberType.toString()), toCodeLocation(ast->typeToken)); + } + if (Q_UNLIKELY(!ast->typeModifier.isEmpty())) { + throw ErrorInfo(Tr::tr("public member with type modifier '%1' not supported").arg( + ast->typeModifier.toString())); + } + if (ast->isReadonlyMember) + p.setFlags(PropertyDeclaration::ReadOnlyFlag); + + m_item->m_propertyDeclarations.insert(p.name(), p); + + const JSSourceValuePtr value = JSSourceValue::create(); + value->setFile(m_file); + if (ast->statement) { + handleBindingRhs(ast->statement, value); + const QStringList bindingName(p.name()); + checkDuplicateBinding(m_item, bindingName, ast->colonToken); + } + + m_item->setProperty(p.name(), value); + return false; +} + +bool ItemReaderASTVisitor::visit(AST::UiScriptBinding *ast) +{ + QBS_CHECK(ast->qualifiedId); + QBS_CHECK(!ast->qualifiedId->name.isEmpty()); + + const QStringList bindingName = toStringList(ast->qualifiedId); + + if (bindingName.length() == 1 && bindingName.front() == QStringLiteral("id")) { + const auto * const expStmt = AST::cast<AST::ExpressionStatement *>(ast->statement); + if (Q_UNLIKELY(!expStmt)) + throw ErrorInfo(Tr::tr("id: must be followed by identifier")); + const auto * const idExp = AST::cast<AST::IdentifierExpression *>(expStmt->expression); + if (Q_UNLIKELY(!idExp || idExp->name.isEmpty())) + throw ErrorInfo(Tr::tr("id: must be followed by identifier")); + if (m_item->type() == ItemType::Module) + throw ErrorInfo(Tr::tr("Module items cannot have an id property.")); + m_item->m_id = idExp->name.toString(); + m_file->ensureIdScope(m_itemPool); + ItemValueConstPtr existingId = m_file->idScope()->itemProperty(m_item->id(), *m_itemPool); + if (existingId) { + ErrorInfo e(Tr::tr("The id '%1' is not unique.").arg(m_item->id())); + e.append(Tr::tr("First occurrence is here."), existingId->item()->location()); + e.append(Tr::tr("Next occurrence is here."), m_item->location()); + throw e; + } + m_file->idScope()->setProperty(m_item->id(), ItemValue::create(m_item)); + return false; + } + + const JSSourceValuePtr value = JSSourceValue::create(); + handleBindingRhs(ast->statement, value); + + Item * const targetItem = targetItemForBinding(bindingName, value); + checkDuplicateBinding(targetItem, bindingName, ast->qualifiedId->identifierToken); + targetItem->setProperty(bindingName.last(), value); + return false; +} + +bool ItemReaderASTVisitor::handleBindingRhs(AST::Statement *statement, + const JSSourceValuePtr &value) +{ + QBS_CHECK(statement); + QBS_CHECK(value); + + if (AST::cast<AST::Block *>(statement)) + value->setHasFunctionForm(); + + value->setFile(m_file); + value->setSourceCode(textViewOf(m_file->content(), statement)); + value->setLocation(statement->firstSourceLocation().startLine, + statement->firstSourceLocation().startColumn); + + bool usesBase, usesOuter, usesOriginal; + IdentifierSearch idsearch; + idsearch.add(StringConstants::baseVar(), &usesBase); + idsearch.add(StringConstants::outerVar(), &usesOuter); + idsearch.add(StringConstants::originalVar(), &usesOriginal); + idsearch.start(statement); + if (usesBase) + value->setSourceUsesBase(); + if (usesOuter) + value->setSourceUsesOuter(); + if (usesOriginal) + value->setSourceUsesOriginal(); + return false; +} + +CodeLocation ItemReaderASTVisitor::toCodeLocation(const AST::SourceLocation &location) const +{ + return CodeLocation(m_file->filePath(), location.startLine, location.startColumn); +} + +Item *ItemReaderASTVisitor::targetItemForBinding(const QStringList &bindingName, + const JSSourceValueConstPtr &value) +{ + Item *targetItem = m_item; + const int c = bindingName.size() - 1; + for (int i = 0; i < c; ++i) { + ValuePtr v = targetItem->ownProperty(bindingName.at(i)); + if (!v) { + const ItemType itemType = i < c - 1 ? ItemType::ModulePrefix : m_instanceItemType; + Item *newItem = Item::create(m_itemPool, itemType); + newItem->setLocation(value->location()); + v = ItemValue::create(newItem); + targetItem->setProperty(bindingName.at(i), v); + } + if (Q_UNLIKELY(v->type() != Value::ItemValueType)) { + QString msg = Tr::tr("Binding to non-item property."); + throw ErrorInfo(msg, value->location()); + } + targetItem = std::static_pointer_cast<ItemValue>(v)->item(); + } + return targetItem; +} + +void ItemReaderASTVisitor::inheritItem(Item *dst, const Item *src) +{ + int insertPos = 0; + for (Item *child : std::as_const(src->m_children)) { + dst->m_children.insert(insertPos++, child); + child->m_parent = dst; + } + + for (const PropertyDeclaration &pd : src->propertyDeclarations()) { + if (pd.flags().testFlag(PropertyDeclaration::ReadOnlyFlag) + && dst->hasOwnProperty(pd.name())) { + throw ErrorInfo(Tr::tr("Cannot set read-only property '%1'.").arg(pd.name()), + dst->property(pd.name())->location()); + } + dst->setPropertyDeclaration(pd.name(), pd); + } + + std::unique_ptr<ModuleItemLocker> locker; + if (src->type() == ItemType::Module) + locker = std::make_unique<ModuleItemLocker>(*src); + for (auto it = src->properties().constBegin(); it != src->properties().constEnd(); ++it) { + ValuePtr &v = dst->m_properties[it.key()]; + if (!v) { + v = it.value(); + continue; + } + if (v->type() == Value::ItemValueType && it.value()->type() != Value::ItemValueType) + throw ErrorInfo(Tr::tr("Binding to non-item property."), v->location()); + if (v->type() != it.value()->type()) + continue; + switch (v->type()) { + case Value::JSSourceValueType: { + JSSourceValuePtr sv = std::static_pointer_cast<JSSourceValue>(v); + QBS_CHECK(!sv->baseValue()); + const JSSourceValuePtr baseValue = std::static_pointer_cast<JSSourceValue>(it.value()); + sv->setBaseValue(baseValue); + for (const JSSourceValue::Alternative &alt : sv->m_alternatives) + alt.value->setBaseValue(baseValue); + break; + } + case Value::ItemValueType: + inheritItem(std::static_pointer_cast<ItemValue>(v)->item(), + std::static_pointer_cast<const ItemValue>(it.value())->item()); + break; + default: + QBS_CHECK(!"unexpected value type"); + } + } +} + +void ItemReaderASTVisitor::checkDeprecationStatus(ItemType itemType, const QString &itemName, + const CodeLocation &itemLocation) +{ + const ItemDeclaration itemDecl = BuiltinDeclarations::instance().declarationsForType(itemType); + const ErrorInfo error = itemDecl.checkForDeprecation(m_visitorState.deprecationWarningMode(), + itemName, itemLocation, m_logger); + if (error.hasError()) + throw error; +} + +void ItemReaderASTVisitor::doCheckItemTypes(const Item *item) +{ + const ItemDeclaration decl = BuiltinDeclarations::instance().declarationsForType(item->type()); + for (const Item * const child : item->children()) { + if (!decl.isChildTypeAllowed(child->type())) { + throw ErrorInfo(Tr::tr("Items of type '%1' cannot contain items of type '%2'.") + .arg(item->typeName(), child->typeName()), child->location()); + } + doCheckItemTypes(child); + } +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/itemreaderastvisitor.h b/src/lib/corelib/loader/itemreaderastvisitor.h new file mode 100644 index 000000000..c47501f2f --- /dev/null +++ b/src/lib/corelib/loader/itemreaderastvisitor.h @@ -0,0 +1,103 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QBS_ITEMREADERASTVISITOR_H +#define QBS_ITEMREADERASTVISITOR_H + +#include <language/forward_decls.h> +#include <language/itemtype.h> + +#include <logging/logger.h> +#include <parser/qmljsastvisitor_p.h> + +#include <QtCore/qhash.h> +#include <QtCore/qstringlist.h> + +#include <memory.h> + +namespace qbs { +class CodeLocation; + +namespace Internal { +class Item; +class ItemPool; +class ItemReaderVisitorState; +class ModuleItemLocker; + +class ItemReaderASTVisitor : public QbsQmlJS::AST::Visitor +{ +public: + ItemReaderASTVisitor(ItemReaderVisitorState &visitorState, FileContextPtr file, + ItemPool *itemPool, Logger &logger); + ~ItemReaderASTVisitor(); + + void checkItemTypes() { doCheckItemTypes(rootItem()); } + + Item *rootItem() const { return m_item; } + +private: + bool visit(QbsQmlJS::AST::UiProgram *uiProgram) override; + bool visit(QbsQmlJS::AST::UiObjectDefinition *ast) override; + bool visit(QbsQmlJS::AST::UiPublicMember *ast) override; + bool visit(QbsQmlJS::AST::UiScriptBinding *ast) override; + + bool handleBindingRhs(QbsQmlJS::AST::Statement *statement, const JSSourceValuePtr &value); + CodeLocation toCodeLocation(const QbsQmlJS::AST::SourceLocation &location) const; + void checkDuplicateBinding(Item *item, const QStringList &bindingName, + const QbsQmlJS::AST::SourceLocation &sourceLocation); + Item *targetItemForBinding(const QStringList &binding, const JSSourceValueConstPtr &value); + static void inheritItem(Item *dst, const Item *src); + void checkDeprecationStatus(ItemType itemType, const QString &itemName, + const CodeLocation &itemLocation); + void doCheckItemTypes(const Item *item); + + ItemReaderVisitorState &m_visitorState; + const FileContextPtr m_file; + ItemPool * const m_itemPool; + Logger &m_logger; + QHash<QStringList, QString> m_typeNameToFile; + Item *m_item = nullptr; + std::unique_ptr<ModuleItemLocker> m_moduleItemLocker; + ItemType m_instanceItemType = ItemType::ModuleInstancePlaceholder; +}; + +} // namespace Internal +} // namespace qbs + +#endif // QBS_ITEMREADERASTVISITOR_H diff --git a/src/lib/corelib/loader/itemreadervisitorstate.cpp b/src/lib/corelib/loader/itemreadervisitorstate.cpp new file mode 100644 index 000000000..6ea23c021 --- /dev/null +++ b/src/lib/corelib/loader/itemreadervisitorstate.cpp @@ -0,0 +1,149 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "itemreadervisitorstate.h" + +#include "itemreaderastvisitor.h" +#include "loaderutils.h" + +#include <language/asttools.h> +#include <language/filecontext.h> +#include <logging/translator.h> +#include <parser/qmljsengine_p.h> +#include <parser/qmljslexer_p.h> +#include <parser/qmljsparser_p.h> +#include <tools/error.h> +#include <tools/stringconstants.h> + +#include <QtCore/qdiriterator.h> +#include <QtCore/qfile.h> +#include <QtCore/qfileinfo.h> +#include <QtCore/qtextstream.h> + +namespace qbs { +namespace Internal { + +ItemReaderVisitorState::ItemReaderVisitorState(ItemReaderCache &cache, Logger &logger) + : m_cache(cache), m_logger(logger) +{ +} + +Item *ItemReaderVisitorState::readFile(const QString &filePath, const QStringList &searchPaths, + ItemPool *itemPool) +{ + const auto setupCacheEntry = [&](ItemReaderCache::AstCacheEntry &entry) { + QFile file(filePath); + if (Q_UNLIKELY(!file.open(QFile::ReadOnly))) + throw ErrorInfo(Tr::tr("Cannot open '%1'.").arg(filePath)); + + QTextStream stream(&file); + setupDefaultCodec(stream); + const QString &code = stream.readAll(); + QbsQmlJS::Lexer lexer(&entry.engine); + lexer.setCode(code, 1); + QbsQmlJS::Parser parser(&entry.engine); + + file.close(); + if (!parser.parse()) { + const QList<QbsQmlJS::DiagnosticMessage> &parserMessages = parser.diagnosticMessages(); + if (Q_UNLIKELY(!parserMessages.empty())) { + ErrorInfo err; + for (const QbsQmlJS::DiagnosticMessage &msg : parserMessages) + err.append(msg.message, toCodeLocation(filePath, msg.loc)); + throw err; + } + } + + entry.code = code; + entry.ast = parser.ast(); + }; + + ItemReaderCache::AstCacheEntry &cacheEntry = m_cache.retrieveOrSetupCacheEntry( + filePath, setupCacheEntry); + const FileContextPtr file = FileContext::create(); + file->setFilePath(QFileInfo(filePath).absoluteFilePath()); + file->setContent(cacheEntry.code); + file->setSearchPaths(searchPaths); + + ItemReaderASTVisitor astVisitor(*this, file, itemPool, m_logger); + { + class ProcessingFlagManager { + public: + ProcessingFlagManager(ItemReaderCache::AstCacheEntry &e, const QString &filePath) + : m_cacheEntry(e) + { + if (!e.addProcessingThread()) + throw ErrorInfo(Tr::tr("Loop detected when importing '%1'.").arg(filePath)); + } + ~ProcessingFlagManager() { m_cacheEntry.removeProcessingThread(); } + + private: + ItemReaderCache::AstCacheEntry &m_cacheEntry; + } processingFlagManager(cacheEntry, filePath); + cacheEntry.ast->accept(&astVisitor); + } + astVisitor.checkItemTypes(); + return astVisitor.rootItem(); +} + +void ItemReaderVisitorState::findDirectoryEntries(const QString &dirPath, QStringList *entries) const +{ + *entries = m_cache.retrieveOrSetDirectoryEntries(dirPath, [&dirPath] { + QStringList fileNames; + QDirIterator dirIter(dirPath, StringConstants::qbsFileWildcards()); + while (dirIter.hasNext()) { + dirIter.next(); + fileNames << dirIter.fileName(); + } + return fileNames; + }); +} + +Item *ItemReaderVisitorState::mostDerivingItem() const +{ + return m_mostDerivingItem; +} + +void ItemReaderVisitorState::setMostDerivingItem(Item *item) +{ + m_mostDerivingItem = item; +} + + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/itemreadervisitorstate.h b/src/lib/corelib/loader/itemreadervisitorstate.h new file mode 100644 index 000000000..cbf1966b6 --- /dev/null +++ b/src/lib/corelib/loader/itemreadervisitorstate.h @@ -0,0 +1,83 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QBS_ITEMREADERVISITORSTATE_H +#define QBS_ITEMREADERVISITORSTATE_H + +#include <tools/deprecationwarningmode.h> +#include <tools/set.h> + +#include <QtCore/qstringlist.h> + +#include <memory> + +namespace qbs { +namespace Internal { +class Item; +class ItemPool; +class ItemReaderCache; +class Logger; + +class ItemReaderVisitorState +{ +public: + ItemReaderVisitorState(ItemReaderCache &cache, Logger &logger); + + Logger &logger() { return m_logger; } + + Item *readFile(const QString &filePath, const QStringList &searchPaths, ItemPool *itemPool); + + void findDirectoryEntries(const QString &dirPath, QStringList *entries) const; + + Item *mostDerivingItem() const; + void setMostDerivingItem(Item *item); + + void setDeprecationWarningMode(DeprecationWarningMode mode) { m_deprecationWarningMode = mode; } + DeprecationWarningMode deprecationWarningMode() const { return m_deprecationWarningMode; } + +private: + DeprecationWarningMode m_deprecationWarningMode = defaultDeprecationWarningMode(); + ItemReaderCache &m_cache; + Logger &m_logger; + Item *m_mostDerivingItem = nullptr; +}; + +} // namespace Internal +} // namespace qbs + +#endif // Include guard. diff --git a/src/lib/corelib/loader/loaderutils.cpp b/src/lib/corelib/loader/loaderutils.cpp new file mode 100644 index 000000000..05a077dbe --- /dev/null +++ b/src/lib/corelib/loader/loaderutils.cpp @@ -0,0 +1,961 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "loaderutils.h" + +#include "itemreader.h" + +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/itempool.h> +#include <language/language.h> +#include <language/resolvedfilecontext.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/fileinfo.h> +#include <tools/progressobserver.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +namespace qbs::Internal { + +QString fullProductDisplayName(const QString &name, const QString &multiplexId) +{ + static const auto multiplexIdToString =[](const QString &id) { + return QString::fromUtf8(QByteArray::fromBase64(id.toUtf8())); + }; + QString result = name; + if (!multiplexId.isEmpty()) + result.append(QLatin1Char(' ')).append(multiplexIdToString(multiplexId)); + return result; +} + +void adjustParametersScopes(Item *item, Item *scope) +{ + if (item->type() == ItemType::ModuleParameters) { + item->setScope(scope); + return; + } + + for (const auto &value : item->properties()) { + if (value->type() == Value::ItemValueType) + adjustParametersScopes(std::static_pointer_cast<ItemValue>(value)->item(), scope); + } +} + +QString ProductContext::uniqueName() const +{ + return ResolvedProduct::uniqueName(name, multiplexConfigurationId); +} + +QString ProductContext::displayName() const +{ + return fullProductDisplayName(name, multiplexConfigurationId); +} + +void ProductContext::handleError(const ErrorInfo &error) +{ + const bool alreadyHadError = delayedError.hasError(); + if (!alreadyHadError) { + delayedError.append(Tr::tr("Error while handling product '%1':") + .arg(name), item->location()); + } + if (error.isInternalError()) { + if (alreadyHadError) { + qCDebug(lcModuleLoader()) << "ignoring subsequent internal error" << error.toString() + << "in product" << name; + return; + } + } + const auto errorItems = error.items(); + for (const ErrorItem &ei : errorItems) + delayedError.append(ei.description(), ei.codeLocation()); + project->topLevelProject->addDisabledItem(item); +} + +TopLevelProjectContext::~TopLevelProjectContext() { qDeleteAll(m_projects); } + +bool TopLevelProjectContext::checkItemCondition(Item *item, Evaluator &evaluator) +{ + if (evaluator.boolValue(item, StringConstants::conditionProperty())) + return true; + addDisabledItem(item); + return false; +} + +void TopLevelProjectContext::checkCancelation() +{ + if (m_progressObserver && m_progressObserver->canceled()) + m_canceled = true; + if (m_canceled) + throw CancelException(); +} + +QString TopLevelProjectContext::sourceCodeForEvaluation(const JSSourceValueConstPtr &value) +{ + const auto sourceCodeGuard = m_sourceCode.lock(); + QString &code = sourceCodeGuard.get()[value->sourceCode()]; + if (!code.isNull()) + return code; + code = value->sourceCodeForEvaluation(); + return code; +} + +ScriptFunctionPtr TopLevelProjectContext::scriptFunctionValue(Item *item, const QString &name) +{ + const JSSourceValuePtr value = item->sourceProperty(name); + QBS_CHECK(value); + const auto scriptFunctionMapGuard = m_scriptFunctionMap.lock(); + ScriptFunctionPtr &script = scriptFunctionMapGuard.get()[value->location()]; + if (!script.get()) { + script = ScriptFunction::create(); + const PropertyDeclaration decl = item->propertyDeclaration(name); + script->sourceCode = sourceCodeAsFunction(value, decl); + script->location = value->location(); + script->fileContext = resolvedFileContext(value->file()); + } + return script; +} + +QString TopLevelProjectContext::sourceCodeAsFunction(const JSSourceValueConstPtr &value, + const PropertyDeclaration &decl) +{ + const auto scriptFunctionMapGuard = m_scriptFunctions.lock(); + QString &scriptFunction = scriptFunctionMapGuard.get()[std::make_pair(value->sourceCode(), + decl.functionArgumentNames())]; + if (!scriptFunction.isNull()) + return scriptFunction; + const QString args = decl.functionArgumentNames().join(QLatin1Char(',')); + if (value->hasFunctionForm()) { + // Insert the argument list. + scriptFunction = value->sourceCodeForEvaluation(); + scriptFunction.insert(10, args); + // Remove the function application "()" that has been + // added in ItemReaderASTVisitor::visitStatement. + scriptFunction.chop(2); + } else { + scriptFunction = QLatin1String("(function(") + args + QLatin1String("){return ") + + value->sourceCode().toString() + QLatin1String(";})"); + } + return scriptFunction; +} + +const ResolvedFileContextPtr &TopLevelProjectContext::resolvedFileContext( + const FileContextConstPtr &ctx) +{ + ResolvedFileContextPtr &result = m_fileContextMap[ctx]; + if (!result) + result = ResolvedFileContext::create(*ctx); + return result; +} + +void TopLevelProjectContext::removeProductToHandle(const ProductContext &product) +{ + std::unique_lock lock(m_productsToHandle.mutex); + m_productsToHandle.data.remove(&product); +} + +bool TopLevelProjectContext::isProductQueuedForHandling(const ProductContext &product) const +{ + std::shared_lock lock(m_productsToHandle.mutex); + return m_productsToHandle.data.contains(&product); +} + +void TopLevelProjectContext::addDisabledItem(Item *item) +{ + m_disabledItems.lock().get() << item; +} + +bool TopLevelProjectContext::isDisabledItem(const Item *item) const +{ + return m_disabledItems.lock_shared().get().contains(item); +} + +void TopLevelProjectContext::setProgressObserver(ProgressObserver *observer) +{ + m_progressObserver = observer; +} + +ProgressObserver *TopLevelProjectContext::progressObserver() const { return m_progressObserver; } + +void TopLevelProjectContext::addQueuedError(const ErrorInfo &error) +{ + m_queuedErrors.lock().get() << error; +} + +void TopLevelProjectContext::addProfileConfig(const QString &profileName, + const QVariantMap &profileConfig) +{ + m_profileConfigs.insert(profileName, profileConfig); +} + +std::optional<QVariantMap> TopLevelProjectContext::profileConfig(const QString &profileName) const +{ + const auto it = m_profileConfigs.constFind(profileName); + if (it == m_profileConfigs.constEnd()) + return {}; + return it.value().toMap(); +} + +void TopLevelProjectContext::addProjectLevelProbe(const ProbeConstPtr &probe) +{ + m_probesInfo.projectLevelProbes << probe; +} + +const std::vector<ProbeConstPtr> TopLevelProjectContext::projectLevelProbes() const +{ + return m_probesInfo.projectLevelProbes; +} + +void TopLevelProjectContext::addProduct(ProductContext &product) +{ + m_productsByName.insert({product.name, &product}); +} + +void TopLevelProjectContext::addProductByType(ProductContext &product, const FileTags &tags) +{ + const auto productsByTypeGuard = m_productsByType.lock(); + for (const FileTag &tag : tags) + productsByTypeGuard.get().insert({tag, &product}); +} + +ProductContext *TopLevelProjectContext::productWithNameAndConstraint( + const QString &name, const std::function<bool (ProductContext &)> &constraint) +{ + const auto candidates = m_productsByName.equal_range(name); + for (auto it = candidates.first; it != candidates.second; ++it) { + ProductContext * const candidate = it->second; + if (constraint(*candidate)) + return candidate; + } + return nullptr; +} + +std::vector<ProductContext *> TopLevelProjectContext::productsWithNameAndConstraint( + const QString &name, const std::function<bool (ProductContext &)> &constraint) +{ + std::vector<ProductContext *> result; + const auto candidates = m_productsByName.equal_range(name); + for (auto it = candidates.first; it != candidates.second; ++it) { + ProductContext * const candidate = it->second; + if (constraint(*candidate)) + result << candidate; + } + return result; +} + +std::vector<ProductContext *> TopLevelProjectContext::productsWithTypeAndConstraint( + const FileTags &tags, const std::function<bool (ProductContext &)> &constraint) +{ + const auto productsByTypeGuard = m_productsByType.lock_shared(); + std::vector<ProductContext *> matchingProducts; + for (const FileTag &typeTag : tags) { + const auto range = productsByTypeGuard.get().equal_range(typeTag); + for (auto it = range.first; it != range.second; ++it) { + if (constraint(*it->second)) + matchingProducts.push_back(it->second); + } + } + return matchingProducts; +} + +std::vector<std::pair<ProductContext *, CodeLocation>> +TopLevelProjectContext::finishedProductsWithBulkDependency(const FileTag &tag) const +{ + return m_reverseBulkDependencies.value(tag); +} + +void TopLevelProjectContext::registerBulkDependencies(ProductContext &product) +{ + for (const auto &tagAndLoc : product.bulkDependencies) { + for (const FileTag &tag : tagAndLoc.first) + m_reverseBulkDependencies[tag].emplace_back(&product, tagAndLoc.second); + } +} + +void TopLevelProjectContext::addProjectNameUsedInOverrides(const QString &name) +{ + m_projectNamesUsedInOverrides << name; +} + +const Set<QString> &TopLevelProjectContext::projectNamesUsedInOverrides() const +{ + return m_projectNamesUsedInOverrides; +} + +void TopLevelProjectContext::addProductNameUsedInOverrides(const QString &name) +{ + m_productNamesUsedInOverrides << name; +} + +const Set<QString> &TopLevelProjectContext::productNamesUsedInOverrides() const +{ + return m_productNamesUsedInOverrides; +} + +void TopLevelProjectContext::addMultiplexConfiguration(const QString &id, const QVariantMap &config) +{ + m_multiplexConfigsById.insert(std::make_pair(id, config)); +} + +QVariantMap TopLevelProjectContext::multiplexConfiguration(const QString &id) const +{ + if (id.isEmpty()) + return {}; + const auto it = m_multiplexConfigsById.find(id); + QBS_CHECK(it != m_multiplexConfigsById.end() && !it->second.isEmpty()); + return it->second; +} + +std::lock_guard<std::mutex> TopLevelProjectContext::moduleProvidersCacheLock() +{ + return std::lock_guard<std::mutex>(m_moduleProvidersCacheMutex); +} + +void TopLevelProjectContext::setModuleProvidersCache(const ModuleProvidersCache &cache) +{ + m_moduleProvidersCache = cache; +} + +ModuleProviderInfo *TopLevelProjectContext::moduleProvider(const ModuleProvidersCacheKey &key) +{ + if (const auto it = m_moduleProvidersCache.find(key); it != m_moduleProvidersCache.end()) + return &(*it); + return nullptr; +} + +ModuleProviderInfo &TopLevelProjectContext::addModuleProvider(const ModuleProvidersCacheKey &key, + const ModuleProviderInfo &provider) +{ + return m_moduleProvidersCache[key] = provider; +} + +void TopLevelProjectContext::addParameterDeclarations(const Item *moduleProto, + const Item::PropertyDeclarationMap &decls) +{ + m_parameterDeclarations.lock().get().insert({moduleProto, decls}); +} + +Item::PropertyDeclarationMap TopLevelProjectContext::parameterDeclarations(Item *moduleProto) const +{ + const auto parameterDeclarationsGuard = m_parameterDeclarations.lock_shared(); + const auto &declarations = parameterDeclarationsGuard.get(); + if (const auto it = declarations.find(moduleProto); it != declarations.end()) { + return it->second; + } + return {}; +} + +void TopLevelProjectContext::setParameters(const Item *moduleProto, const QVariantMap ¶meters) +{ + m_parameters.lock().get().insert({moduleProto, parameters}); +} + +QVariantMap TopLevelProjectContext::parameters(Item *moduleProto) const +{ + const auto parametersGuard = m_parameters.lock_shared(); + const auto ¶meters = parametersGuard.get(); + if (const auto it = parameters.find(moduleProto); it != parameters.end()) { + return it->second; + } + return {}; +} + +void TopLevelProjectContext::addCodeLink(const QString &sourceFile, const CodeRange &sourceRange, + const CodeLocation &target) +{ + const auto codeLinksGuard = m_codeLinks.lock(); + QList<CodeLocation> &links = codeLinksGuard.get()[sourceFile][sourceRange]; + if (!links.contains(target)) + links << target; +} + +QString TopLevelProjectContext::findModuleDirectory( + const QualifiedId &module, const QString &searchPath, + const std::function<QString()> &findOnDisk) +{ + const auto modulePathCacheGuard = m_modulePathCache.lock(); + auto &path = modulePathCacheGuard.get()[{searchPath, module}]; + if (!path) + path = findOnDisk(); + return *path; +} + +QStringList TopLevelProjectContext::getModuleFilesForDirectory( + const QString &dir, const std::function<QStringList ()> &findOnDisk) +{ + const auto moduleFilesGuard = m_moduleFilesPerDirectory.lock(); + auto &list = moduleFilesGuard.get()[dir]; + if (!list) + list = findOnDisk(); + return *list; +} + +void TopLevelProjectContext::removeModuleFileFromDirectoryCache(const QString &filePath) +{ + const auto moduleFilesGuard = m_moduleFilesPerDirectory.lock(); + auto &moduleFiles = moduleFilesGuard.get(); + const auto it = moduleFiles.find(FileInfo::path(filePath)); + QBS_CHECK(it != moduleFiles.end()); + auto &files = it->second; + QBS_CHECK(files); + files->removeOne(filePath); +} + +void TopLevelProjectContext::addUnknownProfilePropertyError(const Item *moduleProto, + const ErrorInfo &error) +{ + m_unknownProfilePropertyErrors.lock().get()[moduleProto].push_back(error); +} + +const std::vector<ErrorInfo> &TopLevelProjectContext::unknownProfilePropertyErrors( + const Item *moduleProto) const +{ + const auto errorsGuard = m_unknownProfilePropertyErrors.lock_shared(); + const auto &errors = errorsGuard.get(); + if (const auto it = errors.find(moduleProto); it != errors.end()) { + return it->second; + } + static const std::vector<ErrorInfo> empty; + return empty; +} + +Item *TopLevelProjectContext::getModulePrototype(const QString &filePath, const QString &profile, + const std::function<Item *()> &produce) +{ + const auto modulePrototypesGuard = m_modulePrototypes.lock(); + auto &prototypeList = modulePrototypesGuard.get()[filePath]; + for (const auto &prototype : prototypeList) { + if (prototype.second == profile) + return prototype.first; + } + Item * const module = produce(); + if (module) + prototypeList.emplace_back(module, profile); + return module; +} + +void TopLevelProjectContext::addLocalProfile(const QString &name, const QVariantMap &values, + const CodeLocation &location) +{ + if (m_localProfiles.contains(name)) + throw ErrorInfo(Tr::tr("Local profile '%1' redefined.").arg(name), location); + m_localProfiles.insert(name, values); +} + +void TopLevelProjectContext::checkForLocalProfileAsTopLevelProfile(const QString &topLevelProfile) +{ + for (auto it = m_localProfiles.cbegin(); it != m_localProfiles.cend(); ++it) { + if (it.key() != topLevelProfile) + continue; + + // This covers the edge case that a locally defined profile was specified as the + // top-level profile, in which case we must invalidate the qbs module prototype that was + // created in early setup before local profiles were handled. + const auto prototypesGuard = m_modulePrototypes.lock(); + auto &modulePrototypes = prototypesGuard.get(); + QBS_CHECK(modulePrototypes.size() == 1); + modulePrototypes.clear(); + break; + } +} + +std::lock_guard<std::mutex> TopLevelProjectContext::probesCacheLock() +{ + return std::lock_guard<std::mutex>(m_probesMutex); +} + +void TopLevelProjectContext::setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes) +{ + for (const ProbeConstPtr& probe : oldProbes) + m_probesInfo.oldProjectProbes[probe->globalId()] << probe; +} + +ProbeConstPtr TopLevelProjectContext::findOldProjectProbe(const QString &id, + const ProbeFilter &filter) const +{ + for (const ProbeConstPtr &oldProbe : m_probesInfo.oldProjectProbes.value(id)) { + if (filter(oldProbe)) + return oldProbe; + } + return {}; +} + +void TopLevelProjectContext::setOldProductProbes( + const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes) +{ + m_probesInfo.oldProductProbes = oldProbes; +} + +ProbeConstPtr TopLevelProjectContext::findOldProductProbe(const QString &productName, + const ProbeFilter &filter) const +{ + for (const ProbeConstPtr &oldProbe : m_probesInfo.oldProductProbes.value(productName)) { + if (filter(oldProbe)) + return oldProbe; + } + return {}; +} + +void TopLevelProjectContext::addNewlyResolvedProbe(const ProbeConstPtr &probe) +{ + m_probesInfo.currentProbes[probe->location()] << probe; +} + +ProbeConstPtr TopLevelProjectContext::findCurrentProbe(const CodeLocation &location, + const ProbeFilter &filter) const +{ + for (const ProbeConstPtr &probe : m_probesInfo.currentProbes.value(location)) { + if (filter(probe)) + return probe; + } + return {}; +} + +void TopLevelProjectContext::collectDataFromEngine(const ScriptEngine &engine) +{ + const auto project = dynamic_cast<TopLevelProject *>(m_projects.front()->project.get()); + QBS_CHECK(project); + project->canonicalFilePathResults.insert(engine.canonicalFilePathResults()); + project->fileExistsResults.insert(engine.fileExistsResults()); + project->directoryEntriesResults.insert(engine.directoryEntriesResults()); + project->fileLastModifiedResults.insert(engine.fileLastModifiedResults()); + project->environment.insert(engine.environment()); + project->buildSystemFiles.unite(engine.imports()); +} + +ItemPool &TopLevelProjectContext::createItemPool() +{ + m_itemPools.push_back(std::make_unique<ItemPool>()); + return *m_itemPools.back(); +} + +class LoaderState::Private +{ +public: + Private(LoaderState &q, const SetupProjectParameters ¶meters, + TopLevelProjectContext &topLevelProject, ItemPool &itemPool, ScriptEngine &engine, + Logger &&logger) + : parameters(parameters), topLevelProject(topLevelProject), itemPool(itemPool), + logger(std::move(logger)), itemReader(q), evaluator(&engine) + { + this->logger.clearWarnings(); + this->logger.storeWarnings(); + } + + const SetupProjectParameters ¶meters; + TopLevelProjectContext &topLevelProject; + ItemPool &itemPool; + + Logger logger; + ItemReader itemReader; + Evaluator evaluator; +}; + +LoaderState::LoaderState(const SetupProjectParameters ¶meters, + TopLevelProjectContext &topLevelProject, ItemPool &itemPool, + ScriptEngine &engine, Logger logger) + : d(makePimpl<Private>(*this, parameters, topLevelProject, itemPool, engine, std::move(logger))) +{ + d->itemReader.init(); +} + +LoaderState::~LoaderState() = default; +const SetupProjectParameters &LoaderState::parameters() const { return d->parameters; } +ItemPool &LoaderState::itemPool() { return d->itemPool; } +Evaluator &LoaderState::evaluator() { return d->evaluator; } +Logger &LoaderState::logger() { return d->logger; } +ItemReader &LoaderState::itemReader() { return d->itemReader; } +TopLevelProjectContext &LoaderState::topLevelProject() { return d->topLevelProject; } + +static QString verbatimValue(LoaderState &state, const ValueConstPtr &value) +{ + QString result; + if (value && value->type() == Value::JSSourceValueType) { + const JSSourceValueConstPtr sourceValue = std::static_pointer_cast<const JSSourceValue>( + value); + result = state.topLevelProject().sourceCodeForEvaluation(sourceValue); + } + return result; +} + +static void resolveRuleArtifactBinding( + LoaderState &state, const RuleArtifactPtr &ruleArtifact, Item *item, + const QStringList &namePrefix, QualifiedIdSet *seenBindings) +{ + for (auto it = item->properties().constBegin(); it != item->properties().constEnd(); ++it) { + const QStringList name = QStringList(namePrefix) << it.key(); + if (it.value()->type() == Value::ItemValueType) { + resolveRuleArtifactBinding(state, ruleArtifact, + std::static_pointer_cast<ItemValue>(it.value())->item(), + name, seenBindings); + } else if (it.value()->type() == Value::JSSourceValueType) { + const auto insertResult = seenBindings->insert(name); + if (!insertResult.second) + continue; + JSSourceValuePtr sourceValue = std::static_pointer_cast<JSSourceValue>(it.value()); + RuleArtifact::Binding rab; + rab.name = name; + rab.code = state.topLevelProject().sourceCodeForEvaluation(sourceValue); + rab.location = sourceValue->location(); + ruleArtifact->bindings.push_back(rab); + } else { + QBS_ASSERT(!"unexpected value type", continue); + } + } +} + +static void resolveRuleArtifact(LoaderState &state, const RulePtr &rule, Item *item) +{ + RuleArtifactPtr artifact = RuleArtifact::create(); + rule->artifacts.push_back(artifact); + artifact->location = item->location(); + + if (const auto sourceProperty = item->sourceProperty(StringConstants::filePathProperty())) + artifact->filePathLocation = sourceProperty->location(); + + artifact->filePath = verbatimValue(state, item->property(StringConstants::filePathProperty())); + artifact->fileTags = state.evaluator().fileTagsValue(item, StringConstants::fileTagsProperty()); + artifact->alwaysUpdated = state.evaluator().boolValue( + item, StringConstants::alwaysUpdatedProperty()); + + QualifiedIdSet seenBindings; + for (Item *obj = item; obj; obj = obj->prototype()) { + for (QMap<QString, ValuePtr>::const_iterator it = obj->properties().constBegin(); + it != obj->properties().constEnd(); ++it) + { + if (it.value()->type() != Value::ItemValueType) + continue; + resolveRuleArtifactBinding( + state, artifact, std::static_pointer_cast<ItemValue>(it.value())->item(), + QStringList(it.key()), &seenBindings); + } + } +} + +void resolveRule(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext) +{ + Evaluator &evaluator = state.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) + return; + + RulePtr rule = Rule::create(); + + // read artifacts + bool hasArtifactChildren = false; + for (Item * const child : item->children()) { + if (Q_UNLIKELY(child->type() != ItemType::Artifact)) { + throw ErrorInfo(Tr::tr("'Rule' can only have children of type 'Artifact'."), + child->location()); + } + hasArtifactChildren = true; + resolveRuleArtifact(state, rule, child); + } + + rule->name = evaluator.stringValue(item, StringConstants::nameProperty()); + rule->prepareScript.initialize(state.topLevelProject().scriptFunctionValue( + item, StringConstants::prepareProperty())); + rule->outputArtifactsScript.initialize(state.topLevelProject().scriptFunctionValue( + item, StringConstants::outputArtifactsProperty())); + rule->outputFileTags = evaluator.fileTagsValue(item, StringConstants::outputFileTagsProperty()); + if (rule->outputArtifactsScript.isValid()) { + if (hasArtifactChildren) + throw ErrorInfo(Tr::tr("The Rule.outputArtifacts script is not allowed in rules " + "that contain Artifact items."), + item->location()); + } + if (!hasArtifactChildren && rule->outputFileTags.empty()) { + throw ErrorInfo(Tr::tr("A rule needs to have Artifact items or a non-empty " + "outputFileTags property."), item->location()); + } + rule->multiplex = evaluator.boolValue(item, StringConstants::multiplexProperty()); + rule->alwaysRun = evaluator.boolValue(item, StringConstants::alwaysRunProperty()); + rule->inputs = evaluator.fileTagsValue(item, StringConstants::inputsProperty()); + rule->inputsFromDependencies + = evaluator.fileTagsValue(item, StringConstants::inputsFromDependenciesProperty()); + bool requiresInputsSet = false; + rule->requiresInputs = evaluator.boolValue(item, StringConstants::requiresInputsProperty(), + &requiresInputsSet); + if (!requiresInputsSet) + rule->requiresInputs = rule->declaresInputs(); + rule->auxiliaryInputs + = evaluator.fileTagsValue(item, StringConstants::auxiliaryInputsProperty()); + rule->excludedInputs + = evaluator.fileTagsValue(item, StringConstants::excludedInputsProperty()); + if (rule->excludedInputs.empty()) { + rule->excludedInputs = evaluator.fileTagsValue( + item, StringConstants::excludedAuxiliaryInputsProperty()); + } + rule->explicitlyDependsOn + = evaluator.fileTagsValue(item, StringConstants::explicitlyDependsOnProperty()); + rule->explicitlyDependsOnFromDependencies = evaluator.fileTagsValue( + item, StringConstants::explicitlyDependsOnFromDependenciesProperty()); + rule->module = moduleContext ? moduleContext->module : projectContext->dummyModule; + if (!rule->multiplex && !rule->declaresInputs()) { + throw ErrorInfo(Tr::tr("Rule has no inputs, but is not a multiplex rule."), + item->location()); + } + if (!rule->multiplex && !rule->requiresInputs) { + throw ErrorInfo(Tr::tr("Rule.requiresInputs is false for non-multiplex rule."), + item->location()); + } + if (!rule->declaresInputs() && rule->requiresInputs) { + throw ErrorInfo(Tr::tr("Rule.requiresInputs is true, but the rule " + "does not declare any input tags."), item->location()); + } + if (productContext) { + rule->product = productContext->product.get(); + productContext->product->rules.push_back(rule); + } else { + projectContext->rules.push_back(rule); + } +} + +void resolveFileTagger(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext) +{ + Evaluator &evaluator = state.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) + return; + std::vector<FileTaggerConstPtr> &fileTaggers = productContext + ? productContext->product->fileTaggers + : projectContext->fileTaggers; + const QStringList patterns = evaluator.stringListValue(item, + StringConstants::patternsProperty()); + if (patterns.empty()) + throw ErrorInfo(Tr::tr("FileTagger.patterns must be a non-empty list."), item->location()); + + const FileTags fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty()); + if (fileTags.empty()) + throw ErrorInfo(Tr::tr("FileTagger.fileTags must not be empty."), item->location()); + + for (const QString &pattern : patterns) { + if (pattern.isEmpty()) + throw ErrorInfo(Tr::tr("A FileTagger pattern must not be empty."), item->location()); + } + + const int priority = evaluator.intValue(item, StringConstants::priorityProperty()); + fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority)); +} + +void resolveJobLimit(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext) +{ + Evaluator &evaluator = state.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) + return; + const QString jobPool = evaluator.stringValue(item, StringConstants::jobPoolProperty()); + if (jobPool.isEmpty()) + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.") + .arg(StringConstants::jobPoolProperty()), item->location()); + bool jobCountWasSet; + const int jobCount = evaluator.intValue(item, StringConstants::jobCountProperty(), -1, + &jobCountWasSet); + if (!jobCountWasSet) { + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + if (jobCount < 0) { + throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + JobLimits &jobLimits = moduleContext + ? moduleContext->jobLimits + : productContext ? productContext->product->jobLimits + : projectContext->jobLimits; + JobLimit jobLimit(jobPool, jobCount); + const int oldLimit = jobLimits.getLimit(jobPool); + if (oldLimit == -1 || oldLimit > jobCount) + jobLimits.setJobLimit(jobLimit); +} + +const FileTag unknownFileTag() +{ + static const FileTag tag("unknown-file-tag"); + return tag; +} + +bool ProductContext::dependenciesResolvingPending() const +{ + return (!dependenciesContext || !dependenciesContext->dependenciesResolved) + && !product && !delayedError.hasError(); +} + +std::pair<ProductDependency, ProductContext *> ProductContext::pendingDependency() const +{ + return dependenciesContext ? dependenciesContext->pendingDependency() + : std::make_pair(ProductDependency::None, nullptr); +} + +TimingData &TimingData::operator+=(const TimingData &other) +{ + dependenciesResolving += other.dependenciesResolving; + moduleProviders += other.moduleProviders; + moduleInstantiation += other.moduleInstantiation; + propertyMerging += other.propertyMerging; + groupsSetup += other.groupsSetup; + groupsResolving += other.groupsResolving; + preparingProducts += other.preparingProducts; + resolvingProducts += other.resolvingProducts; + probes += other.probes; + propertyEvaluation += other.propertyEvaluation; + propertyChecking += other.propertyChecking; + return *this; +} + +DependenciesContext::~DependenciesContext() = default; + +ItemReaderCache::AstCacheEntry &ItemReaderCache::retrieveOrSetupCacheEntry( + const QString &filePath, const std::function<void (AstCacheEntry &)> &setup) +{ + const auto astCacheGuard = m_astCache.lock(); + AstCacheEntry &entry = astCacheGuard.get()[filePath]; + if (!entry.ast) { + setup(entry); + m_filesRead << filePath; + } + return entry; +} + +const QStringList &ItemReaderCache::retrieveOrSetDirectoryEntries( + const QString &dir, const std::function<QStringList ()> &findOnDisk) +{ + const auto directoryEntriesGuard = m_directoryEntries.lock(); + auto &entries = directoryEntriesGuard.get()[dir]; + if (!entries) + entries = findOnDisk(); + return *entries; +} + +bool ItemReaderCache::AstCacheEntry::addProcessingThread() +{ + return m_processingThreads.lock().get().insert(std::this_thread::get_id()).second; +} + +void ItemReaderCache::AstCacheEntry::removeProcessingThread() +{ + m_processingThreads.lock().get().remove(std::this_thread::get_id()); +} + +class DependencyParametersMerger +{ +public: + DependencyParametersMerger(std::vector<Item::Module::ParametersWithPriority> &&candidates) + : m_candidates(std::move(candidates)) { } + QVariantMap merge(); + +private: + void merge(QVariantMap ¤t, const QVariantMap &next, int nextPrio); + + const std::vector<Item::Module::ParametersWithPriority> m_candidates; + + struct Conflict { + Conflict(QStringList path, QVariant val1, QVariant val2, int prio) + : path(std::move(path)), val1(std::move(val1)), val2(std::move(val2)), priority(prio) {} + QStringList path; + QVariant val1; + QVariant val2; + int priority; + }; + std::vector<Conflict> m_conflicts; + QVariantMap m_currentValue; + int m_currentPrio = INT_MIN; + QStringList m_path; +}; + +QVariantMap mergeDependencyParameters(std::vector<Item::Module::ParametersWithPriority> &&candidates) +{ + return DependencyParametersMerger(std::move(candidates)).merge(); +} + +QVariantMap mergeDependencyParameters(const QVariantMap &m1, const QVariantMap &m2) +{ + return mergeDependencyParameters({std::make_pair(m1, 0), std::make_pair(m2, 0)}); +} + +QVariantMap DependencyParametersMerger::merge() +{ + for (const auto &next : m_candidates) { + merge(m_currentValue, next.first, next.second); + m_currentPrio = next.second; + } + + if (!m_conflicts.empty()) { + ErrorInfo error(Tr::tr("Conflicting parameter values encountered:")); + for (const Conflict &conflict : m_conflicts) { + // TODO: Location would be nice ... + error.append(Tr::tr(" Parameter '%1' cannot be both '%2' and '%3'.") + .arg(conflict.path.join(QLatin1Char('.')), + conflict.val1.toString(), conflict.val2.toString())); + } + throw error; + } + + return m_currentValue; +} + +void DependencyParametersMerger::merge(QVariantMap ¤t, const QVariantMap &next, int nextPrio) +{ + for (auto it = next.begin(); it != next.end(); ++it) { + m_path << it.key(); + const QVariant &newValue = it.value(); + QVariant ¤tValue = current[it.key()]; + if (newValue.userType() == QMetaType::QVariantMap) { + QVariantMap mdst = currentValue.toMap(); + merge(mdst, it.value().toMap(), nextPrio); + currentValue = mdst; + } else { + if (m_currentPrio == nextPrio) { + if (currentValue.isValid() && !qVariantsEqual(currentValue, newValue)) + m_conflicts.emplace_back(m_path, currentValue, newValue, m_currentPrio); + } else { + removeIf(m_conflicts, [this](const Conflict &conflict) { + return m_path == conflict.path; + }); + } + currentValue = newValue; + } + m_path.removeLast(); + } +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/loaderutils.h b/src/lib/corelib/loader/loaderutils.h new file mode 100644 index 000000000..4a4aadfb8 --- /dev/null +++ b/src/lib/corelib/loader/loaderutils.h @@ -0,0 +1,464 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +#include <language/filetags.h> +#include <language/forward_decls.h> +#include <language/item.h> +#include <language/moduleproviderinfo.h> +#include <language/propertydeclaration.h> +#include <language/qualifiedid.h> +#include <parser/qmljsengine_p.h> +#include <tools/codelocation.h> +#include <tools/filetime.h> +#include <tools/mutexdata.h> +#include <tools/joblimits.h> +#include <tools/pimpl.h> +#include <tools/set.h> +#include <tools/version.h> + +#include <QHash> +#include <QStringList> +#include <QVariant> + +#include <atomic> +#include <functional> +#include <memory> +#include <mutex> +#include <optional> +#include <shared_mutex> +#include <thread> +#include <utility> +#include <vector> + +namespace qbs { +class SetupProjectParameters; +namespace Internal { +class Evaluator; +class ItemPool; +class ItemReader; +class Logger; +class ProductContext; +class ProgressObserver; +class ProjectContext; +class ScriptEngine; + +using ModulePropertiesPerGroup = std::unordered_map<const Item *, QualifiedIdSet>; +using FileLocations = QHash<std::pair<QString, QString>, CodeLocation>; + +enum class Deferral { Allowed, NotAllowed }; +enum class ProductDependency { None, Single, Bulk }; + +class CancelException { }; + +template<typename DataType, typename MutexType = std::shared_mutex> +struct GuardedData { + DataType data; + mutable MutexType mutex; +}; + +class TimingData +{ +public: + TimingData &operator+=(const TimingData &other); + + qint64 dependenciesResolving = 0; + qint64 moduleProviders = 0; + qint64 moduleInstantiation = 0; + qint64 propertyMerging = 0; + qint64 groupsSetup = 0; + qint64 groupsResolving = 0; + qint64 preparingProducts = 0; + qint64 resolvingProducts = 0; + qint64 schedulingProducts = 0; + qint64 probes = 0; + qint64 propertyEvaluation = 0; + qint64 propertyChecking = 0; +}; + +class ItemReaderCache +{ +public: + class AstCacheEntry + { + public: + QString code; + QbsQmlJS::Engine engine; + QbsQmlJS::AST::UiProgram *ast = nullptr; + + bool addProcessingThread(); + void removeProcessingThread(); + + private: + MutexData<Set<std::thread::id>, std::recursive_mutex> m_processingThreads; + }; + + const Set<QString> &filesRead() const { return m_filesRead; } + AstCacheEntry &retrieveOrSetupCacheEntry(const QString &filePath, + const std::function<void(AstCacheEntry &)> &setup); + const QStringList &retrieveOrSetDirectoryEntries( + const QString &dir, const std::function<QStringList()> &findOnDisk); + +private: + Set<QString> m_filesRead; + MutexData<std::unordered_map<QString, std::optional<QStringList>>, std::mutex> m_directoryEntries; // TODO: Merge with module dir entries cache? + MutexData<std::unordered_map<QString, AstCacheEntry>, std::mutex> m_astCache; +}; + +class DependenciesContext +{ +public: + virtual ~DependenciesContext(); + virtual std::pair<ProductDependency, ProductContext *> pendingDependency() const = 0; + + bool dependenciesResolved = false; +}; + +class ProductContext +{ +public: + QString uniqueName() const; + QString displayName() const; + void handleError(const ErrorInfo &error); + bool dependenciesResolvingPending() const; + std::pair<ProductDependency, ProductContext *> pendingDependency() const; + + QString name; + QString buildDirectory; + Item *item = nullptr; + Item *scope = nullptr; + ProjectContext *project = nullptr; + std::unique_ptr<ProductContext> shadowProduct; + Item *mergedExportItem = nullptr; + std::vector<ProbeConstPtr> probes; + ModulePropertiesPerGroup modulePropertiesSetInGroups; + ErrorInfo delayedError; + QString profileName; + QString multiplexConfigurationId; + QVariantMap profileModuleProperties; // Tree-ified module properties from profile. + QVariantMap moduleProperties; // Tree-ified module properties from profile + overridden values. + std::optional<QVariantMap> providerConfig; + std::optional<QVariantMap> providerQbsModule; + QVariantMap defaultParameters; // In Export item. + QStringList searchPaths; + ResolvedProductPtr product; + TimingData timingData; + std::unique_ptr<DependenciesContext> dependenciesContext; + + // This is needed because complex cyclic dependencies that involve Depends.productTypes + // may only be detected after a product has already been fully resolved. + std::vector<std::pair<FileTags, CodeLocation>> bulkDependencies; + + // The keys are module prototypes, the values specify whether the module's + // condition is true for this product. + std::unordered_map<Item *, bool> modulePrototypeEnabledInfo; + + int dependsItemCount = -1; +}; + +class TopLevelProjectContext +{ +public: + TopLevelProjectContext() = default; + TopLevelProjectContext(const TopLevelProjectContext &) = delete; + TopLevelProjectContext &operator=(const TopLevelProjectContext &) = delete; + ~TopLevelProjectContext(); + + bool checkItemCondition(Item *item, Evaluator &evaluator); + QString sourceCodeForEvaluation(const JSSourceValueConstPtr &value); + ScriptFunctionPtr scriptFunctionValue(Item *item, const QString &name); + QString sourceCodeAsFunction(const JSSourceValueConstPtr &value, + const PropertyDeclaration &decl); + + void setCanceled() { m_canceled = true; } + void checkCancelation(); + bool isCanceled() const { return m_canceled; } + + int productCount() const { return m_productsByName.size(); } + + void addProductToHandle(const ProductContext &product) { m_productsToHandle.data << &product; } + void removeProductToHandle(const ProductContext &product); + bool isProductQueuedForHandling(const ProductContext &product) const; + int productsToHandleCount() const { return m_productsToHandle.data.size(); } + + void addDisabledItem(Item *item); + bool isDisabledItem(const Item *item) const; + + void setProgressObserver(ProgressObserver *observer); + ProgressObserver *progressObserver() const; + + void addProject(ProjectContext *project) { m_projects.push_back(project); } + const std::vector<ProjectContext *> &projects() const { return m_projects; } + + using QueuedErrors = MutexData<std::vector<ErrorInfo>, std::mutex>; + void addQueuedError(const ErrorInfo &error); + QueuedErrors::UniqueConstGuard queuedErrors() const { return m_queuedErrors.lock(); } + + void setProfileConfigs(const QVariantMap &profileConfigs) { m_profileConfigs = profileConfigs; } + void addProfileConfig(const QString &profileName, const QVariantMap &profileConfig); + const QVariantMap &profileConfigs() const { return m_profileConfigs; } + std::optional<QVariantMap> profileConfig(const QString &profileName) const; + + void addProduct(ProductContext &product); + void addProductByType(ProductContext &product, const FileTags &tags); + ProductContext *productWithNameAndConstraint( + const QString &name, const std::function<bool(ProductContext &)> &constraint); + std::vector<ProductContext *> productsWithNameAndConstraint( + const QString &name, const std::function<bool(ProductContext &)> &constraint); + std::vector<ProductContext *> productsWithTypeAndConstraint( + const FileTags &tags, const std::function<bool(ProductContext &)> &constraint); + std::vector<std::pair<ProductContext *, CodeLocation>> + finishedProductsWithBulkDependency(const FileTag &tag) const; + void registerBulkDependencies(ProductContext &product); + + void addProjectNameUsedInOverrides(const QString &name); + const Set<QString> &projectNamesUsedInOverrides() const; + + void addProductNameUsedInOverrides(const QString &name); + const Set<QString> &productNamesUsedInOverrides() const; + + void setBuildDirectory(const QString &buildDir) { m_buildDirectory = buildDir; } + const QString &buildDirectory() const { return m_buildDirectory; } + + void addMultiplexConfiguration(const QString &id, const QVariantMap &config); + QVariantMap multiplexConfiguration(const QString &id) const; + + void setLastResolveTime(const FileTime &time) { m_lastResolveTime = time; } + const FileTime &lastResolveTime() const { return m_lastResolveTime; } + + Set<QString> buildSystemFiles() const { return m_itemReaderCache.filesRead(); } + + std::lock_guard<std::mutex> moduleProvidersCacheLock(); + void setModuleProvidersCache(const ModuleProvidersCache &cache); + const ModuleProvidersCache &moduleProvidersCache() const { return m_moduleProvidersCache; } + ModuleProviderInfo *moduleProvider(const ModuleProvidersCacheKey &key); + ModuleProviderInfo &addModuleProvider(const ModuleProvidersCacheKey &key, + const ModuleProviderInfo &provider); + + void addParameterDeclarations(const Item *moduleProto, + const Item::PropertyDeclarationMap &decls); + Item::PropertyDeclarationMap parameterDeclarations(Item *moduleProto) const; + + void setParameters(const Item *moduleProto, const QVariantMap ¶meters); + QVariantMap parameters(Item *moduleProto) const; + + void addCodeLink(const QString &sourceFile, const CodeRange &sourceRange, + const CodeLocation &target); + CodeLinks codeLinks() const { return m_codeLinks.lock().get(); } + + // An empty string means no matching module directory was found. + QString findModuleDirectory(const QualifiedId &module, const QString &searchPath, + const std::function<QString()> &findOnDisk); + + QStringList getModuleFilesForDirectory(const QString &dir, + const std::function<QStringList()> &findOnDisk); + void removeModuleFileFromDirectoryCache(const QString &filePath); + + void addUnknownProfilePropertyError(const Item *moduleProto, const ErrorInfo &error); + const std::vector<ErrorInfo> &unknownProfilePropertyErrors(const Item *moduleProto) const; + + Item *getModulePrototype(const QString &filePath, const QString &profile, + const std::function<Item *()> &produce); + + void addLocalProfile(const QString &name, const QVariantMap &values, + const CodeLocation &location); + const QVariantMap localProfiles() { return m_localProfiles; } + void checkForLocalProfileAsTopLevelProfile(const QString &topLevelProfile); + + using ProbeFilter = std::function<bool(const ProbeConstPtr &)>; + std::lock_guard<std::mutex> probesCacheLock(); + void setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes); + void setOldProductProbes(const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes); + void addNewlyResolvedProbe(const ProbeConstPtr &probe); + void addProjectLevelProbe(const ProbeConstPtr &probe); + const std::vector<ProbeConstPtr> projectLevelProbes() const; + ProbeConstPtr findOldProjectProbe(const QString &id, const ProbeFilter &filter) const; + ProbeConstPtr findOldProductProbe(const QString &productName, const ProbeFilter &filter) const; + ProbeConstPtr findCurrentProbe(const CodeLocation &location, const ProbeFilter &filter) const; + void incrementProbesCount() { ++m_probesInfo.probesEncountered; } + void incrementReusedCurrentProbesCount() { ++m_probesInfo.probesCachedCurrent; } + void incrementReusedOldProbesCount() { ++m_probesInfo.probesCachedOld; } + void incrementRunProbesCount() { ++m_probesInfo.probesRun; } + int probesEncounteredCount() const { return m_probesInfo.probesEncountered; } + int probesRunCount() const { return m_probesInfo.probesRun; } + int reusedOldProbesCount() const { return m_probesInfo.probesCachedOld; } + int reusedCurrentProbesCount() const { return m_probesInfo.probesCachedCurrent; } + + TimingData &timingData() { return m_timingData; } + ItemReaderCache &itemReaderCache() { return m_itemReaderCache; } + + void incProductDeferrals() { ++m_productDeferrals; } + int productDeferrals() const { return m_productDeferrals; } + + void collectDataFromEngine(const ScriptEngine &engine); + + ItemPool &createItemPool(); + +private: + const ResolvedFileContextPtr &resolvedFileContext(const FileContextConstPtr &ctx); + + std::vector<ProjectContext *> m_projects; + GuardedData<Set<const ProductContext *>> m_productsToHandle; + std::multimap<QString, ProductContext *> m_productsByName; + MutexData<std::unordered_map<QStringView, QString>, std::mutex> m_sourceCode; + std::unordered_map<QString, QVariantMap> m_multiplexConfigsById; + MutexData<QHash<CodeLocation, ScriptFunctionPtr>, std::mutex> m_scriptFunctionMap; + MutexData<std::unordered_map<std::pair<QStringView, QStringList>, QString>, + std::mutex> m_scriptFunctions; + std::unordered_map<FileContextConstPtr, ResolvedFileContextPtr> m_fileContextMap; + Set<QString> m_projectNamesUsedInOverrides; + Set<QString> m_productNamesUsedInOverrides; + MutexData<Set<const Item *>> m_disabledItems; + QueuedErrors m_queuedErrors; + QString m_buildDirectory; + QVariantMap m_profileConfigs; + ProgressObserver *m_progressObserver = nullptr; + TimingData m_timingData; + ModuleProvidersCache m_moduleProvidersCache; + std::mutex m_moduleProvidersCacheMutex; + QVariantMap m_localProfiles; + ItemReaderCache m_itemReaderCache; + QHash<FileTag, std::vector<std::pair<ProductContext *, CodeLocation>>> m_reverseBulkDependencies; + + // For fast look-up when resolving Depends.productTypes. + // The contract is that it contains fully handled, error-free, enabled products. + MutexData<std::multimap<FileTag, ProductContext *>> m_productsByType; + + // The keys are module prototypes. + MutexData<std::unordered_map<const Item *, + Item::PropertyDeclarationMap>> m_parameterDeclarations; + MutexData<std::unordered_map<const Item *, QVariantMap>> m_parameters; + MutexData<std::unordered_map<const Item *, + std::vector<ErrorInfo>>> m_unknownProfilePropertyErrors; + + // The keys are search path + module name, the values are directories. + MutexData<QHash<std::pair<QString, QualifiedId>, std::optional<QString>>, + std::mutex> m_modulePathCache; + + // The keys are file paths, the values are module prototype items accompanied by a profile. + MutexData<std::unordered_map<QString, std::vector<std::pair<Item *, QString>>>, + std::mutex> m_modulePrototypes; + + MutexData<std::map<QString, std::optional<QStringList>>, + std::mutex> m_moduleFilesPerDirectory; + MutexData<CodeLinks> m_codeLinks; + + struct { + QHash<QString, std::vector<ProbeConstPtr>> oldProjectProbes; + QHash<QString, std::vector<ProbeConstPtr>> oldProductProbes; + QHash<CodeLocation, std::vector<ProbeConstPtr>> currentProbes; + std::vector<ProbeConstPtr> projectLevelProbes; + + quint64 probesEncountered = 0; + quint64 probesRun = 0; + quint64 probesCachedCurrent = 0; + quint64 probesCachedOld = 0; + } m_probesInfo; + std::mutex m_probesMutex; + + std::vector<std::unique_ptr<ItemPool>> m_itemPools; + + FileTime m_lastResolveTime; + + std::atomic_bool m_canceled = false; + int m_productDeferrals = 0; +}; + +class ProjectContext +{ +public: + QString name; + Item *item = nullptr; + Item *scope = nullptr; + TopLevelProjectContext *topLevelProject = nullptr; + ProjectContext *parent = nullptr; + std::vector<ProjectContext *> children; + std::vector<ProductContext> products; + std::vector<QStringList> searchPathsStack; + ResolvedProjectPtr project; + std::vector<FileTaggerConstPtr> fileTaggers; + std::vector<RulePtr> rules; + JobLimits jobLimits; + ResolvedModulePtr dummyModule; +}; + +class ModuleContext +{ +public: + ResolvedModulePtr module; + JobLimits jobLimits; +}; + +class LoaderState +{ +public: + LoaderState(const SetupProjectParameters ¶meters, TopLevelProjectContext &topLevelProject, + ItemPool &itemPool, ScriptEngine &engine, Logger logger); + ~LoaderState(); + + Evaluator &evaluator(); + ItemPool &itemPool(); + ItemReader &itemReader(); + Logger &logger(); + const SetupProjectParameters ¶meters() const; + TopLevelProjectContext &topLevelProject(); + +private: + class Private; + Pimpl<Private> d; +}; + +// List must be sorted by priority in ascending order. +[[nodiscard]] QVariantMap mergeDependencyParameters( + std::vector<Item::Module::ParametersWithPriority> &&candidates); +[[nodiscard]] QVariantMap mergeDependencyParameters(const QVariantMap &m1, const QVariantMap &m2); + +QString fullProductDisplayName(const QString &name, const QString &multiplexId); +void adjustParametersScopes(Item *item, Item *scope); +void resolveRule(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext); +void resolveJobLimit(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext); +void resolveFileTagger(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext); +const FileTag unknownFileTag(); + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/localprofiles.cpp b/src/lib/corelib/loader/localprofiles.cpp new file mode 100644 index 000000000..e72128fff --- /dev/null +++ b/src/lib/corelib/loader/localprofiles.cpp @@ -0,0 +1,148 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "localprofiles.h" + +#include "loaderutils.h" + +#include <language/evaluator.h> +#include <language/item.h> +#include <language/qualifiedid.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/translator.h> +#include <tools/profile.h> +#include <tools/scripttools.h> +#include <tools/stringconstants.h> + +namespace qbs::Internal { +class LocalProfiles +{ +public: + LocalProfiles(LoaderState &loaderState) : m_loaderState(loaderState) {} + + void collectProfiles(Item *productOrProject, Item *projectScope); + +private: + void handleProfile(Item *profileItem); + void evaluateProfileValues(const QualifiedId &namePrefix, Item *item, Item *profileItem, + QVariantMap &values); + LoaderState &m_loaderState; +}; + +void collectProfilesFromItems(Item *productOrProject, Item *projectScope, LoaderState &loaderState) +{ + LocalProfiles(loaderState).collectProfiles(productOrProject, projectScope); +} + +void LocalProfiles::handleProfile(Item *profileItem) +{ + QVariantMap values; + evaluateProfileValues(QualifiedId(), profileItem, profileItem, values); + const bool condition = values.take(StringConstants::conditionProperty()).toBool(); + if (!condition) + return; + const QString profileName = values.take(StringConstants::nameProperty()).toString(); + if (profileName.isEmpty()) + throw ErrorInfo(Tr::tr("Every Profile item must have a name"), profileItem->location()); + if (profileName == Profile::fallbackName()) { + throw ErrorInfo(Tr::tr("Reserved name '%1' cannot be used for an actual profile.") + .arg(profileName), profileItem->location()); + } + m_loaderState.topLevelProject().addLocalProfile(profileName, values, profileItem->location()); +} + +void LocalProfiles::evaluateProfileValues(const QualifiedId &namePrefix, Item *item, + Item *profileItem, QVariantMap &values) +{ + const Item::PropertyMap &props = item->properties(); + for (auto it = props.begin(); it != props.end(); ++it) { + QualifiedId name = namePrefix; + name << it.key(); + switch (it.value()->type()) { + case Value::ItemValueType: + evaluateProfileValues(name, std::static_pointer_cast<ItemValue>(it.value())->item(), + profileItem, values); + break; + case Value::VariantValueType: + values.insert(name.join(QLatin1Char('.')), + std::static_pointer_cast<VariantValue>(it.value())->value()); + break; + case Value::JSSourceValueType: + if (item != profileItem) + item->setScope(profileItem); + const ScopedJsValue sv(m_loaderState.evaluator().engine()->context(), + m_loaderState.evaluator().value(item, it.key())); + values.insert(name.join(QLatin1Char('.')), + getJsVariant(m_loaderState.evaluator().engine()->context(), sv)); + break; + } + } +} + +void LocalProfiles::collectProfiles(Item *productOrProject, Item *projectScope) +{ + Item * scope = productOrProject->type() == ItemType::Project ? projectScope : nullptr; + for (auto it = productOrProject->children().begin(); + it != productOrProject->children().end();) { + Item * const childItem = *it; + if (childItem->type() == ItemType::Profile) { + if (!scope) { + const ItemValuePtr itemValue = ItemValue::create(productOrProject); + scope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); + scope->setProperty(StringConstants::productVar(), itemValue); + scope->setFile(productOrProject->file()); + scope->setScope(projectScope); + } + childItem->setScope(scope); + try { + handleProfile(childItem); + } catch (const ErrorInfo &e) { + handlePropertyError(e, m_loaderState.parameters(), m_loaderState.logger()); + } + it = productOrProject->children().erase(it); // TODO: delete item and scope + } else { + if (childItem->type() == ItemType::Product) + collectProfiles(childItem, projectScope); + ++it; + } + } +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/localprofiles.h b/src/lib/corelib/loader/localprofiles.h new file mode 100644 index 000000000..7d9eee88e --- /dev/null +++ b/src/lib/corelib/loader/localprofiles.h @@ -0,0 +1,49 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs::Internal { +class Item; +class LoaderState; + +void collectProfilesFromItems(Item *productOrProject, Item *projectScope, LoaderState &loaderState); + +} // namespace qbs::Internal + diff --git a/src/lib/corelib/loader/moduleinstantiator.cpp b/src/lib/corelib/loader/moduleinstantiator.cpp new file mode 100644 index 000000000..1c0359217 --- /dev/null +++ b/src/lib/corelib/loader/moduleinstantiator.cpp @@ -0,0 +1,340 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "moduleinstantiator.h" + +#include "loaderutils.h" +#include "modulepropertymerger.h" + +#include <language/item.h> +#include <language/itempool.h> +#include <language/qualifiedid.h> +#include <language/value.h> +#include <logging/logger.h> +#include <logging/translator.h> +#include <tools/profiling.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <utility> + +namespace qbs::Internal { + +static std::pair<const Item *, Item *> +getOrSetModuleInstanceItem(Item *container, Item *moduleItem, const QualifiedId &moduleName, + const QString &id, bool replace, LoaderState &loaderState); + +class ModuleInstantiator +{ +public: + ModuleInstantiator(const InstantiationContext &context, LoaderState &loaderState) + : context(context), loaderState(loaderState) {} + + void instantiate(); + +private: + void overrideProperties(); + void setupScope(); + void exchangePlaceholderItem(Item *loadingItem, Item *moduleItemForItemValues); + + const InstantiationContext &context; + LoaderState &loaderState; +}; + +void ModuleInstantiator::instantiate() +{ + AccumulatingTimer timer(loaderState.parameters().logElapsedTime() + ? &context.product.timingData.moduleInstantiation : nullptr); + + // This part needs to be done only once per module and product, and only if the module + // was successfully loaded. + if (context.module && !context.alreadyLoaded) { + context.module->setType(ItemType::ModuleInstance); + overrideProperties(); + setupScope(); + } + + // This strange-looking code deals with the fact that our syntax cannot properly handle + // dependencies on several multiplexed variants of the same product. + // See getOrSetModuleInstanceItem() below for details. + Item * const moduleItemForItemValues + = context.moduleWithSameName ? context.moduleWithSameName + : context.module; + + // Now attach the module instance as an item value to the loading item, potentially + // evicting a previously attached placeholder item and merging its values into the instance. + // Note that we potentially do this twice, once for the actual loading item and once + // for the product item, if the two are different. The reason is this: + // For convenience, in the product item we allow attaching to properties from indirectly + // loaded modules. For instance: + // Product { + // Depends { name: "Qt.core" } + // cpp.cxxLanguageVersion: "c++20" // Works even though there is no Depends item for cpp + // } + // It's debatable whether that's a good feature, but it has been working (accidentally?) + // for a long time, and removing it now would break a lot of existing projects. + exchangePlaceholderItem(context.loadingItem, moduleItemForItemValues); + + if (!context.alreadyLoaded && context.product.item + && context.product.item != context.loadingItem) { + exchangePlaceholderItem(context.product.item, moduleItemForItemValues); + } +} + +void ModuleInstantiator::exchangePlaceholderItem(Item *loadingItem, Item *moduleItemForItemValues) +{ + // If we have a module item, set an item value pointing to it as a property on the loading item. + // Evict a possibly existing placeholder item, and return it to us, so we can merge its values + // into the instance. + const auto &[oldItem, newItem] = getOrSetModuleInstanceItem( + loadingItem, moduleItemForItemValues, context.moduleName, context.id, true, loaderState); + + // The new item always exists, even if we don't have a module item. In that case, the + // function created a placeholder item for us, which we then have to turn into a + // non-present module. + QBS_CHECK(newItem); + if (!moduleItemForItemValues) { + createNonPresentModule(loaderState.itemPool(), context.moduleName.toString(), + QLatin1String("not found"), newItem); + return; + } + + if (!moduleItemForItemValues->isPresentModule()) + return; + + // This will yield false negatives for the case where there is an invalid property attached + // for a module that is actually found by pkg-config via the fallback provider. + // However, this is extremely rare compared to the case where the presence of the fallback + // module simply indicates "not present". + if (moduleItemForItemValues->isFallbackModule()) + return; + + // If the old and the new items are the same, it means the existing item value already + // pointed to a module instance (rather than a placeholder). + // This can happen in two cases: + // a) Multiple identical Depends items on the same level (easily possible with inheritance). + // b) Dependencies to multiplexed variants of the same product + // (see getOrSetModuleInstanceItem() below for details). + if (oldItem == newItem) { + QBS_CHECK(oldItem->type() == ItemType::ModuleInstance); + QBS_CHECK(context.alreadyLoaded || context.exportingProduct); + return; + } + + // In all other cases, our request to set the module instance item must have been honored. + QBS_CHECK(newItem == moduleItemForItemValues); + + // If there was no placeholder item, we don't have to merge any values and are done. + if (!oldItem) + return; + + // If an item was replaced, then it must have been a placeholder. + QBS_CHECK(oldItem->type() == ItemType::ModuleInstancePlaceholder); + + // Prevent setting read-only properties. + for (auto it = oldItem->properties().cbegin(); it != oldItem->properties().cend(); ++it) { + const PropertyDeclaration &pd = moduleItemForItemValues->propertyDeclaration(it.key()); + if (pd.flags().testFlag(PropertyDeclaration::ReadOnlyFlag)) { + throw ErrorInfo(Tr::tr("Cannot set read-only property '%1'.").arg(pd.name()), + it.value()->location()); + } + } + + // Now merge the locally attached values into the actual module instance. + mergeFromLocalInstance(context.product, loadingItem, context.loadingName, oldItem, + moduleItemForItemValues, loaderState); + + // TODO: We'd like to delete the placeholder item here, because it's not + // being referenced anymore and there's a lot of them. However, this + // is not supported by ItemPool. Investigate the use of std::pmr. +} + +Item *retrieveModuleInstanceItem(Item *containerItem, const QualifiedId &name, + LoaderState &loaderState) +{ + return getOrSetModuleInstanceItem(containerItem, nullptr, name, {}, false, loaderState).second; +} + +Item *retrieveQbsItem(Item *containerItem, LoaderState &loaderState) +{ + return retrieveModuleInstanceItem(containerItem, StringConstants::qbsModule(), loaderState); +} + +void ModuleInstantiator::overrideProperties() +{ + // Users can override module properties on the command line with the + // modules.<module-name>.<property-name>:<value> syntax. + // For simplicity and backwards compatibility, qbs properties can also be given without + // the "modules." prefix, i.e. just qbs.<property-name>:<value>. + // In addition, users can override module properties just for certain products + // using the products.<product-name>.<module-name>.<property-name>:<value> syntax. + // Such product-specific overrides have higher precedence. + const QString fullName = context.moduleName.toString(); + const QString generalOverrideKey = QStringLiteral("modules.") + fullName; + const QString perProductOverrideKey = StringConstants::productsOverridePrefix() + + context.product.name + QLatin1Char('.') + fullName; + const SetupProjectParameters ¶meters = loaderState.parameters(); + Logger &logger = loaderState.logger(); + context.module->overrideProperties(parameters.overriddenValuesTree(), generalOverrideKey, + parameters, logger); + if (fullName == StringConstants::qbsModule()) { + context.module->overrideProperties(parameters.overriddenValuesTree(), fullName, parameters, + logger); + } + context.module->overrideProperties(parameters.overriddenValuesTree(), perProductOverrideKey, + parameters, logger); +} + +void ModuleInstantiator::setupScope() +{ + Item * const scope = Item::create(&loaderState.itemPool(), ItemType::Scope); + QBS_CHECK(context.module->file()); + scope->setFile(context.module->file()); + QBS_CHECK(context.product.project->scope); + context.product.project->scope->copyProperty(StringConstants::projectVar(), scope); + if (context.product.scope) + context.product.scope->copyProperty(StringConstants::productVar(), scope); + else + QBS_CHECK(context.moduleName.toString() == StringConstants::qbsModule()); // Dummy product. + + if (!context.module->id().isEmpty()) + scope->setProperty(context.module->id(), ItemValue::create(context.module)); + for (Item * const child : context.module->children()) { + child->setScope(scope); + if (!child->id().isEmpty()) + scope->setProperty(child->id(), ItemValue::create(child)); + } + context.module->setScope(scope); + + if (context.exportingProduct) { + QBS_CHECK(context.exportingProduct->type() == ItemType::Product); + + const auto exportingProductItemValue = ItemValue::create(context.exportingProduct); + scope->setProperty(QStringLiteral("exportingProduct"), exportingProductItemValue); + + const auto importingProductItemValue = ItemValue::create(context.product.item); + scope->setProperty(QStringLiteral("importingProduct"), importingProductItemValue); + + // FIXME: This looks wrong. Introduce exportingProject variable? + scope->setProperty(StringConstants::projectVar(), + ItemValue::create(context.exportingProduct->parent())); + + PropertyDeclaration pd(StringConstants::qbsSourceDirPropertyInternal(), + PropertyDeclaration::String, QString(), + PropertyDeclaration::PropertyNotAvailableInConfig); + context.module->setPropertyDeclaration(pd.name(), pd); + context.module->setProperty(pd.name(), context.exportingProduct->property( + StringConstants::sourceDirectoryProperty())); + } +} + +void instantiateModule(const InstantiationContext &context, LoaderState &loaderState) +{ + ModuleInstantiator(context, loaderState).instantiate(); +} + +// This important function deals with retrieving and setting (pseudo-)module instances from/on +// items. +// Use case 1: Suppose we resolve the dependency cpp in module Qt.core, which also contains +// property bindings for cpp such as "cpp.defines: [...]". +// The "cpp" part of this binding is represented by an ItemValue whose +// item is of type ModuleInstancePlaceholder, originally set up by ItemReaderASTVisitor. +// This function will be called with the actual cpp module item and will +// replace the placeholder item in the item value. It will also return +// the placeholder item for subsequent merging of its properties with the +// ones of the actual module (in ModulePropertyMerger::mergeFromLocalInstance()). +// If there were no cpp property bindings defined in Qt.core, then we'd still +// have to replace the placeholder item, because references to "cpp" on the +// right-hand-side of other properties must refer to the module item. +// This is the common use of this function as employed by resolveProdsucDependencies(). +// Note that if a product has dependencies on more than one variant of a multiplexed +// product, these dependencies are competing for the item value property name, +// i.e. this case is not properly supported by the syntax. You must therefore not +// export properties from multiplexed products that will be different between the +// variants. In this function, the condition manifests itself by a module instance +// being encountered instead of a module instance placeholder, in which case +// nothing is done at all. +// Use case 2: We inject a fake qbs module into a project item, so qbs properties set in profiles +// can be accessed in the project level. Doing this is discouraged, and the +// functionality is kept mostly for backwards compatibility. The moduleItem +// parameter is null in this case, and the item will be created by the function itself. +// Use case 3: A temporary qbs module is attached to a product during low-level setup, namely +// in product multiplexing and setting qbs.profile. +// Use case 4: Module propagation to the the Group level. +// In all cases, the first returned item is the existing one, and the second returned item +// is the new one. Depending on the use case, they might be null and might also be the same item. +std::pair<const Item *, Item *> getOrSetModuleInstanceItem( + Item *container, Item *moduleItem, const QualifiedId &moduleName, const QString &id, + bool replace, LoaderState &loaderState) +{ + Item *instance = container; + const QualifiedId itemValueName + = !id.isEmpty() ? QualifiedId::fromString(id) : moduleName; + for (int i = 0; i < itemValueName.size(); ++i) { + const QString &moduleNameSegment = itemValueName.at(i); + const ValuePtr v = instance->ownProperty(itemValueName.at(i)); + if (v && v->type() == Value::ItemValueType) { + ItemValue * const itemValue = std::static_pointer_cast<ItemValue>(v).get(); + instance = itemValue->item(); + if (i == itemValueName.size() - 1) { + if (replace && instance != moduleItem + && instance->type() == ItemType::ModuleInstancePlaceholder) { + if (!moduleItem) { + moduleItem = Item::create(&loaderState.itemPool(), + ItemType::ModuleInstancePlaceholder); + } + itemValue->setItem(moduleItem); + } + return {instance, itemValue->item()}; + } + } else { + Item *newItem = i < itemValueName.size() - 1 + ? Item::create(&loaderState.itemPool(), ItemType::ModulePrefix) + : moduleItem + ? moduleItem + : Item::create(&loaderState.itemPool(), + ItemType::ModuleInstancePlaceholder); + instance->setProperty(moduleNameSegment, ItemValue::create(newItem)); + instance = newItem; + } + } + return {nullptr, instance}; +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleinstantiator.h b/src/lib/corelib/loader/moduleinstantiator.h new file mode 100644 index 000000000..5e9ad2f44 --- /dev/null +++ b/src/lib/corelib/loader/moduleinstantiator.h @@ -0,0 +1,82 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +#include <QtGlobal> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +namespace qbs::Internal { +class Item; +class LoaderState; +class ProductContext; +class QualifiedId; + +class InstantiationContext { +public: + ProductContext &product; + Item * const loadingItem; + const QString &loadingName; + Item * const module; + Item * const moduleWithSameName; + Item * const exportingProduct; + const QualifiedId &moduleName; + const QString &id; + const bool alreadyLoaded; +}; + +// This function is responsible for setting up a proper module instance from a bunch of items: +// - Set the item type to ItemType::ModuleInstance (from Module or Export). +// - Apply possible command-line overrides for module properties. +// - Replace a possible module instance placeholder in the loading item with the actual instance +// and merge their values employing the ModulePropertyMerger. +// - Setting up the module instance scope. +void instantiateModule(const InstantiationContext &context, LoaderState &loaderState); + +// Helper functions for retrieving/setting module instance items for special purposes. +// Note that these will also create the respective item value if it does not exist yet. +Item *retrieveModuleInstanceItem(Item *containerItem, const QualifiedId &name, + LoaderState &loaderState); +Item *retrieveQbsItem(Item *containerItem, LoaderState &loaderState); + +} // namespace qbs::Internal + diff --git a/src/lib/corelib/loader/moduleloader.cpp b/src/lib/corelib/loader/moduleloader.cpp new file mode 100644 index 000000000..d26849655 --- /dev/null +++ b/src/lib/corelib/loader/moduleloader.cpp @@ -0,0 +1,410 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "moduleloader.h" + +#include "itemreader.h" +#include "loaderutils.h" +#include "moduleproviderloader.h" + +#include <api/languageinfo.h> +#include <language/evaluator.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/codelocation.h> +#include <tools/error.h> +#include <tools/fileinfo.h> +#include <tools/hostosinfo.h> +#include <tools/profiling.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <QDirIterator> +#include <QHash> + +#include <unordered_map> +#include <utility> + +namespace qbs::Internal { + +class ModuleLoader +{ +public: + ModuleLoader( + LoaderState &loaderState, + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName) + : m_loaderState(loaderState) + , m_product(product) + , m_dependsItemLocation(dependsItemLocation) + , m_moduleName(moduleName) + {} + + Item *load(); + +private: + std::pair<Item *, bool> loadModuleFile(const QString &moduleName, const QString &filePath); + Item *getModulePrototype(const QString &moduleName, const QString &filePath); + Item *createAndInitModuleItem(const QString &moduleName, const QString &filePath); + bool evaluateModuleCondition(Item *module, const QString &fullModuleName); + void checkForUnknownProfileProperties(const Item *module); + QString findModuleDirectory(const QString &searchPath); + QStringList findModuleDirectories(); + QStringList getModuleFilePaths(const QString &dir); + + LoaderState &m_loaderState; + ProductContext &m_product; + const CodeLocation &m_dependsItemLocation; + const QualifiedId &m_moduleName; +}; + +struct PrioritizedItem +{ + PrioritizedItem(Item *item, int priority, int searchPathIndex) + : item(item), priority(priority), searchPathIndex(searchPathIndex) { } + + Item * const item; + int priority = 0; + const int searchPathIndex; +}; + +static Item *chooseModuleCandidate(const std::vector<PrioritizedItem> &candidates, + const QString &moduleName) +{ + // TODO: This should also consider the version requirement. + + auto maxIt = std::max_element( + candidates.begin(), candidates.end(), + [] (const PrioritizedItem &a, const PrioritizedItem &b) { + if (a.priority < b.priority) + return true; + if (a.priority > b.priority) + return false; + return a.searchPathIndex > b.searchPathIndex; + }); + + size_t nmax = std::count_if( + candidates.begin(), candidates.end(), + [maxIt] (const PrioritizedItem &i) { + return i.priority == maxIt->priority && i.searchPathIndex == maxIt->searchPathIndex; + }); + + if (nmax > 1) { + ErrorInfo e(Tr::tr("There is more than one equally prioritized candidate for module '%1'.") + .arg(moduleName)); + for (size_t i = 0; i < candidates.size(); ++i) { + const auto candidate = candidates.at(i); + if (candidate.priority == maxIt->priority) { + //: The %1 denotes the number of the candidate. + e.append(Tr::tr("candidate %1").arg(i + 1), candidates.at(i).item->location()); + } + } + throw e; + } + + return maxIt->item; +} + +Item *searchAndLoadModuleFile( + LoaderState &loaderState, + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName) +{ + return ModuleLoader(loaderState, product, dependsItemLocation, moduleName).load(); +} + +Item *ModuleLoader::load() +{ + SearchPathsManager searchPathsManager(m_loaderState.itemReader()); + + QStringList existingPaths = findModuleDirectories(); + if (existingPaths.isEmpty()) { // no suitable names found, try to use providers + AccumulatingTimer providersTimer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.moduleProviders : nullptr); + auto result = ModuleProviderLoader(m_loaderState) + .executeModuleProviders(m_product, m_dependsItemLocation, m_moduleName); + if (result.searchPaths) { + qCDebug(lcModuleLoader) << "Re-checking for module" << m_moduleName.toString() + << "with newly added search paths from module provider"; + m_loaderState.itemReader().pushExtraSearchPaths(*result.searchPaths); + existingPaths = findModuleDirectories(); + } + } + + const QString fullName = m_moduleName.toString(); + bool triedToLoadModule = false; + std::vector<PrioritizedItem> candidates; + candidates.reserve(size_t(existingPaths.size())); + for (int i = 0; i < existingPaths.size(); ++i) { + const QStringList &moduleFileNames = getModuleFilePaths(existingPaths.at(i)); + for (const QString &filePath : moduleFileNames) { + const auto [module, triedToLoad] = loadModuleFile(fullName, filePath); + if (module) + candidates.emplace_back(module, 0, i); + if (!triedToLoad) + m_loaderState.topLevelProject().removeModuleFileFromDirectoryCache(filePath); + triedToLoadModule = triedToLoadModule || triedToLoad; + } + } + + if (candidates.empty()) + return nullptr; + + Item *moduleItem = nullptr; + if (candidates.size() == 1) { + moduleItem = candidates.at(0).item; + } else { + for (auto &candidate : candidates) { + ModuleItemLocker lock(*candidate.item); + candidate.priority = m_loaderState.evaluator() + .intValue(candidate.item, StringConstants::priorityProperty(), + candidate.priority); + } + moduleItem = chooseModuleCandidate(candidates, fullName); + } + + checkForUnknownProfileProperties(moduleItem); + return moduleItem; +} + +std::pair<Item *, bool> ModuleLoader::loadModuleFile(const QString &moduleName, + const QString &filePath) +{ + qCDebug(lcModuleLoader) << "loadModuleFile" << moduleName << "from" << filePath; + + Item * const module = getModulePrototype(moduleName, filePath); + if (!module) + return {nullptr, false}; + + const auto it = m_product.modulePrototypeEnabledInfo.find(module); + if (it != m_product.modulePrototypeEnabledInfo.end()) { + qCDebug(lcModuleLoader) << "prototype cache hit (level 2)"; + return {it->second ? module : nullptr, true}; + } + + if (!evaluateModuleCondition(module, moduleName)) { + qCDebug(lcModuleLoader) << "condition of module" << moduleName << "is false"; + m_product.modulePrototypeEnabledInfo.insert({module, false}); + return {nullptr, true}; + } + + m_product.modulePrototypeEnabledInfo.insert({module, true}); + return {module, true}; +} + +Item * ModuleLoader::getModulePrototype(const QString &moduleName, const QString &filePath) +{ + bool fromCache = true; + Item * const module = m_loaderState.topLevelProject().getModulePrototype( + filePath, m_product.profileName, [&] { + fromCache = false; + return createAndInitModuleItem(moduleName, filePath); + }); + + if (fromCache) + qCDebug(lcModuleLoader) << "prototype cache hit (level 1)"; + return module; +} + +Item *ModuleLoader::createAndInitModuleItem(const QString &moduleName, const QString &filePath) +{ + Item * const module = m_loaderState.itemReader().setupItemFromFile(filePath, {}); + if (module->type() != ItemType::Module) { + qCDebug(lcModuleLoader).nospace() + << "Alleged module " << moduleName << " has type '" + << module->typeName() << "', so it's not a module after all."; + return nullptr; + } + + // Not technically needed, but we want to keep the invariant in item.cpp. + ModuleItemLocker locker(*module); + + module->setProperty(StringConstants::nameProperty(), VariantValue::create(moduleName)); + if (moduleName == StringConstants::qbsModule()) { + module->setProperty(QStringLiteral("hostPlatform"), + VariantValue::create(HostOsInfo::hostOSIdentifier())); + module->setProperty(QStringLiteral("hostArchitecture"), + VariantValue::create(HostOsInfo::hostOSArchitecture())); + module->setProperty(QStringLiteral("libexecPath"), + VariantValue::create(m_loaderState.parameters().libexecPath())); + + const Version qbsVersion = LanguageInfo::qbsVersion(); + module->setProperty(QStringLiteral("versionMajor"), + VariantValue::create(qbsVersion.majorVersion())); + module->setProperty(QStringLiteral("versionMinor"), + VariantValue::create(qbsVersion.minorVersion())); + module->setProperty(QStringLiteral("versionPatch"), + VariantValue::create(qbsVersion.patchLevel())); + } else { + Item::PropertyDeclarationMap decls; + const auto &moduleChildren = module->children(); + for (Item *param : moduleChildren) { + if (param->type() == ItemType::Parameter) { + const auto ¶mDecls = param->propertyDeclarations(); + for (auto it = paramDecls.begin(); it != paramDecls.end(); ++it) + decls.insert(it.key(), it.value()); + } else if (param->type() == ItemType::Parameters) { + adjustParametersScopes(param, param); + Evaluator &evaluator = m_loaderState.evaluator(); + QVariantMap parameters = getJsVariant(evaluator.engine()->context(), + evaluator.scriptValue(param)).toMap(); + m_loaderState.topLevelProject().setParameters(module, parameters); + } + } + m_loaderState.topLevelProject().addParameterDeclarations(module, decls); + } + + // Module properties that are defined in the profile are used as default values. + // This is the reason we need to have different items per profile. + const QVariantMap profileModuleProperties + = m_product.profileModuleProperties.value(moduleName).toMap(); + for (auto it = profileModuleProperties.cbegin(); it != profileModuleProperties.cend(); ++it) { + if (Q_UNLIKELY(!module->hasProperty(it.key()))) { + m_loaderState.topLevelProject().addUnknownProfilePropertyError( + module, {Tr::tr("Unknown property: %1.%2").arg(moduleName, it.key())}); + continue; + } + const PropertyDeclaration decl = module->propertyDeclaration(it.key()); + VariantValuePtr v = VariantValue::create( + PropertyDeclaration::convertToPropertyType(it.value(), decl.type(), + QStringList(moduleName), it.key())); + v->markAsSetByProfile(); + module->setProperty(it.key(), v); + } + + return module; +} + +bool ModuleLoader::evaluateModuleCondition(Item *module, const QString &fullModuleName) +{ + ModuleItemLocker locker(*module); + + // Temporarily make the product's qbs module instance available, so the condition + // can use qbs.targetOS etc. + class TempQbsModuleProvider { + public: + TempQbsModuleProvider(const ProductContext &product, + Item *module, const QString &moduleName) + : m_module(module), m_needsQbsItem(moduleName != StringConstants::qbsModule()) + { + if (m_needsQbsItem) { + m_prevQbsItemValue = module->property(StringConstants::qbsModule()); + module->setProperty(StringConstants::qbsModule(), + product.item->property(StringConstants::qbsModule())); + } + } + ~TempQbsModuleProvider() + { + if (!m_needsQbsItem) + return; + if (m_prevQbsItemValue) + m_module->setProperty(StringConstants::qbsModule(), m_prevQbsItemValue); + else + m_module->removeProperty(StringConstants::qbsModule()); + } + private: + Item * const m_module; + ValuePtr m_prevQbsItemValue; + const bool m_needsQbsItem; + }; + + const TempQbsModuleProvider tempQbs(m_product, module, fullModuleName); + return m_loaderState.evaluator().boolValue(module, StringConstants::conditionProperty()); +} + +void ModuleLoader::checkForUnknownProfileProperties(const Item *module) +{ + const std::vector<ErrorInfo> &errors + = m_loaderState.topLevelProject().unknownProfilePropertyErrors(module); + if (errors.empty()) + return; + + ErrorInfo error(Tr::tr("Loading module '%1' for product '%2' failed due to invalid values " + "in profile '%3':") + .arg(m_moduleName.toString(), m_product.displayName(), m_product.profileName)); + for (const ErrorInfo &e : errors) + error.append(e.toString()); + handlePropertyError(error, m_loaderState.parameters(), m_loaderState.logger()); +} + +QString ModuleLoader::findModuleDirectory(const QString &searchPath) +{ + // isFileCaseCorrect is a very expensive call on macOS, so we cache the value for the + // modules and search paths we've already processed + return m_loaderState.topLevelProject().findModuleDirectory(m_moduleName, searchPath, + [&] { + QString dirPath = searchPath + QStringLiteral("/modules"); + for (const QString &moduleNamePart : m_moduleName) { + dirPath = FileInfo::resolvePath(dirPath, moduleNamePart); + if (!FileInfo::exists(dirPath) || !FileInfo::isFileCaseCorrect(dirPath)) + return QString(); + } + return dirPath; + }); +} + +QStringList ModuleLoader::findModuleDirectories() +{ + const QStringList &searchPaths = m_loaderState.itemReader().allSearchPaths(); + QStringList result; + result.reserve(searchPaths.size()); + for (const auto &path: searchPaths) { + const QString dirPath = findModuleDirectory(path); + if (!dirPath.isEmpty()) + result.append(dirPath); + } + return result; +} + +QStringList ModuleLoader::getModuleFilePaths(const QString &dir) +{ + return m_loaderState.topLevelProject().getModuleFilesForDirectory(dir, [&] { + QStringList moduleFiles; + QDirIterator dirIter(dir, StringConstants::qbsFileWildcards()); + while (dirIter.hasNext()) + moduleFiles += dirIter.next(); + return moduleFiles; + }); +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleloader.h b/src/lib/corelib/loader/moduleloader.h new file mode 100644 index 000000000..b07f01836 --- /dev/null +++ b/src/lib/corelib/loader/moduleloader.h @@ -0,0 +1,58 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs { +class CodeLocation; +namespace Internal { +class Item; +class LoaderState; +class ProductContext; +class QualifiedId; + +Item *searchAndLoadModuleFile( + LoaderState &loaderState, + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName); + +} // namespace Internal +} // namespace qbs + diff --git a/src/lib/corelib/loader/modulepropertymerger.cpp b/src/lib/corelib/loader/modulepropertymerger.cpp new file mode 100644 index 000000000..c9dd56ccf --- /dev/null +++ b/src/lib/corelib/loader/modulepropertymerger.cpp @@ -0,0 +1,323 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "modulepropertymerger.h" + +#include "loaderutils.h" + +#include <language/evaluator.h> +#include <language/item.h> +#include <language/value.h> +#include <logging/translator.h> +#include <tools/profiling.h> +#include <tools/set.h> +#include <tools/setupprojectparameters.h> + +#include <unordered_set> + +namespace qbs::Internal { +class ModulePropertyMerger +{ +public: + ModulePropertyMerger(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + + void mergeFromLocalInstance(Item *loadingItem, const QString &loadingName, + const Item *localInstance, Item *globalInstance); + void doFinalMerge(); + +private: + int compareValuePriorities(const ValueConstPtr &v1, const ValueConstPtr &v2); + ValuePtr mergeListValues(const ValuePtr ¤tHead, const ValuePtr &newElem); + void mergePropertyFromLocalInstance(Item *loadingItem, const QString &loadingName, + Item *globalInstance, const QString &name, + const ValuePtr &value); + bool doFinalMerge(Item *moduleItem); + bool doFinalMerge(const PropertyDeclaration &propertyDecl, ValuePtr &propertyValue); + + ProductContext & m_product; + LoaderState &m_loaderState; +}; + +void mergeFromLocalInstance(ProductContext &product, Item *loadingItem, const QString &loadingName, + const Item *localInstance, Item *globalInstance, + LoaderState &loaderState) +{ + ModulePropertyMerger(product, loaderState).mergeFromLocalInstance( + loadingItem, loadingName, localInstance, globalInstance); +} + +void doFinalMerge(ProductContext &product, LoaderState &loaderState) +{ + ModulePropertyMerger(product, loaderState).doFinalMerge(); +} + +void ModulePropertyMerger::mergeFromLocalInstance(Item *loadingItem, const QString &loadingName, + const Item *localInstance, Item *globalInstance) +{ + AccumulatingTimer t(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyMerging : nullptr); + + for (auto it = localInstance->properties().constBegin(); + it != localInstance->properties().constEnd(); ++it) { + mergePropertyFromLocalInstance(loadingItem, loadingName, globalInstance, it.key(), + it.value()); + } +} + +void ModulePropertyMerger::doFinalMerge() +{ + AccumulatingTimer t(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyMerging : nullptr); + + std::unordered_set<const Item *> itemsToInvalidate; + for (const Item::Module &module : m_product.item->modules()) { + if (doFinalMerge(module.item)) + itemsToInvalidate.insert(module.item); + } + + // For each module item, if it requires invalidation, we also add modules that depend on + // that module item, all the way up to the Product item. + std::unordered_set<const Item *> visitedItems; + const auto collectDependentItems = + [&itemsToInvalidate, &visitedItems](const Item *item, const auto &collect) -> bool + { + const bool alreadyInSet = itemsToInvalidate.count(item); + if (!visitedItems.insert(item).second) // item handled already + return alreadyInSet; + bool addItem = false; + for (const Item::Module &m : item->modules()) { + if (collect(m.item, collect)) + addItem = true; + } + if (addItem && !alreadyInSet) + itemsToInvalidate.insert(item); + return addItem || alreadyInSet; + }; + collectDependentItems(m_product.item, collectDependentItems); + for (const Item * const item : itemsToInvalidate) + m_loaderState.evaluator().clearCache(item); +} + +int ModulePropertyMerger::compareValuePriorities(const ValueConstPtr &v1, const ValueConstPtr &v2) +{ + QBS_CHECK(v1); + QBS_CHECK(v2); + QBS_CHECK(v1->scope() != v2->scope()); + QBS_CHECK(v1->type() == Value::JSSourceValueType || v2->type() == Value::JSSourceValueType); + + const int prio1 = v1->priority(m_product.item); + const int prio2 = v2->priority(m_product.item); + if (prio1 != prio2) + return prio1 - prio2; + const int prioDiff = v1->scopeName().compare(v2->scopeName()); // Sic! See 8ff1dd0044 + QBS_CHECK(prioDiff != 0); + return prioDiff; +} + +ValuePtr ModulePropertyMerger::mergeListValues(const ValuePtr ¤tHead, const ValuePtr &newElem) +{ + QBS_CHECK(newElem); + QBS_CHECK(!newElem->next()); + + if (!currentHead) + return !newElem->expired(m_product.item) ? newElem : newElem->next(); + + QBS_CHECK(!currentHead->expired(m_product.item)); + + if (newElem->expired(m_product.item)) + return currentHead; + + if (compareValuePriorities(currentHead, newElem) < 0) { + newElem->setNext(currentHead); + return newElem; + } + currentHead->setNext(mergeListValues(currentHead->next(), newElem)); + return currentHead; +} + +void ModulePropertyMerger::mergePropertyFromLocalInstance( + Item *loadingItem, const QString &loadingName, Item *globalInstance, + const QString &name, const ValuePtr &value) +{ + if (loadingItem->type() == ItemType::Project) { + throw ErrorInfo(Tr::tr("Module properties cannot be set in Project items."), + value->location()); + } + + const PropertyDeclaration decl = globalInstance->propertyDeclaration(name); + if (!decl.isValid()) { + if (value->type() == Value::ItemValueType || value->createdByPropertiesBlock()) + return; + throw ErrorInfo(Tr::tr("Property '%1' is not declared.") + .arg(name), value->location()); + } + if (const ErrorInfo error = decl.checkForDeprecation( + m_loaderState.parameters().deprecationWarningMode(), value->location(), + m_loaderState.logger()); error.hasError()) { + handlePropertyError(error, m_loaderState.parameters(), m_loaderState.logger()); + return; + } + if (value->setInternally()) { // E.g. qbs.architecture after multiplexing. + globalInstance->setProperty(decl.name(), value); + return; + } + QBS_CHECK(value->type() != Value::ItemValueType); + const ValuePtr globalVal = globalInstance->ownProperty(decl.name()); + value->setScope(loadingItem, loadingName); + QBS_CHECK(globalVal); + + // Values set internally cannot be overridden by JS values. + // The same goes for values set on the command line. + // Note that in both cases, there is no merging for list properties: The override is absolute. + if (globalVal->setInternally() || globalVal->setByCommandLine()) + return; + + QBS_CHECK(value->type() == Value::JSSourceValueType); + + if (decl.isScalar()) { + QBS_CHECK(!globalVal->expired(m_product.item)); + QBS_CHECK(!value->expired(m_product.item)); + if (compareValuePriorities(globalVal, value) < 0) { + value->setCandidates(globalVal->candidates()); + globalVal->setCandidates({}); + value->addCandidate(globalVal); + globalInstance->setProperty(decl.name(), value); + } else { + globalVal->addCandidate(value); + } + } else { + if (const ValuePtr &newChainStart = mergeListValues(globalVal, value); + newChainStart != globalVal) { + globalInstance->setProperty(decl.name(), newChainStart); + } + } +} + +bool ModulePropertyMerger::doFinalMerge(Item *moduleItem) +{ + if (!moduleItem->isPresentModule()) + return false; + bool mustInvalidateCache = false; + for (auto it = moduleItem->properties().begin(); it != moduleItem->properties().end(); ++it) { + if (doFinalMerge(moduleItem->propertyDeclaration(it.key()), it.value())) + mustInvalidateCache = true; + } + return mustInvalidateCache; +} + +bool ModulePropertyMerger::doFinalMerge(const PropertyDeclaration &propertyDecl, + ValuePtr &propertyValue) +{ + if (propertyValue->type() == Value::VariantValueType) { + QBS_CHECK(!propertyValue->next()); + return false; + } + if (!propertyDecl.isValid()) + return false; // Caught later by dedicated checker. + propertyValue->resetPriority(); + if (propertyDecl.isScalar()) { + if (propertyValue->candidates().empty()) + return false; + std::pair<int, std::vector<ValuePtr>> candidatesWithHighestPrio; + candidatesWithHighestPrio.first = propertyValue->priority(m_product.item); + candidatesWithHighestPrio.second.push_back(propertyValue); + for (const ValuePtr &v : propertyValue->candidates()) { + const int prio = v->priority(m_product.item); + if (prio < candidatesWithHighestPrio.first) + continue; + if (prio > candidatesWithHighestPrio.first) { + candidatesWithHighestPrio.first = prio; + candidatesWithHighestPrio.second = {v}; + continue; + } + candidatesWithHighestPrio.second.push_back(v); + } + ValuePtr chosenValue = candidatesWithHighestPrio.second.front(); + if (int(candidatesWithHighestPrio.second.size()) > 1) { + ErrorInfo error(Tr::tr("Conflicting scalar values for property '%1'.") + .arg(propertyDecl.name())); + error.append({}, chosenValue->location()); + QBS_CHECK(chosenValue->type() == Value::JSSourceValueType); + QStringView sourcCode = static_cast<JSSourceValue *>( + chosenValue.get())->sourceCode(); + for (int i = 1; i < int(candidatesWithHighestPrio.second.size()); ++i) { + const ValuePtr &v = candidatesWithHighestPrio.second.at(i); + QBS_CHECK(v->type() == Value::JSSourceValueType); + + // Note that this is a bit silly: The source code could still evaluate to + // different values in the end. + if (static_cast<JSSourceValue *>(v.get())->sourceCode() != sourcCode) + error.append({}, v->location()); + } + if (error.items().size() > 2) + m_loaderState.logger().printWarning(error); + } + + if (propertyValue == chosenValue) + return false; + std::vector<ValuePtr> candidates = propertyValue->candidates(); + candidates.erase(std::find(candidates.begin(), candidates.end(), chosenValue)); + chosenValue->setCandidates(candidates); + chosenValue->addCandidate(propertyValue); + propertyValue->setCandidates({}); + propertyValue = chosenValue; + return true; + } + if (!propertyValue->next()) + return false; + std::vector<ValuePtr> singleValuesBefore; + for (ValuePtr current = propertyValue; current;) { + singleValuesBefore.push_back(current); + const ValuePtr next = current->next(); + if (next) + current->setNext({}); + current = next; + } + ValuePtr newValue; + for (const ValuePtr &v : singleValuesBefore) + newValue = mergeListValues(newValue, v); + std::vector<ValuePtr> singleValuesAfter; + for (ValuePtr current = propertyValue; current; current = current->next()) + singleValuesAfter.push_back(current); + propertyValue = newValue; + return singleValuesBefore != singleValuesAfter; +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/modulepropertymerger.h b/src/lib/corelib/loader/modulepropertymerger.h new file mode 100644 index 000000000..a6643216f --- /dev/null +++ b/src/lib/corelib/loader/modulepropertymerger.h @@ -0,0 +1,87 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +#include <QtGlobal> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +namespace qbs::Internal { +class Item; +class LoaderState; +class ProductContext; + +// This module comprises functions for collecting values attached to module properties +// in different contexts. +// For example, in the Qt.core module you will find a property binding such as this: +// cpp.defines: "QT_CORE_LIB" +// while in the Qt.widgets module, it will look like this: +// cpp.defines: "QT_WIDGETS_LIB" +// A product with a dependency on both these modules will end up with a value of +// ["QT_WIDGETS_LIB", "QT_CORE_LIB"], plus potentially other defines set elsewhere. +// Each of these values is assigned a priority that roughly corresponds to the "level" at which +// the module containing the property binding resides in the dependency hierarchy. +// For list properties, the priorities determine the order of the respecive values in the +// final array, for scalar values they determine which one survives. Different scalar values +// with the same priority trigger a warning message. +// Since the right-hand side of a binding can refer to properties of the surrounding context, +// each such value gets its own scope. + +// This function is called when a module is loaded via a Depends item. +// loadingItem is the product or module containing the Depends item. +// loadingName is the name of that module. It is used as a tie-breaker for list property values +// with equal priority. +// localInstance is the module instance placeholder in the ItemValue of a property binding, +// i.e. the "cpp" in "cpp.defines". +// globalInstance is the actual module into which the properties from localInstance get merged. +void mergeFromLocalInstance(ProductContext &product, Item *loadingItem, + const QString &loadingName, const Item *localInstance, + Item *globalInstance, LoaderState &loaderState); + +// This function is called after all dependencies have been resolved. It uses its global +// knowledge of module priorities to potentially adjust the order of list values or +// favor different scalar values. It can also remove previously merged-in values again; +// this can happen if a module fails to load after it already merged some values, or +// if it fails validation in the end. +void doFinalMerge(ProductContext &product, LoaderState &loaderState); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleproviderloader.cpp b/src/lib/corelib/loader/moduleproviderloader.cpp new file mode 100644 index 000000000..d0789facd --- /dev/null +++ b/src/lib/corelib/loader/moduleproviderloader.cpp @@ -0,0 +1,416 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Copyright (C) 2021 Ivan Komissarov (abbapoh@gmail.com) +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "moduleproviderloader.h" + +#include "itemreader.h" +#include "probesresolver.h" + +#include <language/builtindeclarations.h> +#include <language/evaluator.h> +#include <language/item.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/fileinfo.h> +#include <tools/jsliterals.h> +#include <tools/scripttools.h> +#include <tools/setupprojectparameters.h> +#include <tools/stlutils.h> +#include <tools/stringconstants.h> + +#include <QtCore/qcryptographichash.h> +#include <QtCore/qjsondocument.h> +#include <QtCore/qjsonobject.h> + +namespace qbs { +namespace Internal { + +static QString getConfigHash(const QVariantMap& config) +{ + QJsonDocument doc; + doc.setObject(QJsonObject::fromVariantMap(config)); + return QString::fromLatin1( + QCryptographicHash::hash(doc.toJson(), QCryptographicHash::Sha1).toHex().left(16)); +} + +ModuleProviderLoader::ModuleProviderLoader(LoaderState &loaderState) + : m_loaderState(loaderState) {} + +ModuleProviderLoader::ModuleProviderResult ModuleProviderLoader::executeModuleProviders( + ProductContext &productContext, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName) +{ + ModuleProviderLoader::ModuleProviderResult result; + try { + std::vector<Provider> providersToRun; + qCDebug(lcModuleLoader) << "Module" << moduleName.toString() + << "not found, checking for module providers"; + const auto providerNames = getModuleProviders(productContext.item); + if (providerNames) { + providersToRun = transformed<std::vector<Provider>>(*providerNames, [](const auto &name) { + return Provider{name, ModuleProviderLookup::Named}; }); + } else { + for (QualifiedId providerName = moduleName; !providerName.empty(); + providerName.pop_back()) { + providersToRun.push_back({providerName, ModuleProviderLookup::Scoped}); + } + } + result = executeModuleProvidersHelper( + productContext, dependsItemLocation, moduleName, providersToRun); + } catch (const ErrorInfo &error) { + auto ei = error; + ei.prepend( + Tr::tr("Error executing provider for module '%1':").arg(moduleName.toString()), + dependsItemLocation); + productContext.handleError(ei); + } + return result; +} + +ModuleProviderLoader::ModuleProviderResult ModuleProviderLoader::executeModuleProvidersHelper( + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, + const std::vector<Provider> &providers) +{ + if (providers.empty()) + return {}; + QStringList allSearchPaths; + ModuleProviderResult result; + setupModuleProviderConfig(product); + const auto qbsModule = evaluateQbsModule(product); + for (const auto &[name, lookupType] : providers) { + const auto &[info, fromCache] = findOrCreateProviderInfo( + product, dependsItemLocation, moduleName, name, lookupType, qbsModule); + if (info.providerFile.isEmpty()) { + if (lookupType == ModuleProviderLookup::Named) + throw ErrorInfo(Tr::tr("Unknown provider '%1'").arg(name.toString())); + continue; + } + if (fromCache) + qCDebug(lcModuleLoader) << "Re-using provider" << name.toString() << "from cache"; + + result.providerFound = true; + if (info.searchPaths.empty()) { + qCDebug(lcModuleLoader) + << "Module provider did run, but did not set up any modules."; + continue; + } + qCDebug(lcModuleLoader) << "Module provider added" << info.searchPaths.size() + << "new search path(s)"; + + allSearchPaths << info.searchPaths; + } + if (allSearchPaths.isEmpty()) + return result; + + result.searchPaths = std::move(allSearchPaths); + + return result; +} + +std::pair<const ModuleProviderInfo &, bool /*fromCache*/> +ModuleProviderLoader::findOrCreateProviderInfo( + ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const QualifiedId &name, ModuleProviderLookup lookupType, + const QVariantMap &qbsModule) +{ + QBS_CHECK(product.providerConfig); + const QVariantMap config = product.providerConfig->value(name.toString()).toMap(); + std::lock_guard lock(m_loaderState.topLevelProject().moduleProvidersCacheLock()); + ModuleProvidersCacheKey cacheKey{name.toString(), {}, config, qbsModule, int(lookupType)}; + // TODO: get rid of non-eager providers and eliminate following if-logic + // first, try to find eager provider (stored with an empty module name) + if (ModuleProviderInfo *provider = m_loaderState.topLevelProject().moduleProvider(cacheKey)) + return {*provider, true}; + // second, try to find non-eager provider for a specific module name + std::get<1>(cacheKey) = moduleName.toString(); // override moduleName + if (ModuleProviderInfo *provider = m_loaderState.topLevelProject().moduleProvider(cacheKey)) + return {*provider, true}; + bool isEager = false; + ModuleProviderInfo info; + info.name = name; + info.config = config; + info.providerFile = findModuleProviderFile(name, lookupType); + if (!info.providerFile.isEmpty()) { + qCDebug(lcModuleLoader) << "Running provider" << name << "at" << info.providerFile; + std::tie(info.searchPaths, isEager) = evaluateModuleProvider( + product, + dependsItemLocation, + moduleName, + name, + info.providerFile, + config, + qbsModule); + info.transientOutput = m_loaderState.parameters().dryRun(); + } + std::get<1>(cacheKey) = isEager ? QString() : moduleName.toString(); + return {m_loaderState.topLevelProject().addModuleProvider(cacheKey, info), false}; +} + +void ModuleProviderLoader::setupModuleProviderConfig(ProductContext &product) +{ + if (product.providerConfig) + return; + QVariantMap providerConfig; + const ItemValueConstPtr configItemValue = + product.item->itemProperty(StringConstants::moduleProviders(), m_loaderState.itemPool()); + if (configItemValue) { + const std::function<void(const Item *, QualifiedId)> collectMap + = [this, &providerConfig, &collectMap](const Item *item, const QualifiedId &name) { + const Item::PropertyMap &props = item->properties(); + for (auto it = props.begin(); it != props.end(); ++it) { + QVariant value; + switch (it.value()->type()) { + case Value::ItemValueType: { + const auto childItem = static_cast<ItemValue *>(it.value().get())->item(); + childItem->setScope(item->scope()); + collectMap(childItem, QualifiedId(name) << it.key()); + continue; + } + case Value::JSSourceValueType: { + it.value()->setScope(item->scope(), {}); + const ScopedJsValue sv(m_loaderState.evaluator().engine()->context(), + m_loaderState.evaluator().value(item, it.key())); + value = getJsVariant(m_loaderState.evaluator().engine()->context(), sv); + break; + } + case Value::VariantValueType: + value = static_cast<VariantValue *>(it.value().get())->value(); + break; + } + QVariantMap m = providerConfig.value(name.toString()).toMap(); + m.insert(it.key(), value); + providerConfig.insert(name.toString(), m); + } + }; + configItemValue->item()->setScope(product.item); + collectMap(configItemValue->item(), QualifiedId()); + } + for (auto it = product.moduleProperties.begin(); it != product.moduleProperties.end(); ++it) { + if (!it.key().startsWith(QStringLiteral("moduleProviders."))) + continue; + const QString provider = it.key().mid(QStringLiteral("moduleProviders.").size()); + const QVariantMap providerConfigFromBuildConfig = it.value().toMap(); + if (providerConfigFromBuildConfig.empty()) + continue; + QVariantMap currentMapForProvider = providerConfig.value(provider).toMap(); + for (auto propIt = providerConfigFromBuildConfig.begin(); + propIt != providerConfigFromBuildConfig.end(); ++propIt) { + currentMapForProvider.insert(propIt.key(), propIt.value()); + } + providerConfig.insert(provider, currentMapForProvider); + } + product.providerConfig = providerConfig; +} + +std::optional<std::vector<QualifiedId>> ModuleProviderLoader::getModuleProviders(Item *item) +{ + while (item) { + const auto providers = m_loaderState.evaluator().optionalStringListValue( + item, StringConstants::qbsModuleProviders()); + if (providers) { + return transformed<std::vector<QualifiedId>>(*providers, [](const auto &provider) { + return QualifiedId::fromString(provider); }); + } + item = item->parent(); + } + return std::nullopt; +} + +QString ModuleProviderLoader::findModuleProviderFile( + const QualifiedId &name, ModuleProviderLookup lookupType) +{ + for (const QString &path : m_loaderState.itemReader().allSearchPaths()) { + QString fullPath = FileInfo::resolvePath(path, QStringLiteral("module-providers")); + switch (lookupType) { + case ModuleProviderLookup::Named: { + const auto result = + FileInfo::resolvePath(fullPath, name.toString() + QStringLiteral(".qbs")); + if (FileInfo::exists(result)) { + fullPath = result; + break; + } + [[fallthrough]]; + } + case ModuleProviderLookup::Scoped: + for (const QString &component : name) + fullPath = FileInfo::resolvePath(fullPath, component); + fullPath = FileInfo::resolvePath(fullPath, QStringLiteral("provider.qbs")); + break; + } + if (!FileInfo::exists(fullPath)) { + qCDebug(lcModuleLoader) << "No module provider found at" << fullPath; + continue; + } + return fullPath; + } + return {}; +} + +QVariantMap ModuleProviderLoader::evaluateQbsModule(ProductContext &product) const +{ + if (product.providerQbsModule) + return *product.providerQbsModule; + const QString properties[] = { + QStringLiteral("sysroot"), + QStringLiteral("toolchain"), + }; + const auto qbsItemValue = std::static_pointer_cast<ItemValue>( + product.item->property(StringConstants::qbsModule())); + QVariantMap result; + for (const auto &property : properties) { + const ScopedJsValue val(m_loaderState.evaluator().engine()->context(), + m_loaderState.evaluator().value(qbsItemValue->item(), property)); + auto value = getJsVariant(m_loaderState.evaluator().engine()->context(), val); + if (!value.isValid()) + continue; + + // The xcode module sets qbs.sysroot; the resulting value is bogus before the probes + // have run. + if (property == QLatin1String("sysroot") && !FileInfo::isAbsolute(value.toString())) + continue; + + result[property] = std::move(value); + } + return *(product.providerQbsModule = result); +} + +Item *ModuleProviderLoader::createProviderScope( + const ProductContext &product, const QVariantMap &qbsModule) +{ + const auto qbsItemValue = std::static_pointer_cast<ItemValue>( + product.item->property(StringConstants::qbsModule())); + + Item *fakeQbsModule = Item::create(&m_loaderState.itemPool(), ItemType::Scope); + + for (auto it = qbsModule.begin(), end = qbsModule.end(); it != end; ++it) { + fakeQbsModule->setProperty(it.key(), VariantValue::create(it.value())); + } + + Item *scope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); + scope->setFile(qbsItemValue->item()->file()); + scope->setProperty(StringConstants::qbsModule(), ItemValue::create(fakeQbsModule)); + return scope; +} + +ModuleProviderLoader::EvaluationResult ModuleProviderLoader::evaluateModuleProvider( + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, + const QualifiedId &name, + const QString &providerFile, + const QVariantMap &moduleConfig, + const QVariantMap &qbsModule) +{ + qCDebug(lcModuleLoader) << "Instantiating module provider at" << providerFile; + const QString projectBuildDir = product.project->item->variantProperty( + StringConstants::buildDirectoryProperty())->value().toString(); + const QString searchPathBaseDir = ModuleProviderInfo::outputDirPath(projectBuildDir, name); + + // include qbs module into hash + auto jsConfig = moduleConfig; + jsConfig[StringConstants::qbsModule()] = qbsModule; + + QString outputBaseDir = searchPathBaseDir + QLatin1Char('/') + getConfigHash(jsConfig); + Item * const providerItem = m_loaderState.itemReader().setupItemFromFile( + providerFile, dependsItemLocation); + if (providerItem->type() != ItemType::ModuleProvider) { + throw ErrorInfo(Tr::tr("File '%1' declares an item of type '%2', " + "but '%3' was expected.") + .arg(providerFile, providerItem->typeName(), + BuiltinDeclarations::instance().nameForType(ItemType::ModuleProvider))); + } + + Item * const scope = createProviderScope(product, qbsModule); + for (auto it = providerItem->properties().begin(); it != providerItem->properties().end(); ++it) + it.value()->setScope(scope, {}); + providerItem->setProperty( + StringConstants::nameProperty(), + VariantValue::create(name.toString())); + providerItem->setProperty( + QStringLiteral("outputBaseDir"), + VariantValue::create(outputBaseDir)); + providerItem->overrideProperties(moduleConfig, name, + m_loaderState.parameters(), m_loaderState.logger()); + + const bool isEager = m_loaderState.evaluator().boolValue( + providerItem, StringConstants::isEagerProperty()); + if (!isEager) { + providerItem->setProperty( + StringConstants::moduleNameProperty(), + VariantValue::create(moduleName.toString())); + } + + ProbesResolver(m_loaderState).resolveProbes(product, providerItem); + + EvalContextSwitcher contextSwitcher(m_loaderState.evaluator().engine(), + EvalContext::ModuleProvider); + + checkAllowedValues(providerItem); + auto searchPaths = m_loaderState.evaluator().stringListValue( + providerItem, QStringLiteral("relativeSearchPaths")); + auto prependBaseDir = [&outputBaseDir](const auto &path) { + return outputBaseDir + QLatin1Char('/') + path; + }; + std::transform(searchPaths.begin(), searchPaths.end(), searchPaths.begin(), prependBaseDir); + return {searchPaths, isEager}; +} + +void ModuleProviderLoader::checkAllowedValues(Item *providerItem) +{ + for (const auto &propertyDeclaration : providerItem->propertyDeclarations()) { + if (!propertyDeclaration.shouldCheckAllowedValues()) + continue; + const auto &name = propertyDeclaration.name(); + if (name == QStringLiteral("relativeSearchPaths")) + continue; + const auto value = m_loaderState.evaluator().variantValue(providerItem, name); + const auto propertyValue = providerItem->property(name); + propertyDeclaration.checkAllowedValues( + value, propertyValue->location(), name, m_loaderState); + } +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/moduleproviderloader.h b/src/lib/corelib/loader/moduleproviderloader.h new file mode 100644 index 000000000..471cfbb02 --- /dev/null +++ b/src/lib/corelib/loader/moduleproviderloader.h @@ -0,0 +1,112 @@ +/**************************************************************************** +** +** Copyright (C) 2021 The Qt Company Ltd. +** Copyright (C) 2021 Ivan Komissarov (abbapoh@gmail.com) +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef MODULEPROVIDERLOADER_H +#define MODULEPROVIDERLOADER_H + +#include "loaderutils.h" + +#include <language/forward_decls.h> +#include <language/moduleproviderinfo.h> + +#include <QtCore/qvariant.h> + +#include <optional> +#include <utility> +#include <vector> + +namespace qbs::Internal { +class Item; +class LoaderState; +class ProductContext; + +class ModuleProviderLoader +{ +public: + explicit ModuleProviderLoader(LoaderState &loaderState); + + struct ModuleProviderResult { + bool providerFound = false; + std::optional<QStringList> searchPaths; + }; + ModuleProviderResult executeModuleProviders( + ProductContext &productContext, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName); + +private: + enum class ModuleProviderLookup { Scoped, Named }; + struct Provider { + QualifiedId name; + ModuleProviderLookup lookup; + }; + ModuleProviderResult executeModuleProvidersHelper( + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, + const std::vector<Provider> &providers); + std::pair<const ModuleProviderInfo &, bool> + findOrCreateProviderInfo(ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const QualifiedId &name, + ModuleProviderLookup lookupType, const QVariantMap &qbsModule); + void setupModuleProviderConfig(ProductContext &product); + + std::optional<std::vector<QualifiedId>> getModuleProviders(Item *item); + + QString findModuleProviderFile(const QualifiedId &name, ModuleProviderLookup lookupType); + QVariantMap evaluateQbsModule(ProductContext &product) const; + Item *createProviderScope(const ProductContext &product, const QVariantMap &qbsModule); + using EvaluationResult = std::pair<QStringList, bool /*isEager*/>; + EvaluationResult evaluateModuleProvider( + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, + const QualifiedId &name, + const QString &providerFile, + const QVariantMap &moduleConfig, + const QVariantMap &qbsModule); + void checkAllowedValues(Item *providerItem); + + LoaderState &m_loaderState; +}; + +} // namespace qbs::Internal + +#endif // MODULEPROVIDERLOADER_H diff --git a/src/lib/corelib/loader/probesresolver.cpp b/src/lib/corelib/loader/probesresolver.cpp new file mode 100644 index 000000000..269e9c4b6 --- /dev/null +++ b/src/lib/corelib/loader/probesresolver.cpp @@ -0,0 +1,293 @@ +/**************************************************************************** +** +** Copyright (C) 2022 The Qt Company Ltd. +** Copyright (C) 2022 Raphaël Cotty <raphael.cotty@gmail.com> +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "probesresolver.h" + +#include "itemreader.h" +#include "loaderutils.h" + +#include <api/languageinfo.h> +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/item.h> +#include <language/language.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/logger.h> +#include <logging/translator.h> +#include <tools/profiling.h> +#include <tools/scripttools.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <quickjs.h> + +namespace qbs { +namespace Internal { + +static QString probeGlobalId(Item *probe) +{ + QString id; + + for (Item *obj = probe; obj; obj = obj->prototype()) { + if (!obj->id().isEmpty()) { + id = obj->id(); + break; + } + } + + if (id.isEmpty()) + return {}; + + QBS_CHECK(probe->file()); + return id + QLatin1Char('_') + probe->file()->filePath(); +} + +ProbesResolver::ProbesResolver(LoaderState &loaderState) : m_loaderState(loaderState) {} + +void ProbesResolver::resolveProbes(ProductContext &productContext, Item *item) +{ + AccumulatingTimer probesTimer(m_loaderState.parameters().logElapsedTime() + ? &productContext.timingData.probes : nullptr); + + EvalContextSwitcher evalContextSwitcher(m_loaderState.evaluator().engine(), + EvalContext::ProbeExecution); + for (Item * const child : item->children()) { + if (child->type() == ItemType::Probe) + resolveProbe(productContext, item, child); + } +} + +void ProbesResolver::resolveProbe(ProductContext &productContext, Item *parent, + Item *probe) +{ + qCDebug(lcModuleLoader) << "Resolving Probe at " << probe->location().toString(); + const QString &probeId = probeGlobalId(probe); + if (Q_UNLIKELY(probeId.isEmpty())) + throw ErrorInfo(Tr::tr("Probe.id must be set."), probe->location()); + const bool isProjectLevelProbe + = parent->type() == ItemType::Project + || productContext.name.startsWith(StringConstants::shadowProductPrefix()); + const JSSourceValueConstPtr configureScript + = probe->sourceProperty(StringConstants::configureProperty()); + QBS_CHECK(configureScript); + if (Q_UNLIKELY(configureScript->sourceCode() == StringConstants::undefinedValue())) + throw ErrorInfo(Tr::tr("Probe.configure must be set."), probe->location()); + using ProbeProperty = std::pair<QString, ScopedJsValue>; + std::vector<ProbeProperty> probeBindings; + Evaluator &evaluator = m_loaderState.evaluator(); + ScriptEngine * const engine = evaluator.engine(); + JSContext * const ctx = engine->context(); + QVariantMap initialProperties; + for (Item *obj = probe; obj; obj = obj->prototype()) { + const Item::PropertyMap &props = obj->properties(); + for (auto it = props.cbegin(); it != props.cend(); ++it) { + const QString &name = it.key(); + if (name == StringConstants::configureProperty()) + continue; + const JSValue value = evaluator.value(probe, name); + probeBindings.emplace_back(name, ScopedJsValue(ctx, value)); + if (name != StringConstants::conditionProperty()) + initialProperties.insert(name, getJsVariant(ctx, value)); + } + } + const bool condition = evaluator.boolValue(probe, StringConstants::conditionProperty()); + const QString &sourceCode = configureScript->sourceCode().toString(); + ProbeConstPtr resolvedProbe; + std::lock_guard lock(m_loaderState.topLevelProject().probesCacheLock()); + m_loaderState.topLevelProject().incrementProbesCount(); + if (isProjectLevelProbe) { + resolvedProbe = findOldProjectProbe(probeId, condition, initialProperties, sourceCode); + } else { + resolvedProbe = findOldProductProbe(productContext.uniqueName(), condition, + initialProperties, sourceCode); + } + if (!resolvedProbe) { + resolvedProbe = findCurrentProbe(probe->location(), condition, initialProperties); + if (resolvedProbe) { + qCDebug(lcModuleLoader) << "probe results cached from current run"; + m_loaderState.topLevelProject().incrementReusedCurrentProbesCount(); + } + } else { + qCDebug(lcModuleLoader) << "probe results cached from earlier run"; + m_loaderState.topLevelProject().incrementReusedOldProbesCount(); + } + ScopedJsValue configureScope(ctx, JS_UNDEFINED); + std::vector<QString> importedFilesUsedInConfigure; + if (!condition) { + qCDebug(lcModuleLoader) << "Probe disabled; skipping"; + } else if (!resolvedProbe) { + m_loaderState.topLevelProject().incrementRunProbesCount(); + qCDebug(lcModuleLoader) << "configure script needs to run"; + const Evaluator::FileContextScopes fileCtxScopes + = evaluator.fileContextScopes(configureScript->file()); + configureScope.setValue(engine->newObject()); + for (const ProbeProperty &b : probeBindings) + setJsProperty(ctx, configureScope, b.first, JS_DupValue(ctx, b.second)); + engine->clearRequestedProperties(); + ScopedJsValue sv(ctx, engine->evaluate(JsValueOwner::Caller, + configureScript->sourceCodeForEvaluation(), {}, 1, + {fileCtxScopes.fileScope, fileCtxScopes.importScope, configureScope})); + if (JsException ex = engine->checkAndClearException(configureScript->location())) + throw ex.toErrorInfo(); + importedFilesUsedInConfigure = engine->importedFilesUsedInScript(); + } else { + importedFilesUsedInConfigure = resolvedProbe->importedFilesUsed(); + } + QVariantMap properties; + VariantValuePtr storedValue; + QMap<QString, VariantValuePtr> storedValues; + for (const ProbeProperty &b : probeBindings) { + QVariant newValue; + if (resolvedProbe) { + newValue = resolvedProbe->properties().value(b.first); + } else { + if (condition) { + JSValue v = getJsProperty(ctx, configureScope, b.first); + const JSValue saved = v; + ScopedJsValue valueMgr(ctx, saved); + const PropertyDeclaration decl = probe->propertyDeclaration(b.first); + evaluator.convertToPropertyType(decl, probe->location(), v); + + // If the value was converted from scalar to array as per our convenience + // functionality, then the original value is now the only element of a + // newly allocated array and thus gets deleted via that array. + // The array itself is owned by the script engine, so we must stay out of + // memory management here. + if (v != saved) + valueMgr.setValue(JS_UNDEFINED); + + if (JsException ex = engine->checkAndClearException({})) + throw ex.toErrorInfo(); + newValue = getJsVariant(ctx, v); + // Special case, string and path lists are represented as js arrays but we don't + // want to make them const as we do for object lists. Converting to QStringList + // allows to distinguish between these two cases in ScriptEngine::asJsValue + if (newValue.userType() == QMetaType::QVariantList + && (decl.type() == PropertyDeclaration::StringList + || decl.type() == PropertyDeclaration::PathList)) { +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) + newValue.convert(QMetaType(QMetaType::QStringList)); +#else + newValue.convert(QMetaType::QStringList); +#endif + } + } else { + newValue = initialProperties.value(b.first); + } + } + if (!qVariantsEqual(newValue, getJsVariant(ctx, b.second))) { + if (!resolvedProbe) + storedValue = VariantValue::createStored(newValue); + else + storedValue = resolvedProbe->values().value(b.first); + + probe->setProperty(b.first, storedValue); + } + if (!resolvedProbe) { + properties.insert(b.first, newValue); + storedValues[b.first] = storedValue; + } + } + if (!resolvedProbe) { + resolvedProbe = Probe::create(probeId, probe->location(), condition, + sourceCode, properties, initialProperties, storedValues, + importedFilesUsedInConfigure); + m_loaderState.topLevelProject().addNewlyResolvedProbe(resolvedProbe); + } + if (isProjectLevelProbe) + m_loaderState.topLevelProject().addProjectLevelProbe(resolvedProbe); + else + productContext.probes << resolvedProbe; +} + +ProbeConstPtr ProbesResolver::findOldProjectProbe( + const QString &globalId, + bool condition, + const QVariantMap &initialProperties, + const QString &sourceCode) const +{ + if (m_loaderState.parameters().forceProbeExecution()) + return {}; + return m_loaderState.topLevelProject().findOldProjectProbe(globalId, + [&](const ProbeConstPtr &oldProbe) { + return probeMatches(oldProbe, condition, initialProperties, sourceCode, CompareScript::Yes); + }); +} + +ProbeConstPtr ProbesResolver::findOldProductProbe( + const QString &productName, + bool condition, + const QVariantMap &initialProperties, + const QString &sourceCode) const +{ + if (m_loaderState.parameters().forceProbeExecution()) + return {}; + return m_loaderState.topLevelProject().findOldProductProbe(productName, + [&](const ProbeConstPtr &oldProbe) { + return probeMatches(oldProbe, condition, initialProperties, sourceCode, CompareScript::Yes); + }); +} + +ProbeConstPtr ProbesResolver::findCurrentProbe( + const CodeLocation &location, + bool condition, + const QVariantMap &initialProperties) const +{ + return m_loaderState.topLevelProject().findCurrentProbe(location, + [&](const ProbeConstPtr &probe) { + return probeMatches(probe, condition, initialProperties, QString(), CompareScript::No); + }); +} + +bool ProbesResolver::probeMatches(const ProbeConstPtr &probe, bool condition, + const QVariantMap &initialProperties, const QString &configureScript, + CompareScript compareScript) const +{ + return probe->condition() == condition + && qVariantMapsEqual(probe->initialProperties(), initialProperties) + && (compareScript == CompareScript::No + || (probe->configureScript() == configureScript + && !probe->needsReconfigure(m_loaderState.topLevelProject().lastResolveTime()))); +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/probesresolver.h b/src/lib/corelib/loader/probesresolver.h new file mode 100644 index 000000000..4c49861d3 --- /dev/null +++ b/src/lib/corelib/loader/probesresolver.h @@ -0,0 +1,80 @@ +/**************************************************************************** +** +** Copyright (C) 2022 The Qt Company Ltd. +** Copyright (C) 2022 Raphaël Cotty <raphael.cotty@gmail.com> +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef PROBESRESOLVER_H +#define PROBESRESOLVER_H + +#include <language/forward_decls.h> +#include <tools/codelocation.h> + +#include <QString> + +namespace qbs::Internal { +class Item; +class LoaderState; +class ProductContext; + +class ProbesResolver +{ +public: + explicit ProbesResolver(LoaderState &loaderState); + void resolveProbes(ProductContext &productContext, Item *item); + +private: + ProbeConstPtr findOldProjectProbe(const QString &globalId, bool condition, + const QVariantMap &initialProperties, + const QString &sourceCode) const; + ProbeConstPtr findOldProductProbe(const QString &productName, bool condition, + const QVariantMap &initialProperties, + const QString &sourceCode) const; + ProbeConstPtr findCurrentProbe(const CodeLocation &location, bool condition, + const QVariantMap &initialProperties) const; + enum class CompareScript { No, Yes }; + bool probeMatches(const ProbeConstPtr &probe, bool condition, + const QVariantMap &initialProperties, const QString &configureScript, + CompareScript compareScript) const; + void resolveProbe(ProductContext &productContext, Item *parent, Item *probe); + + LoaderState &m_loaderState; +}; + +} // namespace qbs::Internal + +#endif // PROBESRESOLVER_H diff --git a/src/lib/corelib/loader/productitemmultiplexer.cpp b/src/lib/corelib/loader/productitemmultiplexer.cpp new file mode 100644 index 000000000..0adb3d7fd --- /dev/null +++ b/src/lib/corelib/loader/productitemmultiplexer.cpp @@ -0,0 +1,259 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "productitemmultiplexer.h" + +#include "loaderutils.h" +#include "moduleinstantiator.h" + +#include <language/evaluator.h> +#include <language/item.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/translator.h> +#include <tools/scripttools.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <QJsonDocument> + +#include <vector> + +// This module deals with product multiplexing over the various defined axes. +// For instance, a product with qbs.architectures: ["x86", "arm"] will get multiplexed into +// two products with qbs.architecture: "x86" and qbs.architecture: "arm", respectively. +namespace qbs::Internal { +namespace { +using MultiplexRow = std::vector<VariantValuePtr>; +using MultiplexTable = std::vector<MultiplexRow>; +class MultiplexInfo +{ +public: + std::vector<QString> properties; + MultiplexTable table; + bool aggregate = false; + VariantValuePtr multiplexedType; + + QString toIdString(size_t row, LoaderState &loaderState) const; +}; +} // namespace + +class ProductItemMultiplexer +{ +public: + ProductItemMultiplexer(const QString &productName, Item *productItem, Item *tempQbsModuleItem, + const std::function<void()> &dropTempQbsModule, LoaderState &loaderState) + : m_loaderState(loaderState), m_productName(productName), m_productItem(productItem), + m_tempQbsModuleItem(tempQbsModuleItem), m_dropTempQbsModule(dropTempQbsModule) {} + + QList<Item *> multiplex(); +private: + MultiplexInfo extractMultiplexInfo(); + MultiplexTable combine(const MultiplexTable &table, const MultiplexRow &values); + + LoaderState &m_loaderState; + const QString &m_productName; + Item * const m_productItem; + Item * const m_tempQbsModuleItem; + const std::function<void()> &m_dropTempQbsModule; +}; + +QList<Item *> ProductItemMultiplexer::multiplex() +{ + const auto multiplexInfo = extractMultiplexInfo(); + m_dropTempQbsModule(); + if (multiplexInfo.table.size() > 1) + m_productItem->setProperty(StringConstants::multiplexedProperty(), VariantValue::trueValue()); + VariantValuePtr productNameValue = VariantValue::create(m_productName); + Item *aggregator = multiplexInfo.aggregate ? m_productItem->clone(m_loaderState.itemPool()) + : nullptr; + QList<Item *> additionalProductItems; + std::vector<VariantValuePtr> multiplexConfigurationIdValues; + for (size_t row = 0; row < multiplexInfo.table.size(); ++row) { + Item *item = m_productItem; + const auto &mprow = multiplexInfo.table.at(row); + QBS_CHECK(mprow.size() == multiplexInfo.properties.size()); + if (row > 0) { + item = m_productItem->clone(m_loaderState.itemPool()); + additionalProductItems.push_back(item); + } + const QString multiplexConfigurationId = multiplexInfo.toIdString(row, m_loaderState); + const VariantValuePtr multiplexConfigurationIdValue + = VariantValue::create(multiplexConfigurationId); + if (multiplexInfo.table.size() > 1 || aggregator) { + multiplexConfigurationIdValues.push_back(multiplexConfigurationIdValue); + item->setProperty(StringConstants::multiplexConfigurationIdProperty(), + multiplexConfigurationIdValue); + } + if (multiplexInfo.multiplexedType) + item->setProperty(StringConstants::typeProperty(), multiplexInfo.multiplexedType); + for (size_t column = 0; column < mprow.size(); ++column) { + Item * const qbsItem = retrieveQbsItem(item, m_loaderState); + const QString &propertyName = multiplexInfo.properties.at(column); + const VariantValuePtr &mpvalue = mprow.at(column); + qbsItem->setProperty(propertyName, mpvalue); + } + } + + if (aggregator) { + additionalProductItems << aggregator; + + // Add dependencies to all multiplexed instances. + for (const auto &v : multiplexConfigurationIdValues) { + Item *dependsItem = Item::create(&m_loaderState.itemPool(), ItemType::Depends); + dependsItem->setProperty(StringConstants::nameProperty(), productNameValue); + dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), v); + dependsItem->setProperty(StringConstants::profilesProperty(), + VariantValue::create(QStringList())); + dependsItem->setFile(aggregator->file()); + dependsItem->setupForBuiltinType(m_loaderState.parameters().deprecationWarningMode(), + m_loaderState.logger()); + Item::addChild(aggregator, dependsItem); + } + } + + return additionalProductItems; +} + +MultiplexInfo ProductItemMultiplexer::extractMultiplexInfo() +{ + static const QString mpmKey = QStringLiteral("multiplexMap"); + + Evaluator &evaluator = m_loaderState.evaluator(); + JSContext * const ctx = evaluator.engine()->context(); + const ScopedJsValue multiplexMap(ctx, evaluator.value(m_tempQbsModuleItem, mpmKey)); + const QStringList multiplexByQbsProperties = evaluator.stringListValue( + m_productItem, StringConstants::multiplexByQbsPropertiesProperty()); + + MultiplexInfo multiplexInfo; + multiplexInfo.aggregate = evaluator.boolValue( + m_productItem, StringConstants::aggregateProperty()); + + const QString multiplexedType = evaluator.stringValue( + m_productItem, StringConstants::multiplexedTypeProperty()); + if (!multiplexedType.isEmpty()) + multiplexInfo.multiplexedType = VariantValue::create(multiplexedType); + + Set<QString> uniqueMultiplexByQbsProperties; + for (const QString &key : multiplexByQbsProperties) { + const QString mappedKey = getJsStringProperty(ctx, multiplexMap, key); + if (mappedKey.isEmpty()) + throw ErrorInfo(Tr::tr("There is no entry for '%1' in 'qbs.multiplexMap'.").arg(key)); + + if (!uniqueMultiplexByQbsProperties.insert(mappedKey).second) + continue; + + const ScopedJsValue arr(ctx, evaluator.value(m_tempQbsModuleItem, key)); + if (JS_IsUndefined(arr)) + continue; + if (!JS_IsArray(ctx, arr)) + throw ErrorInfo(Tr::tr("Property '%1' must be an array.").arg(key)); + + const quint32 arrlen = getJsIntProperty(ctx, arr, StringConstants::lengthProperty()); + if (arrlen == 0) + continue; + + MultiplexRow mprow; + mprow.reserve(arrlen); + QVariantList entriesForKey; + for (quint32 i = 0; i < arrlen; ++i) { + const ScopedJsValue sv(ctx, JS_GetPropertyUint32(ctx, arr, i)); + const QVariant value = getJsVariant(ctx, sv); + if (entriesForKey.contains(value)) + continue; + entriesForKey << value; + mprow.push_back(VariantValue::create(value)); + } + multiplexInfo.table = combine(multiplexInfo.table, mprow); + multiplexInfo.properties.push_back(mappedKey); + } + return multiplexInfo; +} + +MultiplexTable ProductItemMultiplexer::combine(const MultiplexTable &table, + const MultiplexRow &values) +{ + MultiplexTable result; + if (table.empty()) { + result.resize(values.size()); + for (size_t i = 0; i < values.size(); ++i) { + MultiplexRow row; + row.resize(1); + row[0] = values.at(i); + result[i] = row; + } + } else { + for (const auto &row : table) { + for (const auto &value : values) { + MultiplexRow newRow = row; + newRow.push_back(value); + result.push_back(newRow); + } + } + } + return result; +} + +QString MultiplexInfo::toIdString(size_t row, LoaderState &loaderState) const +{ + const auto &mprow = table.at(row); + QVariantMap multiplexConfiguration; + for (size_t column = 0; column < mprow.size(); ++column) { + const QString &propertyName = properties.at(column); + const VariantValuePtr &mpvalue = mprow.at(column); + multiplexConfiguration.insert(propertyName, mpvalue->value()); + } + QString id = QString::fromUtf8(QJsonDocument::fromVariant(multiplexConfiguration) + .toJson(QJsonDocument::Compact) + .toBase64()); + + // Cache for later use in multiplexIdToVariantMap() + loaderState.topLevelProject().addMultiplexConfiguration(id, multiplexConfiguration); + + return id; +} + +QList<Item *> multiplex(const QString &productName, Item *productItem, Item *tempQbsModuleItem, + const std::function<void ()> &dropTempQbsModule, LoaderState &loaderState) +{ + return ProductItemMultiplexer(productName, productItem, tempQbsModuleItem, dropTempQbsModule, + loaderState).multiplex(); +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productitemmultiplexer.h b/src/lib/corelib/loader/productitemmultiplexer.h new file mode 100644 index 000000000..e02e21793 --- /dev/null +++ b/src/lib/corelib/loader/productitemmultiplexer.h @@ -0,0 +1,64 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +#include <QList> + +#include <functional> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +namespace qbs::Internal { +class Item; +class LoaderState; + +// Checks whether the product item is to be multiplexed and returns the list of additional +// product items. In the normal, non-multiplex case, this list is empty. +QList<Item *> multiplex( + const QString &productName, + Item *productItem, + Item *tempQbsModuleItem, + const std::function<void()> &dropTempQbsModule, + LoaderState &loaderState + ); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productresolver.cpp b/src/lib/corelib/loader/productresolver.cpp new file mode 100644 index 000000000..a026a76d6 --- /dev/null +++ b/src/lib/corelib/loader/productresolver.cpp @@ -0,0 +1,1618 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "productresolver.h" + +#include "dependenciesresolver.h" +#include "groupshandler.h" +#include "loaderutils.h" +#include "modulepropertymerger.h" +#include "probesresolver.h" + +#include <jsextensions/jsextensions.h> +#include <jsextensions/moduleproperties.h> +#include <language/artifactproperties.h> +#include <language/builtindeclarations.h> +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/item.h> +#include <language/language.h> +#include <language/propertymapinternal.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/fileinfo.h> +#include <tools/jsliterals.h> +#include <tools/profiling.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <algorithm> + +namespace qbs::Internal { + +class PropertiesEvaluator +{ +public: + PropertiesEvaluator(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + + QVariantMap evaluateProperties(Item *item, bool lookupPrototype, bool checkErrors); + QVariantMap evaluateProperties(const Item *item, const Item *propertiesContainer, + const QVariantMap &tmplt, bool lookupPrototype, + bool checkErrors); + void evaluateProperty(const Item *item, const QString &propName, const ValuePtr &propValue, + QVariantMap &result, bool checkErrors); + +private: + ProductContext &m_product; + LoaderState &m_loaderState; +}; + +// Dependency resolving, Probe execution. +// Run for real products and shadow products. +class ProductResolverStage1 +{ +public: + ProductResolverStage1(ProductContext &product, Deferral deferral, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState), m_deferral(deferral) {} + void start(); + +private: + void resolveProbes(); + void resolveProbes(Item *item); + void runModuleProbes(const Item::Module &module); + void updateModulePresentState(const Item::Module &module); + bool validateModule(const Item::Module &module); + void handleModuleSetupError(const Item::Module &module, const ErrorInfo &error); + void checkPropertyDeclarations(); + void mergeDependencyParameters(); + void checkDependencyParameterDeclarations(const Item *productItem, + const QString &productName) const; + void collectCodeLinks(); + + ProductContext &m_product; + LoaderState &m_loaderState; + const Deferral m_deferral; +}; + +// Setting up ResolvedProduct, incuding property evaluation and handling Product child items. +// Run only for real products. +class ProductResolverStage2 +{ +public: + ProductResolverStage2(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + void start(); + +private: + void resolveProductFully(); + void createProductConfig(); + void resolveGroup(Item *item); + void resolveGroupFully(Item *item, bool isEnabled); + QVariantMap resolveAdditionalModuleProperties(const Item *group, + const QVariantMap ¤tValues); + SourceArtifactPtr createSourceArtifact(const QString &fileName, const GroupPtr &group, + bool wildcard, const CodeLocation &filesLocation, + ErrorInfo *errorInfo); + void resolveExport(Item *exportItem); + std::unique_ptr<ExportedItem> resolveExportChild(const Item *item, + const ExportedModule &module); + void setupExportedProperties(const Item *item, const QString &namePrefix, + std::vector<ExportedProperty> &properties); + QVariantMap evaluateModuleValues(Item *item, bool lookupPrototype = true); + + void resolveScanner(Item *item, ModuleContext &moduleContext); + void resolveModules(); + void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, + const QVariantMap ¶meters, JobLimits &jobLimits); + void applyFileTaggers(); + void finalizeArtifactProperties(); + void collectProductDependencies(); + + ProductContext &m_product; + LoaderState &m_loaderState; + GroupConstPtr m_currentGroup; + FileLocations m_sourceArtifactLocations; + PropertiesEvaluator m_propertiesEvaluator{m_product, m_loaderState}; + + using ArtifactPropertiesInfo = std::pair<ArtifactPropertiesPtr, std::vector<CodeLocation>>; + QHash<QStringList, ArtifactPropertiesInfo> m_artifactPropertiesPerFilter; +}; + +class ExportsResolver +{ +public: + ExportsResolver(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + void start(); + +private: + void resolveShadowProduct(); + void collectPropertiesForExportItem(Item *productModuleInstance); + void collectPropertiesForExportItem(const QualifiedId &moduleName, const ValuePtr &value, + Item *moduleInstance, QVariantMap &moduleProps); + void collectPropertiesForModuleInExportItem(const Item::Module &module); + void adaptExportedPropertyValues(); + void collectExportedProductDependencies(); + + ProductContext &m_product; + LoaderState &m_loaderState; + PropertiesEvaluator m_propertiesEvaluator{m_product, m_loaderState}; +}; + +void resolveProduct(ProductContext &product, Deferral deferral, LoaderState &loaderState) +{ + try { + ProductResolverStage1(product, deferral, loaderState).start(); + } catch (const ErrorInfo &err) { + if (err.isCancelException()) { + loaderState.topLevelProject().setCanceled(); + return; + } + product.handleError(err); + } + + if (product.dependenciesResolvingPending()) + return; + + if (product.name.startsWith(StringConstants::shadowProductPrefix())) + return; + + // TODO: The weird double-forwarded error handling can hopefully be simplified now. + try { + ProductResolverStage2(product, loaderState).start(); + } catch (const ErrorInfo &err) { + if (err.isCancelException()) { + loaderState.topLevelProject().setCanceled(); + return; + } + loaderState.topLevelProject().addQueuedError(err); + } +} + +void setupExports(ProductContext &product, LoaderState &loaderState) +{ + ExportsResolver(product, loaderState).start(); +} + +void ProductResolverStage1::start() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + topLevelProject.checkCancelation(); + + if (m_product.delayedError.hasError()) + return; + + resolveDependencies(m_product, m_deferral, m_loaderState); + QBS_CHECK(m_product.dependenciesContext); + if (!m_product.dependenciesContext->dependenciesResolved) + return; + if (m_product.delayedError.hasError()) + return; + + // Run probes for modules and product. + resolveProbes(); + + // After the probes have run, we can switch on the evaluator cache. + Evaluator &evaluator = m_loaderState.evaluator(); + FileTags fileTags = evaluator.fileTagsValue(m_product.item, StringConstants::typeProperty()); + EvalCacheEnabler cacheEnabler(&evaluator, evaluator.stringValue( + m_product.item, + StringConstants::sourceDirectoryProperty())); + + // Run module validation scripts. + for (const Item::Module &module : m_product.item->modules()) { + if (!validateModule(module)) + return; + fileTags += evaluator.fileTagsValue( + module.item, StringConstants::additionalProductTypesProperty()); + } + + // Disable modules that have been pulled in only by now-disabled modules. + // Note that this has to happen in the reverse order compared to the other loops, + // with the leaves checked last. + for (auto it = m_product.item->modules().rbegin(); it != m_product.item->modules().rend(); ++it) + updateModulePresentState(*it); + + // Now do the canonical module property values merge. Note that this will remove + // previously attached values from modules that failed validation. + // Evaluator cache entries that could potentially change due to this will be purged. + doFinalMerge(m_product, m_loaderState); + + const bool enabled = topLevelProject.checkItemCondition(m_product.item, evaluator); + + collectCodeLinks(); + mergeDependencyParameters(); + checkDependencyParameterDeclarations(m_product.item, m_product.name); + + setupGroups(m_product, m_loaderState); + + // Collect the full list of fileTags, including the values contributed by modules. + if (!m_product.delayedError.hasError() && enabled + && !m_product.name.startsWith(StringConstants::shadowProductPrefix())) { + topLevelProject.addProductByType(m_product, fileTags); + m_product.item->setProperty(StringConstants::typeProperty(), + VariantValue::create(sorted(fileTags.toStringList()))); + } + + checkPropertyDeclarations(); +} + +void ProductResolverStage1::resolveProbes() +{ + for (const Item::Module &module : m_product.item->modules()) { + runModuleProbes(module); + if (m_product.delayedError.hasError()) + return; + } + resolveProbes(m_product.item); +} + +void ProductResolverStage1::resolveProbes(Item *item) +{ + ProbesResolver(m_loaderState).resolveProbes(m_product, item); +} + +void ProductResolverStage1::runModuleProbes(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + if (module.product && m_loaderState.topLevelProject().isDisabledItem(module.product->item)) { + createNonPresentModule(m_loaderState.itemPool(), module.name.toString(), + QLatin1String("module's exporting product is disabled"), + module.item); + return; + } + try { + resolveProbes(module.item); + if (module.versionRange.minimum.isValid() + || module.versionRange.maximum.isValid()) { + if (module.versionRange.maximum.isValid() + && module.versionRange.minimum >= module.versionRange.maximum) { + throw ErrorInfo(Tr::tr("Impossible version constraint [%1,%2) set for module " + "'%3'").arg(module.versionRange.minimum.toString(), + module.versionRange.maximum.toString(), + module.name.toString())); + } + const Version moduleVersion = Version::fromString( + m_loaderState.evaluator().stringValue(module.item, + StringConstants::versionProperty())); + if (moduleVersion < module.versionRange.minimum) { + throw ErrorInfo(Tr::tr("Module '%1' has version %2, but it needs to be " + "at least %3.").arg(module.name.toString(), + moduleVersion.toString(), + module.versionRange.minimum.toString())); + } + if (module.versionRange.maximum.isValid() + && moduleVersion >= module.versionRange.maximum) { + throw ErrorInfo(Tr::tr("Module '%1' has version %2, but it needs to be " + "lower than %3.").arg(module.name.toString(), + moduleVersion.toString(), + module.versionRange.maximum.toString())); + } + } + } catch (const ErrorInfo &error) { + handleModuleSetupError(module, error); + } +} + +void ProductResolverStage1::updateModulePresentState(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + if (module.name.first() == StringConstants::qbsModule()) + return; + bool hasPresentLoadingItem = false; + for (const Item::Module::LoadContext &loadContext : module.loadContexts) { + const Item * const loadingItem = loadContext.loadingItem(); + if (loadingItem == m_product.item) { + hasPresentLoadingItem = true; + break; + } + if (!loadingItem->isPresentModule()) + continue; + if (loadingItem->prototype() && loadingItem->prototype()->type() == ItemType::Export) { + QBS_CHECK(loadingItem->prototype()->parent()->type() == ItemType::Product); + if (m_loaderState.topLevelProject().isDisabledItem(loadingItem->prototype()->parent())) + continue; + } + hasPresentLoadingItem = true; + break; + } + if (!hasPresentLoadingItem) { + createNonPresentModule(m_loaderState.itemPool(), module.name.toString(), + QLatin1String("imported only by disabled module(s)"), + module.item); + } +} + +bool ProductResolverStage1::validateModule(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return true; + try { + m_loaderState.evaluator().boolValue(module.item, StringConstants::validateProperty()); + for (const auto &dep : module.item->modules()) { + if (dep.required && !dep.item->isPresentModule()) { + throw ErrorInfo(Tr::tr("Module '%1' depends on module '%2', which was not " + "loaded successfully") + .arg(module.name.toString(), dep.name.toString())); + } + } + } catch (const ErrorInfo &error) { + handleModuleSetupError(module, error); + if (m_product.delayedError.hasError()) + return false; + } + return true; +} + +void ProductResolverStage1::handleModuleSetupError(const Item::Module &module, + const ErrorInfo &error) +{ + if (module.required) { + m_product.handleError(error); + } else { + qCDebug(lcModuleLoader()) << "non-required module" << module.name.toString() + << "found, but not usable in product" << m_product.name + << error.toString(); + createNonPresentModule(m_loaderState.itemPool(), module.name.toString(), + QStringLiteral("failed validation"), module.item); + } +} + +void ProductResolverStage1::checkPropertyDeclarations() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyChecking : nullptr); + qbs::Internal::checkPropertyDeclarations(m_product.item, m_loaderState); +} + +void ProductResolverStage1::mergeDependencyParameters() +{ + if (m_loaderState.topLevelProject().isDisabledItem(m_product.item)) + return; + + for (Item::Module &module : m_product.item->modules()) { + if (module.name.toString() == StringConstants::qbsModule()) + continue; + if (!module.item->isPresentModule()) + continue; + + using PP = Item::Module::ParametersWithPriority; + std::vector<PP> priorityList; + const QVariantMap defaultParameters = module.product + ? module.product->defaultParameters + : m_loaderState.topLevelProject().parameters(module.item->prototype()); + priorityList.emplace_back(defaultParameters, INT_MIN); + for (const Item::Module::LoadContext &context : module.loadContexts) { + const QVariantMap ¶meters = context.parameters.first; + + // Empty parameter maps and inactive loading modules do not contribute to the + // final parameter map. + if (parameters.isEmpty()) + continue; + if (context.loadingItem()->type() == ItemType::ModuleInstance + && !context.loadingItem()->isPresentModule()) { + continue; + } + + // Build a list sorted by priority. + static const auto cmp = [](const PP &elem, int prio) { return elem.second < prio; }; + const auto it = std::lower_bound(priorityList.begin(), priorityList.end(), + context.parameters.second, cmp); + priorityList.insert(it, context.parameters); + } + + module.parameters = qbs::Internal::mergeDependencyParameters(std::move(priorityList)); + } +} + +class DependencyParameterDeclarationCheck +{ +public: + DependencyParameterDeclarationCheck(const QString &productName, const Item *productItem, + const TopLevelProjectContext &topLevelProject) + : m_productName(productName), m_productItem(productItem), m_topLevelProject(topLevelProject) + {} + + void operator()(const QVariantMap ¶meters) const { check(parameters, QualifiedId()); } + +private: + void check(const QVariantMap ¶meters, const QualifiedId &moduleName) const + { + for (auto it = parameters.begin(); it != parameters.end(); ++it) { + if (it.value().userType() == QMetaType::QVariantMap) { + check(it.value().toMap(), QualifiedId(moduleName) << it.key()); + } else { + const auto &deps = m_productItem->modules(); + auto m = std::find_if(deps.begin(), deps.end(), + [&moduleName] (const Item::Module &module) { + return module.name == moduleName; + }); + + if (m == deps.end()) { + const QualifiedId fullName = QualifiedId(moduleName) << it.key(); + throw ErrorInfo(Tr::tr("Cannot set parameter '%1', " + "because '%2' does not have a dependency on '%3'.") + .arg(fullName.toString(), m_productName, moduleName.toString()), + m_productItem->location()); + } + + const auto decls = m_topLevelProject.parameterDeclarations( + m->item->rootPrototype()); + if (!decls.contains(it.key())) { + const QualifiedId fullName = QualifiedId(moduleName) << it.key(); + throw ErrorInfo(Tr::tr("Parameter '%1' is not declared.") + .arg(fullName.toString()), m_productItem->location()); + } + } + } + } + + bool moduleExists(const QualifiedId &name) const + { + const auto &deps = m_productItem->modules(); + return any_of(deps, [&name](const Item::Module &module) { + return module.name == name; + }); + } + + const QString &m_productName; + const Item * const m_productItem; + const TopLevelProjectContext &m_topLevelProject; +}; + +void ProductResolverStage1::checkDependencyParameterDeclarations(const Item *productItem, + const QString &productName) const +{ + DependencyParameterDeclarationCheck dpdc(productName, productItem, + m_loaderState.topLevelProject()); + for (const Item::Module &dep : productItem->modules()) { + if (!dep.parameters.empty()) + dpdc(dep.parameters); + } +} + +void ProductResolverStage1::collectCodeLinks() +{ + for (const Item::Module &module : m_product.item->modules()) { + if (module.name.first() == StringConstants::qbsModule()) + continue; + for (const Item::Module::LoadContext &context : module.loadContexts) { + m_loaderState.topLevelProject().addCodeLink( + context.dependsItem->location().filePath(), + context.dependsItem->codeRange(), + module.product ? module.product->item->location() + : module.item->location()); + } + } +} + + +void ProductResolverStage2::start() +{ + m_loaderState.evaluator().clearPropertyDependencies(); + + ResolvedProductPtr product = ResolvedProduct::create(); + product->enabled = m_product.project->project->enabled; + product->moduleProperties = PropertyMapInternal::create(); + product->project = m_product.project->project; + m_product.product = product; + product->location = m_product.item->location(); + const auto errorFromDelayedError = [&] { + if (m_product.delayedError.hasError()) { + ErrorInfo errorInfo; + + // First item is "main error", gets prepended again in the catch clause. + const QList<ErrorItem> &items = m_product.delayedError.items(); + for (int i = 1; i < items.size(); ++i) + errorInfo.append(items.at(i)); + + m_product.delayedError.clear(); + return errorInfo; + } + return ErrorInfo(); + }; + + // Even if we previously encountered an error, try to continue for as long as possible + // to provide IDEs with useful data (e.g. the list of files). + // If we encounter a follow-up error, suppress it and report the original one instead. + try { + resolveProductFully(); + if (const ErrorInfo error = errorFromDelayedError(); error.hasError()) + throw error; + } catch (ErrorInfo e) { + if (const ErrorInfo error = errorFromDelayedError(); error.hasError()) + e = error; + QString mainErrorString = !product->name.isEmpty() + ? Tr::tr("Error while handling product '%1':").arg(product->name) + : Tr::tr("Error while handling product:"); + ErrorInfo fullError(mainErrorString, product->location); + appendError(fullError, e); + if (!product->enabled) { + qCDebug(lcProjectResolver) << fullError.toString(); + return; + } + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw fullError; + m_loaderState.logger().printWarning(fullError); + m_loaderState.logger().printWarning( + ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") + .arg(product->name), product->location)); + product->enabled = false; + } +} + +void ProductResolverStage2::resolveProductFully() +{ + Item * const item = m_product.item; + const ResolvedProductPtr product = m_product.product; + Evaluator &evaluator = m_loaderState.evaluator(); + product->name = evaluator.stringValue(item, StringConstants::nameProperty()); + + // product->buildDirectory() isn't valid yet, because the productProperties map is not ready. + m_product.buildDirectory = evaluator.stringValue( + item, StringConstants::buildDirectoryProperty()); + product->multiplexConfigurationId = evaluator.stringValue( + item, StringConstants::multiplexConfigurationIdProperty()); + qCDebug(lcProjectResolver) << "resolveProduct" << product->uniqueName(); + product->enabled = product->enabled + && evaluator.boolValue(item, StringConstants::conditionProperty()); + const VariantValuePtr type = item->variantProperty(StringConstants::typeProperty()); + if (type) + product->fileTags = FileTags::fromStringList(type->value().toStringList()); + product->targetName = evaluator.stringValue(item, StringConstants::targetNameProperty()); + product->sourceDirectory = evaluator.stringValue( + item, StringConstants::sourceDirectoryProperty()); + product->destinationDirectory = evaluator.stringValue( + item, StringConstants::destinationDirProperty()); + + if (product->destinationDirectory.isEmpty()) { + product->destinationDirectory = m_product.buildDirectory; + } else { + product->destinationDirectory = FileInfo::resolvePath( + product->topLevelProject()->buildDirectory, + product->destinationDirectory); + } + product->probes = m_product.probes; + createProductConfig(); + product->productProperties.insert(StringConstants::destinationDirProperty(), + product->destinationDirectory); + ModuleProperties::init(evaluator.engine(), evaluator.scriptValue(item), product.get()); + + QList<Item *> subItems = item->children(); + const ValuePtr filesProperty = item->property(StringConstants::filesProperty()); + if (filesProperty) { + Item *fakeGroup = Item::create(&m_loaderState.itemPool(), ItemType::Group); + fakeGroup->setFile(item->file()); + fakeGroup->setLocation(item->location()); + fakeGroup->setScope(item); + fakeGroup->setProperty(StringConstants::nameProperty(), VariantValue::create(product->name)); + fakeGroup->setProperty(StringConstants::filesProperty(), filesProperty); + fakeGroup->setProperty(StringConstants::excludeFilesProperty(), + item->property(StringConstants::excludeFilesProperty())); + fakeGroup->setProperty(StringConstants::overrideTagsProperty(), + VariantValue::falseValue()); + fakeGroup->setupForBuiltinType(m_loaderState.parameters().deprecationWarningMode(), + m_loaderState.logger()); + subItems.prepend(fakeGroup); + } + + for (Item * const child : std::as_const(subItems)) { + switch (child->type()) { + case ItemType::Rule: + resolveRule(m_loaderState, child, m_product.project, &m_product, nullptr); + break; + case ItemType::FileTagger: + resolveFileTagger(m_loaderState, child, nullptr, &m_product); + break; + case ItemType::JobLimit: + resolveJobLimit(m_loaderState, child, nullptr, &m_product, nullptr); + break; + case ItemType::Group: + resolveGroup(child); + break; + case ItemType::Export: + resolveExport(child); + break; + default: + break; + } + } + + for (const ProjectContext *p = m_product.project; p; p = p->parent) { + JobLimits tempLimits = p->jobLimits; + product->jobLimits = tempLimits.update(product->jobLimits); + } + + resolveModules(); + applyFileTaggers(); + finalizeArtifactProperties(); + + for (const RulePtr &rule : m_product.project->rules) { + RulePtr clonedRule = rule->clone(); + clonedRule->product = product.get(); + product->rules.push_back(clonedRule); + } + + collectProductDependencies(); +} + +void ProductResolverStage2::createProductConfig() +{ + EvalCacheEnabler cachingEnabler(&m_loaderState.evaluator(), + m_product.product->sourceDirectory); + m_product.product->moduleProperties->setValue(evaluateModuleValues(m_product.item)); + m_product.product->productProperties = m_propertiesEvaluator.evaluateProperties( + m_product.item, m_product.item, QVariantMap(), true, true); +} + +void ProductResolverStage2::resolveGroup(Item *item) +{ + const bool parentEnabled = m_currentGroup ? m_currentGroup->enabled + : m_product.product->enabled; + const bool isEnabled = parentEnabled + && m_loaderState.evaluator().boolValue(item, StringConstants::conditionProperty()); + try { + resolveGroupFully(item, isEnabled); + } catch (const ErrorInfo &error) { + if (!isEnabled) { + qCDebug(lcProjectResolver) << "error resolving group at" << item->location() + << error.toString(); + return; + } + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw; + m_loaderState.logger().printWarning(error); + } +} + +void ProductResolverStage2::resolveGroupFully(Item *item, bool isEnabled) +{ + AccumulatingTimer groupTimer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.groupsResolving + : nullptr); + + const auto getGroupPropertyMap = [&](const ArtifactProperties *existingProps) { + PropertyMapPtr moduleProperties; + bool newPropertyMapRequired = false; + if (existingProps) + moduleProperties = existingProps->propertyMap(); + if (!moduleProperties) { + newPropertyMapRequired = true; + moduleProperties = m_currentGroup + ? m_currentGroup->properties + : m_product.product->moduleProperties; + } + const QVariantMap newModuleProperties = resolveAdditionalModuleProperties( + item, moduleProperties->value()); + if (!newModuleProperties.empty()) { + if (newPropertyMapRequired) + moduleProperties = PropertyMapInternal::create(); + moduleProperties->setValue(newModuleProperties); + } + return moduleProperties; + }; + + Evaluator &evaluator = m_loaderState.evaluator(); + QStringList files = evaluator.stringListValue(item, StringConstants::filesProperty()); + bool fileTagsSet; + const FileTags fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty(), + &fileTagsSet); + const QStringList fileTagsFilter + = evaluator.stringListValue(item, StringConstants::fileTagsFilterProperty()); + if (!fileTagsFilter.empty()) { + if (Q_UNLIKELY(!files.empty())) + throw ErrorInfo(Tr::tr("Group.files and Group.fileTagsFilters are exclusive."), + item->location()); + + if (!isEnabled) + return; + + ArtifactPropertiesInfo &apinfo = m_artifactPropertiesPerFilter[fileTagsFilter]; + if (apinfo.first) { + const auto it = std::find_if(apinfo.second.cbegin(), apinfo.second.cend(), + [item](const CodeLocation &loc) { + return item->location().filePath() == loc.filePath(); + }); + if (it != apinfo.second.cend()) { + ErrorInfo error(Tr::tr("Conflicting fileTagsFilter in Group items.")); + error.append(Tr::tr("First item"), *it); + error.append(Tr::tr("Second item"), item->location()); + throw error; + } + } else { + apinfo.first = ArtifactProperties::create(); + apinfo.first->setFileTagsFilter(FileTags::fromStringList(fileTagsFilter)); + m_product.product->artifactProperties.push_back(apinfo.first); + } + apinfo.second.push_back(item->location()); + apinfo.first->setPropertyMapInternal(getGroupPropertyMap(apinfo.first.get())); + apinfo.first->addExtraFileTags(fileTags); + return; + } + QStringList patterns; + for (int i = files.size(); --i >= 0;) { + if (FileInfo::isPattern(files[i])) + patterns.push_back(files.takeAt(i)); + } + GroupPtr group = ResolvedGroup::create(); + bool prefixWasSet = false; + group->prefix = evaluator.stringValue(item, StringConstants::prefixProperty(), QString(), + &prefixWasSet); + if (!prefixWasSet && m_currentGroup) + group->prefix = m_currentGroup->prefix; + if (!group->prefix.isEmpty()) { + for (auto it = files.rbegin(), end = files.rend(); it != end; ++it) + it->prepend(group->prefix); + } + group->location = item->location(); + group->enabled = isEnabled; + group->properties = getGroupPropertyMap(nullptr); + group->fileTags = fileTags; + group->overrideTags = evaluator.boolValue(item, StringConstants::overrideTagsProperty()); + if (group->overrideTags && fileTagsSet) { + if (group->fileTags.empty() ) + group->fileTags.insert(unknownFileTag()); + } else if (m_currentGroup) { + group->fileTags.unite(m_currentGroup->fileTags); + } + + const CodeLocation filesLocation = item->property(StringConstants::filesProperty())->location(); + const VariantValueConstPtr moduleProp = item->variantProperty( + StringConstants::modulePropertyInternal()); + if (moduleProp) + group->targetOfModule = moduleProp->value().toString(); + ErrorInfo fileError; + if (!patterns.empty()) { + group->wildcards = std::make_unique<SourceWildCards>(); + SourceWildCards *wildcards = group->wildcards.get(); + wildcards->excludePatterns = evaluator.stringListValue( + item, StringConstants::excludeFilesProperty()); + wildcards->patterns = patterns; + wildcards->prefix = group->prefix; + wildcards->baseDir = FileInfo::path(item->file()->filePath()); + wildcards->buildDir = m_product.project->project->topLevelProject()->buildDirectory; + wildcards->expandPatterns(); + for (const QString &fileName : wildcards->expandedFiles) + createSourceArtifact(fileName, group, true, filesLocation, &fileError); + } + + for (const QString &fileName : std::as_const(files)) + createSourceArtifact(fileName, group, false, filesLocation, &fileError); + if (fileError.hasError()) { + if (group->enabled) { + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw ErrorInfo(fileError); + m_loaderState.logger().printWarning(fileError); + } else { + qCDebug(lcProjectResolver) << "error for disabled group:" << fileError.toString(); + } + } + group->name = evaluator.stringValue(item, StringConstants::nameProperty()); + if (group->name.isEmpty()) + group->name = Tr::tr("Group %1").arg(m_product.product->groups.size()); + m_product.product->groups.push_back(group); + + class GroupContextSwitcher { + public: + GroupContextSwitcher(ProductResolverStage2 &resolver, const GroupConstPtr &newGroup) + : m_resolver(resolver), m_oldGroup(resolver.m_currentGroup) { + resolver.m_currentGroup = newGroup; + } + ~GroupContextSwitcher() { m_resolver.m_currentGroup = m_oldGroup; } + private: + ProductResolverStage2 &m_resolver; + const GroupConstPtr m_oldGroup; + }; + GroupContextSwitcher groupSwitcher(*this, group); + for (Item * const childItem : item->children()) + resolveGroup(childItem); +} + +SourceArtifactPtr ProductResolverStage2::createSourceArtifact( + const QString &fileName, const GroupPtr &group, bool wildcard, + const CodeLocation &filesLocation, ErrorInfo *errorInfo) +{ + const QString &baseDir = FileInfo::path(group->location.filePath()); + const QString absFilePath = QDir::cleanPath(FileInfo::resolvePath(baseDir, fileName)); + if (!wildcard && !FileInfo(absFilePath).exists()) { + if (errorInfo) + errorInfo->append(Tr::tr("File '%1' does not exist.").arg(absFilePath), filesLocation); + m_product.product->missingSourceFiles << absFilePath; + return {}; + } + if (group->enabled) { + CodeLocation &loc = m_sourceArtifactLocations[ + std::make_pair(group->targetOfModule, absFilePath)]; + if (loc.isValid()) { + if (errorInfo) { + errorInfo->append(Tr::tr("Duplicate source file '%1'.").arg(absFilePath)); + errorInfo->append(Tr::tr("First occurrence is here."), loc); + errorInfo->append(Tr::tr("Next occurrence is here."), filesLocation); + } + return {}; + } + loc = filesLocation; + } + SourceArtifactPtr artifact = SourceArtifactInternal::create(); + artifact->absoluteFilePath = absFilePath; + artifact->fileTags = group->fileTags; + artifact->overrideFileTags = group->overrideTags; + artifact->properties = group->properties; + artifact->targetOfModule = group->targetOfModule; + artifact->fromWildcard = wildcard; + group->files.push_back(artifact); + return artifact; +} + +static QualifiedIdSet propertiesToEvaluate(std::deque<QualifiedId> initialProps, + const PropertyDependencies &deps) +{ + std::deque<QualifiedId> remainingProps = std::move(initialProps); + QualifiedIdSet allProperties; + while (!remainingProps.empty()) { + const QualifiedId prop = remainingProps.front(); + remainingProps.pop_front(); + const auto insertResult = allProperties.insert(prop); + if (!insertResult.second) + continue; + transform(deps.value(prop), remainingProps, [](const QualifiedId &id) { return id; }); + } + return allProperties; +} + +QVariantMap ProductResolverStage2::resolveAdditionalModuleProperties( + const Item *group, const QVariantMap ¤tValues) +{ + // Step 1: Retrieve the properties directly set in the group + const ModulePropertiesPerGroup &mp = m_product.modulePropertiesSetInGroups; + const auto it = mp.find(group); + if (it == mp.end()) + return {}; + const QualifiedIdSet &propsSetInGroup = it->second; + + // Step 2: Gather all properties that depend on these properties. + const QualifiedIdSet &propsToEval = propertiesToEvaluate( + rangeTo<std::deque<QualifiedId>>(propsSetInGroup), + m_loaderState.evaluator().propertyDependencies()); + + // Step 3: Evaluate all these properties and replace their values in the map + QVariantMap modulesMap = currentValues; + QHash<QString, QStringList> propsPerModule; + for (auto fullPropName : propsToEval) { + const QString moduleName + = QualifiedId(fullPropName.mid(0, fullPropName.size() - 1)).toString(); + propsPerModule[moduleName] << fullPropName.last(); + } + EvalCacheEnabler cachingEnabler(&m_loaderState.evaluator(), + m_product.product->sourceDirectory); + for (const Item::Module &module : group->modules()) { + const QString &fullModName = module.name.toString(); + const QStringList propsForModule = propsPerModule.take(fullModName); + if (propsForModule.empty()) + continue; + QVariantMap reusableValues = modulesMap.value(fullModName).toMap(); + for (const QString &prop : std::as_const(propsForModule)) + reusableValues.remove(prop); + modulesMap.insert(fullModName, m_propertiesEvaluator.evaluateProperties( + module.item, module.item, reusableValues, true, true)); + } + return modulesMap; +} + +static QString getLineAtLocation(const CodeLocation &loc, const QString &content) +{ + int pos = 0; + int currentLine = 1; + while (currentLine < loc.line()) { + while (content.at(pos++) != QLatin1Char('\n')) + ; + ++currentLine; + } + const int eolPos = content.indexOf(QLatin1Char('\n'), pos); + return content.mid(pos, eolPos - pos); +} + +static bool usesImport(const ExportedProperty &prop, const QRegularExpression ®ex) +{ + return prop.sourceCode.indexOf(regex) != -1; +} + +static bool usesImport(const ExportedItem &item, const QRegularExpression ®ex) +{ + return any_of(item.properties, + [regex](const ExportedProperty &p) { return usesImport(p, regex); }) + || any_of(item.children, + [regex](const ExportedItemPtr &child) { return usesImport(*child, regex); }); +} + +static bool usesImport(const ExportedModule &module, const QString &name) +{ + // Imports are used in three ways: + // (1) var f = new TextFile(...); + // (2) var path = FileInfo.joinPaths(...) + // (3) var obj = DataCollection; + const QString pattern = QStringLiteral("\\b%1\\b"); + + const QRegularExpression regex(pattern.arg(name)); // std::regex is much slower + return any_of(module.m_properties, + [regex](const ExportedProperty &p) { return usesImport(p, regex); }) + || any_of(module.children, + [regex](const ExportedItemPtr &child) { return usesImport(*child, regex); }); +} + +void ProductResolverStage2::resolveExport(Item *exportItem) +{ + ExportedModule &exportedModule = m_product.product->exportedModule; + setupExportedProperties(exportItem, QString(), exportedModule.m_properties); + static const auto cmpFunc = [](const ExportedProperty &p1, const ExportedProperty &p2) { + return p1.fullName < p2.fullName; + }; + std::sort(exportedModule.m_properties.begin(), exportedModule.m_properties.end(), cmpFunc); + + transform(exportItem->children(), exportedModule.children, + [&exportedModule, this](const auto &child) { + return resolveExportChild(child, exportedModule); }); + + for (const JsImport &jsImport : exportItem->file()->jsImports()) { + if (usesImport(exportedModule, jsImport.scopeName)) { + exportedModule.importStatements << getLineAtLocation(jsImport.location, + exportItem->file()->content()); + } + } + const auto builtInImports = JsExtensions::extensionNames(); + for (const QString &builtinImport: builtInImports) { + if (usesImport(exportedModule, builtinImport)) + exportedModule.importStatements << QStringLiteral("import qbs.") + builtinImport; + } + exportedModule.importStatements.sort(); +} + +// TODO: This probably wouldn't be necessary if we had item serialization. +std::unique_ptr<ExportedItem> ProductResolverStage2::resolveExportChild( + const Item *item, const ExportedModule &module) +{ + std::unique_ptr<ExportedItem> exportedItem(new ExportedItem); + + // This is the type of the built-in base item. It may turn out that we need to support + // derived items under Export. In that case, we probably need a new Item member holding + // the original type name. + exportedItem->name = item->typeName(); + + transform(item->children(), exportedItem->children, [&module, this](const auto &child) { + return resolveExportChild(child, module); }); + + setupExportedProperties(item, QString(), exportedItem->properties); + return exportedItem; +} + +void ProductResolverStage2::setupExportedProperties(const Item *item, const QString &namePrefix, + std::vector<ExportedProperty> &properties) +{ + const auto &props = item->properties(); + for (auto it = props.cbegin(); it != props.cend(); ++it) { + const QString qualifiedName = namePrefix.isEmpty() + ? it.key() : namePrefix + QLatin1Char('.') + it.key(); + if ((item->type() == ItemType::Export || item->type() == ItemType::Properties) + && qualifiedName == StringConstants::prefixMappingProperty()) { + continue; + } + const ValuePtr &v = it.value(); + if (v->type() == Value::ItemValueType) { + setupExportedProperties(std::static_pointer_cast<ItemValue>(v)->item(), + qualifiedName, properties); + continue; + } + ExportedProperty exportedProperty; + exportedProperty.fullName = qualifiedName; + exportedProperty.type = item->propertyDeclaration(it.key()).type(); + if (v->type() == Value::VariantValueType) { + exportedProperty.sourceCode = toJSLiteral( + std::static_pointer_cast<VariantValue>(v)->value()); + } else { + QBS_CHECK(v->type() == Value::JSSourceValueType); + const JSSourceValue * const sv = static_cast<JSSourceValue *>(v.get()); + exportedProperty.sourceCode = sv->sourceCode().toString(); + } + const ItemDeclaration itemDecl + = BuiltinDeclarations::instance().declarationsForType(item->type()); + PropertyDeclaration propertyDecl; + const auto itemProperties = itemDecl.properties(); + for (const PropertyDeclaration &decl : itemProperties) { + if (decl.name() == it.key()) { + propertyDecl = decl; + exportedProperty.isBuiltin = true; + break; + } + } + + // Do not add built-in properties that were left at their default value. + if (!exportedProperty.isBuiltin + || m_loaderState.evaluator().isNonDefaultValue(item, it.key())) { + properties.push_back(exportedProperty); + } + } + + // Order the list of properties, so the output won't look so random. + static const auto less = [](const ExportedProperty &p1, const ExportedProperty &p2) -> bool { + const int p1ComponentCount = p1.fullName.count(QLatin1Char('.')); + const int p2ComponentCount = p2.fullName.count(QLatin1Char('.')); + if (p1.isBuiltin && !p2.isBuiltin) + return true; + if (!p1.isBuiltin && p2.isBuiltin) + return false; + if (p1ComponentCount < p2ComponentCount) + return true; + if (p1ComponentCount > p2ComponentCount) + return false; + return p1.fullName < p2.fullName; + }; + std::sort(properties.begin(), properties.end(), less); +} + +QVariantMap ProductResolverStage2::evaluateModuleValues(Item *item, bool lookupPrototype) +{ + QVariantMap moduleValues; + for (const Item::Module &module : item->modules()) { + if (!module.item->isPresentModule()) + continue; + const QString fullName = module.name.toString(); + moduleValues[fullName] = m_propertiesEvaluator.evaluateProperties( + module.item, lookupPrototype, true); + } + return moduleValues; +} + +void ProductResolverStage2::resolveScanner(Item *item, ModuleContext &moduleContext) +{ + Evaluator &evaluator = m_loaderState.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) { + qCDebug(lcProjectResolver) << "scanner condition is false"; + return; + } + + ResolvedScannerPtr scanner = ResolvedScanner::create(); + scanner->module = moduleContext.module; + scanner->inputs = evaluator.fileTagsValue(item, StringConstants::inputsProperty()); + scanner->recursive = evaluator.boolValue(item, StringConstants::recursiveProperty()); + scanner->searchPathsScript.initialize(m_loaderState.topLevelProject().scriptFunctionValue( + item, StringConstants::searchPathsProperty())); + scanner->scanScript.initialize(m_loaderState.topLevelProject().scriptFunctionValue( + item, StringConstants::scanProperty())); + m_product.product->scanners.push_back(scanner); +} + +void ProductResolverStage2::resolveModules() +{ + JobLimits jobLimits; + for (const Item::Module &m : m_product.item->modules()) + resolveModule(m.name, m.item, m.product, m.parameters, jobLimits); + for (int i = 0; i < jobLimits.count(); ++i) { + const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i); + if (m_product.product->jobLimits.getLimit(moduleJobLimit.pool()) == -1) + m_product.product->jobLimits.setJobLimit(moduleJobLimit); + } +} + +void ProductResolverStage2::resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, + const QVariantMap ¶meters, JobLimits &jobLimits) +{ + if (!item->isPresentModule()) + return; + + ModuleContext moduleContext; + moduleContext.module = ResolvedModule::create(); + + const ResolvedModulePtr &module = moduleContext.module; + module->name = moduleName.toString(); + module->isProduct = isProduct; + module->product = m_product.product.get(); + module->setupBuildEnvironmentScript.initialize(m_loaderState.topLevelProject() + .scriptFunctionValue(item, StringConstants::setupBuildEnvironmentProperty())); + module->setupRunEnvironmentScript.initialize(m_loaderState.topLevelProject() + .scriptFunctionValue(item, StringConstants::setupRunEnvironmentProperty())); + + for (const Item::Module &m : item->modules()) { + if (m.item->isPresentModule()) + module->moduleDependencies += m.name.toString(); + } + + m_product.product->modules.push_back(module); + if (!parameters.empty()) + m_product.product->moduleParameters[module] = parameters; + + for (Item *child : item->children()) { + switch (child->type()) { + case ItemType::Rule: + resolveRule(m_loaderState, child, nullptr, &m_product, &moduleContext); + break; + case ItemType::FileTagger: + resolveFileTagger(m_loaderState, child, nullptr, &m_product); + break; + case ItemType::JobLimit: + resolveJobLimit(m_loaderState, child, nullptr, nullptr, &moduleContext); + break; + case ItemType::Scanner: + resolveScanner(child, moduleContext); + break; + default: + break; + } + } + for (int i = 0; i < moduleContext.jobLimits.count(); ++i) { + const JobLimit &newJobLimit = moduleContext.jobLimits.jobLimitAt(i); + const int oldLimit = jobLimits.getLimit(newJobLimit.pool()); + if (oldLimit == -1 || oldLimit > newJobLimit.limit()) + jobLimits.setJobLimit(newJobLimit); + } +} + +void ProductResolverStage2::applyFileTaggers() +{ + m_product.product->fileTaggers << m_product.project->fileTaggers; + m_product.product->fileTaggers = sorted(m_product.product->fileTaggers, + [] (const FileTaggerConstPtr &a, const FileTaggerConstPtr &b) { + return a->priority() > b->priority(); + }); + for (const SourceArtifactPtr &artifact : m_product.product->allEnabledFiles()) { + if (!artifact->overrideFileTags || artifact->fileTags.empty()) { + const QString fileName = FileInfo::fileName(artifact->absoluteFilePath); + const FileTags fileTags = m_product.product->fileTagsForFileName(fileName); + artifact->fileTags.unite(fileTags); + if (artifact->fileTags.empty()) + artifact->fileTags.insert(unknownFileTag()); + qCDebug(lcProjectResolver) << "adding file tags" << artifact->fileTags + << "to" << fileName; + } + } +} + +void ProductResolverStage2::finalizeArtifactProperties() +{ + for (const SourceArtifactPtr &artifact : m_product.product->allEnabledFiles()) { + for (const auto &artifactProperties : m_product.product->artifactProperties) { + if (!artifact->isTargetOfModule() + && artifact->fileTags.intersects(artifactProperties->fileTagsFilter())) { + // FIXME: Should be merged, not overwritten. + artifact->properties = artifactProperties->propertyMap(); + } + } + + // Let a positive value of qbs.install imply the file tag "installable". + if (artifact->properties->qbsPropertyValue(StringConstants::installProperty()).toBool()) + artifact->fileTags += "installable"; + } +} + +void ProductResolverStage2::collectProductDependencies() +{ + const ResolvedProductPtr &product = m_product.product; + if (!product) + return; + for (const Item::Module &module : m_product.item->modules()) { + if (!module.product) + continue; + const ResolvedProductPtr &dep = module.product->product; + QBS_CHECK(dep); + QBS_CHECK(dep != product); + product->dependencies << dep; + product->dependencyParameters.insert(dep, module.parameters); // TODO: Streamline this with normal module dependencies? + } + + // TODO: We might want to keep the topological sorting and get rid of "module module dependencies". + std::sort(product->dependencies.begin(),product->dependencies.end(), + [](const ResolvedProductPtr &p1, const ResolvedProductPtr &p2) { + return p1->fullDisplayName() < p2->fullDisplayName(); + }); +} + +void ExportsResolver::start() +{ + resolveShadowProduct(); + collectExportedProductDependencies(); +} + +void ExportsResolver::resolveShadowProduct() +{ + if (!m_product.product->enabled) + return; + if (!m_product.shadowProduct) + return; + for (const auto &m : m_product.shadowProduct->item->modules()) { + if (m.name.toString() != m_product.product->name) + continue; + collectPropertiesForExportItem(m.item); + for (const auto &dep : m.item->modules()) + collectPropertiesForModuleInExportItem(dep); + break; + } + try { + adaptExportedPropertyValues(); + } catch (const ErrorInfo &) {} +} + +class TempScopeSetter +{ +public: + TempScopeSetter(const ValuePtr &value, Item *newScope) : m_value(value), m_oldScope(value->scope()) + { + value->setScope(newScope, {}); + } + ~TempScopeSetter() { if (m_value) m_value->setScope(m_oldScope, {}); } + + TempScopeSetter(const TempScopeSetter &) = delete; + TempScopeSetter &operator=(const TempScopeSetter &) = delete; + TempScopeSetter &operator=(TempScopeSetter &&) = delete; + + TempScopeSetter(TempScopeSetter &&other) noexcept + : m_value(std::move(other.m_value)), m_oldScope(other.m_oldScope) + { + other.m_value.reset(); + other.m_oldScope = nullptr; + } + +private: + ValuePtr m_value; + Item *m_oldScope; +}; + +void ExportsResolver::collectPropertiesForExportItem( + const QualifiedId &moduleName, const ValuePtr &value, Item *moduleInstance, + QVariantMap &moduleProps) +{ + QBS_CHECK(value->type() == Value::ItemValueType); + Item * const itemValueItem = std::static_pointer_cast<ItemValue>(value)->item(); + if (itemValueItem->propertyDeclarations().isEmpty()) { + for (const Item::Module &module : moduleInstance->modules()) { + if (module.name == moduleName) { + itemValueItem->setPropertyDeclarations(module.item->propertyDeclarations()); + break; + } + } + } + if (itemValueItem->type() == ItemType::ModuleInstancePlaceholder) { + struct EvalPreparer { + EvalPreparer(Item *valueItem, const QualifiedId &moduleName) + : valueItem(valueItem), + hadName(!!valueItem->variantProperty(StringConstants::nameProperty())) + { + if (!hadName) { + // Evaluator expects a name here. + valueItem->setProperty(StringConstants::nameProperty(), + VariantValue::create(moduleName.toString())); + } + } + ~EvalPreparer() + { + if (!hadName) + valueItem->removeProperty(StringConstants::nameProperty()); + } + Item * const valueItem; + const bool hadName; + }; + EvalPreparer ep(itemValueItem, moduleName); + std::vector<TempScopeSetter> tss; + for (const ValuePtr &v : itemValueItem->properties()) + tss.emplace_back(v, moduleInstance); + moduleProps.insert(moduleName.toString(), m_propertiesEvaluator.evaluateProperties( + itemValueItem, false, false)); + return; + } + QBS_CHECK(itemValueItem->type() == ItemType::ModulePrefix); + const Item::PropertyMap &props = itemValueItem->properties(); + for (auto it = props.begin(); it != props.end(); ++it) { + QualifiedId fullModuleName = moduleName; + fullModuleName << it.key(); + collectPropertiesForExportItem(fullModuleName, it.value(), moduleInstance, moduleProps); + } +} + +void ExportsResolver::collectPropertiesForExportItem(Item *productModuleInstance) +{ + if (!productModuleInstance->isPresentModule()) + return; + Item * const exportItem = productModuleInstance->prototype(); + QBS_CHECK(exportItem); + QBS_CHECK(exportItem->type() == ItemType::Export); + const ItemDeclaration::Properties exportDecls = BuiltinDeclarations::instance() + .declarationsForType(ItemType::Export).properties(); + ExportedModule &exportedModule = m_product.product->exportedModule; + const auto &props = exportItem->properties(); + for (auto it = props.begin(); it != props.end(); ++it) { + const auto match + = [it](const PropertyDeclaration &decl) { return decl.name() == it.key(); }; + if (it.key() != StringConstants::prefixMappingProperty() && + std::find_if(exportDecls.begin(), exportDecls.end(), match) != exportDecls.end()) { + continue; + } + if (it.value()->type() == Value::ItemValueType) { + collectPropertiesForExportItem(it.key(), it.value(), productModuleInstance, + exportedModule.modulePropertyValues); + } else { + TempScopeSetter tss(it.value(), productModuleInstance); + m_propertiesEvaluator.evaluateProperty( + exportItem, it.key(), it.value(), exportedModule.propertyValues, false); + } + } +} + +// Collects module properties assigned to in other (higher-level) modules. +void ExportsResolver::collectPropertiesForModuleInExportItem(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + ExportedModule &exportedModule = m_product.product->exportedModule; + if (module.product || module.name.first() == StringConstants::qbsModule()) + return; + const auto checkName = [module](const ExportedModuleDependency &d) { + return module.name.toString() == d.name; + }; + if (any_of(exportedModule.moduleDependencies, checkName)) + return; + + Item *modulePrototype = module.item->prototype(); + while (modulePrototype && modulePrototype->type() != ItemType::Module) + modulePrototype = modulePrototype->prototype(); + if (!modulePrototype) // Can happen for broken products in relaxed mode. + return; + ModuleItemLocker locker(*modulePrototype); + const Item::PropertyMap &props = modulePrototype->properties(); + ExportedModuleDependency dep; + dep.name = module.name.toString(); + for (auto it = props.begin(); it != props.end(); ++it) { + if (it.value()->type() == Value::ItemValueType) + collectPropertiesForExportItem(it.key(), it.value(), module.item, dep.moduleProperties); + } + exportedModule.moduleDependencies.push_back(dep); + + for (const auto &dep : module.item->modules()) + collectPropertiesForModuleInExportItem(dep); +} + +void ExportsResolver::adaptExportedPropertyValues() +{ + QBS_CHECK(m_product.shadowProduct); + ExportedModule &m = m_product.product->exportedModule; + const QVariantList prefixList = m.propertyValues.take( + StringConstants::prefixMappingProperty()).toList(); + const QString shadowProductName = m_loaderState.evaluator().stringValue( + m_product.shadowProduct->item, StringConstants::nameProperty()); + const QString shadowProductBuildDir = m_loaderState.evaluator().stringValue( + m_product.shadowProduct->item, StringConstants::buildDirectoryProperty()); + QVariantMap prefixMap; + for (const QVariant &v : prefixList) { + const QVariantMap o = v.toMap(); + prefixMap.insert(o.value(QStringLiteral("prefix")).toString(), + o.value(QStringLiteral("replacement")).toString()); + } + const auto valueRefersToImportingProduct + = [shadowProductName, shadowProductBuildDir](const QString &value) { + return value.toLower().contains(shadowProductName.toLower()) + || value.contains(shadowProductBuildDir); + }; + static const auto stringMapper = [](const QVariantMap &mappings, const QString &value) + -> QString { + for (auto it = mappings.cbegin(); it != mappings.cend(); ++it) { + if (value.startsWith(it.key())) + return it.value().toString() + value.mid(it.key().size()); + } + return value; + }; + const auto stringListMapper = [&valueRefersToImportingProduct]( + const QVariantMap &mappings, const QStringList &value) -> QStringList { + QStringList result; + result.reserve(value.size()); + for (const QString &s : value) { + if (!valueRefersToImportingProduct(s)) + result.push_back(stringMapper(mappings, s)); + } + return result; + }; + const std::function<QVariant(const QVariantMap &, const QVariant &)> mapper + = [&stringListMapper, &mapper]( + const QVariantMap &mappings, const QVariant &value) -> QVariant { + switch (static_cast<QMetaType::Type>(value.userType())) { + case QMetaType::QString: + return stringMapper(mappings, value.toString()); + case QMetaType::QStringList: + return stringListMapper(mappings, value.toStringList()); + case QMetaType::QVariantMap: { + QVariantMap m = value.toMap(); + for (auto it = m.begin(); it != m.end(); ++it) + it.value() = mapper(mappings, it.value()); + return m; + } + default: + return value; + } + }; + for (auto it = m.propertyValues.begin(); it != m.propertyValues.end(); ++it) + it.value() = mapper(prefixMap, it.value()); + for (auto it = m.modulePropertyValues.begin(); it != m.modulePropertyValues.end(); ++it) + it.value() = mapper(prefixMap, it.value()); + for (ExportedModuleDependency &dep : m.moduleDependencies) { + for (auto it = dep.moduleProperties.begin(); it != dep.moduleProperties.end(); ++it) + it.value() = mapper(prefixMap, it.value()); + } +} + +void ExportsResolver::collectExportedProductDependencies() +{ + if (!m_product.shadowProduct) + return; + const ResolvedProductPtr exportingProduct = m_product.product; + if (!exportingProduct || !exportingProduct->enabled) + return; + Item * const importingProductItem = m_product.shadowProduct->item; + + std::vector<std::pair<ResolvedProductPtr, QVariantMap>> directDeps; + for (const Item::Module &m : importingProductItem->modules()) { + if (m.name.toString() != exportingProduct->name) + continue; + for (const Item::Module &dep : m.item->modules()) { + if (dep.product) + directDeps.emplace_back(dep.product->product, m.parameters); + } + } + for (const auto &dep : directDeps) { + if (!contains(exportingProduct->exportedModule.productDependencies, + dep.first->uniqueName())) { + exportingProduct->exportedModule.productDependencies.push_back( + dep.first->uniqueName()); + } + if (!dep.second.isEmpty()) { + exportingProduct->exportedModule.dependencyParameters.insert(dep.first, + dep.second); + } + } + auto &productDeps = exportingProduct->exportedModule.productDependencies; + std::sort(productDeps.begin(), productDeps.end()); +} + +QVariantMap PropertiesEvaluator::evaluateProperties( + const Item *item, const Item *propertiesContainer, const QVariantMap &tmplt, + bool lookupPrototype, bool checkErrors) +{ + AccumulatingTimer propEvalTimer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyEvaluation + : nullptr); + QVariantMap result = tmplt; + for (auto it = propertiesContainer->properties().begin(); + it != propertiesContainer->properties().end(); ++it) { + evaluateProperty(item, it.key(), it.value(), result, checkErrors); + } + return lookupPrototype && propertiesContainer->prototype() + && propertiesContainer->prototype()->type() != ItemType::Module + ? evaluateProperties(item, propertiesContainer->prototype(), result, true, checkErrors) + : result; +} + +QVariantMap PropertiesEvaluator::evaluateProperties(Item *item, bool lookupPrototype, + bool checkErrors) +{ + const QVariantMap tmplt; + return evaluateProperties(item, item, tmplt, lookupPrototype, checkErrors); +} + +void PropertiesEvaluator::evaluateProperty( + const Item *item, const QString &propName, const ValuePtr &propValue, QVariantMap &result, + bool checkErrors) +{ + JSContext * const ctx = m_loaderState.evaluator().engine()->context(); + switch (propValue->type()) { + case Value::ItemValueType: + { + // Ignore items. Those point to module instances + // and are handled in evaluateModuleValues(). + break; + } + case Value::JSSourceValueType: + { + if (result.contains(propName)) + break; + const PropertyDeclaration pd = item->propertyDeclaration(propName); + if (pd.flags().testFlag(PropertyDeclaration::PropertyNotAvailableInConfig)) { + break; + } + const ScopedJsValue scriptValue(ctx, m_loaderState.evaluator().property(item, propName)); + if (JsException ex = m_loaderState.evaluator().engine()->checkAndClearException( + propValue->location())) { + if (checkErrors) + throw ex.toErrorInfo(); + } + + // NOTE: Loses type information if scriptValue.isUndefined == true, + // as such QScriptValues become invalid QVariants. + QVariant v; + if (JS_IsFunction(ctx, scriptValue)) { + v = getJsString(ctx, scriptValue); + } else { + v = getJsVariant(ctx, scriptValue); + QVariantMap m = v.toMap(); + if (m.contains(StringConstants::importScopeNamePropertyInternal())) { + QVariantMap tmp = m; + const ScopedJsValue proto(ctx, JS_GetPrototype(ctx, scriptValue)); + m = getJsVariant(ctx, proto).toMap(); + for (auto it = tmp.begin(); it != tmp.end(); ++it) + m.insert(it.key(), it.value()); + v = m; + } + } + + if (pd.type() == PropertyDeclaration::Path && v.isValid()) { + v = v.toString(); + } else if (pd.type() == PropertyDeclaration::PathList + || pd.type() == PropertyDeclaration::StringList) { + v = v.toStringList(); + } else if (pd.type() == PropertyDeclaration::VariantList) { + v = v.toList(); + } + + // Enforce proper type for undefined values (note that path degrades to string). + if (!v.isValid()) + v = pd.typedNullValue(); + + pd.checkAllowedValues(v, propValue->location(), propName, m_loaderState); + + result[propName] = v; + break; + } + case Value::VariantValueType: + { + if (result.contains(propName)) + break; + VariantValuePtr vvp = std::static_pointer_cast<VariantValue>(propValue); + QVariant v = vvp->value(); + + const PropertyDeclaration pd = item->propertyDeclaration(propName); + if (v.isNull() && !pd.isScalar()) // QTBUG-51237 + v = QStringList(); + + pd.checkAllowedValues(v, propValue->location(), propName, m_loaderState); + result[propName] = v; + break; + } + } +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productresolver.h b/src/lib/corelib/loader/productresolver.h new file mode 100644 index 000000000..34c49b2bb --- /dev/null +++ b/src/lib/corelib/loader/productresolver.h @@ -0,0 +1,57 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs::Internal { +enum class Deferral; +class LoaderState; +class ProductContext; + +// Responsibilities: +// - Resolving dependencies to modules and other products (via DependenciesResolver). +// - Module validation. +// - Running probes (via ProbesResolver) in Product and Module items. +// - Evaluating product and module properties. +// - Handling all Product child items, such as FileTaggers, Rules and so on. +void resolveProduct(ProductContext &product, Deferral deferral, LoaderState &loaderState); + +void setupExports(ProductContext &product, LoaderState &loaderState); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productscollector.cpp b/src/lib/corelib/loader/productscollector.cpp new file mode 100644 index 000000000..042dbd160 --- /dev/null +++ b/src/lib/corelib/loader/productscollector.cpp @@ -0,0 +1,754 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "productscollector.h" + +#include "dependenciesresolver.h" +#include "itemreader.h" +#include "loaderutils.h" +#include "localprofiles.h" +#include "productitemmultiplexer.h" +#include "probesresolver.h" + +#include <language/builtindeclarations.h> +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/item.h> +#include <language/language.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/fileinfo.h> +#include <tools/profile.h> +#include <tools/profiling.h> +#include <language/scriptengine.h> +#include <tools/set.h> +#include <tools/settings.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <QDir> +#include <QDirIterator> + +namespace qbs::Internal { + +class ProductsCollector::Private +{ +public: + Private(LoaderState &loaderState) : loaderState(loaderState) {} + + void handleProject(Item *projectItem, ProjectContext *parentProject, + const Set<QString> &referencedFilePaths); + QList<Item *> multiplexProductItem(ProductContext &dummyContext, Item *productItem); + void prepareProduct(ProjectContext &projectContext, Item *productItem, + ProductContext *mainProduct = nullptr); + void handleSubProject(ProjectContext &projectContext, Item *projectItem, + const Set<QString> &referencedFilePaths); + void copyProperties(const Item *sourceProject, Item *targetProject); + QList<Item *> loadReferencedFile( + const QString &relativePath, const CodeLocation &referencingLocation, + const Set<QString> &referencedFilePaths, ProductContext &dummyContext); + bool mergeExportItems(ProductContext &productContext); + bool checkExportItemCondition(Item *exportItem, const ProductContext &product); + void initProductProperties(const ProductContext &product); + void checkProjectNamesInOverrides(); + void collectProductsByNameAndItem(); + void checkProductNamesInOverrides(); + void mergeProperty(Item *dst, const QString &name, const ValuePtr &value); + + LoaderState &loaderState; + Settings settings{loaderState.parameters().settingsDirectory()}; + Set<QString> disabledProjects; + Version qbsVersion; + Item *tempScopeItem = nullptr; + +private: + // TODO: Put this in loaderutils + class TempBaseModuleAttacher { + public: + TempBaseModuleAttacher(Private *d, ProductContext &product); + ~TempBaseModuleAttacher() { drop(); } + void drop(); + Item *tempBaseModuleItem() const { return m_tempBaseModule; } + + private: + Item * const m_productItem; + ValuePtr m_origQbsValue; + Item *m_tempBaseModule = nullptr; + }; +}; + +ProductsCollector::ProductsCollector(LoaderState &loaderState) + : d(makePimpl<Private>(loaderState)) {} + +ProductsCollector::~ProductsCollector() = default; + +void ProductsCollector::run(Item *rootProject) +{ + d->handleProject(rootProject, nullptr, {rootProject->file()->filePath()}); + d->checkProjectNamesInOverrides(); + d->collectProductsByNameAndItem(); + d->checkProductNamesInOverrides(); + d->loaderState.topLevelProject().checkForLocalProfileAsTopLevelProfile( + d->loaderState.parameters().topLevelProfile()); +} + +void ProductsCollector::Private::handleProject(Item *projectItem, ProjectContext *parentProject, + const Set<QString> &referencedFilePaths) +{ + const SetupProjectParameters ¶meters = loaderState.parameters(); + Evaluator &evaluator = loaderState.evaluator(); + Logger &logger = loaderState.logger(); + ItemReader &itemReader = loaderState.itemReader(); + TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); + + auto p = std::make_unique<ProjectContext>(); + auto &projectContext = *p; + projectContext.item = projectItem; + projectContext.parent = parentProject; + projectContext.topLevelProject = &topLevelProject; + ItemValuePtr itemValue = ItemValue::create(projectItem); + projectContext.scope = Item::create(&loaderState.itemPool(), ItemType::Scope); + projectContext.scope->setFile(projectItem->file()); + projectContext.scope->setProperty(StringConstants::projectVar(), itemValue); + ProductContext dummyProduct; + dummyProduct.project = &projectContext; + dummyProduct.moduleProperties = parameters.finalBuildConfigurationTree(); + dummyProduct.profileModuleProperties = dummyProduct.moduleProperties; + dummyProduct.profileName = parameters.topLevelProfile(); + loadBaseModule(dummyProduct, projectItem, loaderState); + + projectItem->overrideProperties(parameters.overriddenValuesTree(), + StringConstants::projectPrefix(), parameters, logger); + projectContext.name = evaluator.stringValue(projectItem, StringConstants::nameProperty()); + if (projectContext.name.isEmpty()) { + projectContext.name = FileInfo::baseName(projectItem->location().filePath()); + projectItem->setProperty(StringConstants::nameProperty(), + VariantValue::create(projectContext.name)); + } + if (parentProject) + parentProject->children.push_back(p.get()); + topLevelProject.addProject(p.release()); + projectItem->overrideProperties(parameters.overriddenValuesTree(), + StringConstants::projectsOverridePrefix() + projectContext.name, + parameters, logger); + if (!topLevelProject.checkItemCondition(projectItem, evaluator)) { + disabledProjects.insert(projectContext.name); + return; + } + SearchPathsManager searchPathsManager(itemReader, itemReader.readExtraSearchPaths(projectItem) + << projectItem->file()->dirPath()); + projectContext.searchPathsStack = itemReader.extraSearchPathsStack(); + + const QString minVersionStr + = evaluator.stringValue(projectItem, StringConstants::minimumQbsVersionProperty(), + QStringLiteral("1.3.0")); + const Version minVersion = Version::fromString(minVersionStr); + if (!minVersion.isValid()) { + throw ErrorInfo(Tr::tr("The value '%1' of Project.minimumQbsVersion " + "is not a valid version string.") + .arg(minVersionStr), projectItem->location()); + } + if (!qbsVersion.isValid()) + qbsVersion = Version::fromString(QLatin1String(QBS_VERSION)); + if (qbsVersion < minVersion) { + throw ErrorInfo(Tr::tr("The project requires at least qbs version %1, but " + "this is qbs version %2.").arg(minVersion.toString(), + qbsVersion.toString())); + } + + for (Item * const child : projectItem->children()) + child->setScope(projectContext.scope); + + ProbesResolver(loaderState).resolveProbes(dummyProduct, projectItem); + + collectProfilesFromItems(projectItem, projectContext.scope, loaderState); + + QList<Item *> multiplexedProducts; + for (Item * const child : projectItem->children()) { + if (child->type() == ItemType::Product) + multiplexedProducts << multiplexProductItem(dummyProduct, child); + } + for (Item * const additionalProductItem : std::as_const(multiplexedProducts)) + Item::addChild(projectItem, additionalProductItem); + + const QList<Item *> originalChildren = projectItem->children(); + for (Item * const child : originalChildren) { + switch (child->type()) { + case ItemType::Product: + prepareProduct(projectContext, child); + break; + case ItemType::SubProject: + handleSubProject(projectContext, child, referencedFilePaths); + break; + case ItemType::Project: + copyProperties(projectItem, child); + handleProject(child, &projectContext, referencedFilePaths); + break; + default: + break; + } + } + + const QStringList refs = evaluator.stringListValue( + projectItem, StringConstants::referencesProperty()); + const CodeLocation referencingLocation + = projectItem->property(StringConstants::referencesProperty())->location(); + QList<Item *> additionalProjectChildren; + for (const QString &filePath : refs) { + try { + additionalProjectChildren << loadReferencedFile( + filePath, referencingLocation, referencedFilePaths, dummyProduct); + } catch (const ErrorInfo &error) { + if (parameters.productErrorMode() == ErrorHandlingMode::Strict) + throw; + logger.printWarning(error); + } + } + for (Item * const subItem : std::as_const(additionalProjectChildren)) { + Item::addChild(projectContext.item, subItem); + switch (subItem->type()) { + case ItemType::Product: + prepareProduct(projectContext, subItem); + break; + case ItemType::Project: + copyProperties(projectItem, subItem); + handleProject(subItem, &projectContext, + Set<QString>(referencedFilePaths) << subItem->file()->filePath()); + break; + default: + break; + } + } +} + +QList<Item *> ProductsCollector::Private::multiplexProductItem(ProductContext &dummyContext, + Item *productItem) +{ + // Overriding the product item properties must be done here already, because multiplexing + // properties might depend on product properties. + const QString &nameKey = StringConstants::nameProperty(); + QString productName = loaderState.evaluator().stringValue(productItem, nameKey); + if (productName.isEmpty()) { + productName = FileInfo::completeBaseName(productItem->file()->filePath()); + productItem->setProperty(nameKey, VariantValue::create(productName)); + } + productItem->overrideProperties(loaderState.parameters().overriddenValuesTree(), + StringConstants::productsOverridePrefix() + productName, + loaderState.parameters(), loaderState.logger()); + dummyContext.item = productItem; + TempBaseModuleAttacher tbma(this, dummyContext); + return multiplex(productName, productItem, tbma.tempBaseModuleItem(), + [&] { tbma.drop(); }, loaderState); +} + +void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, Item *productItem, + ProductContext *mainProduct) +{ + const SetupProjectParameters ¶meters = loaderState.parameters(); + Evaluator &evaluator = loaderState.evaluator(); + TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); + + AccumulatingTimer timer(parameters.logElapsedTime() + ? &topLevelProject.timingData().preparingProducts : nullptr); + topLevelProject.checkCancelation(); + qCDebug(lcModuleLoader) << "prepareProduct" << productItem->file()->filePath(); + + if (mainProduct) + mainProduct->shadowProduct = std::make_unique<ProductContext>(); + else + projectContext.products.emplace_back(); + ProductContext &productContext = mainProduct + ? *mainProduct->shadowProduct : projectContext.products.back(); + productContext.item = productItem; + productContext.project = &projectContext; + + // Retrieve name, profile and multiplex id. + productContext.name = evaluator.stringValue(productItem, StringConstants::nameProperty()); + QBS_CHECK(!productContext.name.isEmpty()); + const ItemValueConstPtr qbsItemValue = productItem->itemProperty(StringConstants::qbsModule(), + loaderState.itemPool()); + if (qbsItemValue && qbsItemValue->item()->hasProperty(StringConstants::profileProperty())) { + TempBaseModuleAttacher tbma(this, productContext); + productContext.profileName = evaluator.stringValue( + tbma.tempBaseModuleItem(), StringConstants::profileProperty(), QString()); + } else { + productContext.profileName = parameters.topLevelProfile(); + } + productContext.multiplexConfigurationId = evaluator.stringValue( + productItem, StringConstants::multiplexConfigurationIdProperty()); + QBS_CHECK(!productContext.profileName.isEmpty()); + + // Set up full module property map based on the profile. + std::optional<QVariantMap> flatConfig + = topLevelProject.profileConfig(productContext.profileName); + if (!flatConfig) { + const Profile profile(productContext.profileName, &settings, + loaderState.topLevelProject().localProfiles()); + if (!profile.exists()) { + ErrorInfo error(Tr::tr("Profile '%1' does not exist.").arg(profile.name()), + productItem->location()); + productContext.handleError(error); + return; + } + flatConfig = SetupProjectParameters::expandedBuildConfiguration( + profile, parameters.configurationName()); + topLevelProject.addProfileConfig(productContext.profileName, *flatConfig); + } + productContext.profileModuleProperties = SetupProjectParameters::finalBuildConfigurationTree( + *flatConfig, {}); + productContext.moduleProperties = SetupProjectParameters::finalBuildConfigurationTree( + *flatConfig, parameters.overriddenValues()); + initProductProperties(productContext); + + // Set up product scope. This is mainly for using the "product" and "project" + // variables in some contexts. + ItemValuePtr itemValue = ItemValue::create(productItem); + productContext.scope = Item::create(&loaderState.itemPool(), ItemType::Scope); + productContext.scope->setProperty(StringConstants::productVar(), itemValue); + productContext.scope->setFile(productItem->file()); + productContext.scope->setScope(productContext.project->scope); + + // If there are any child items with an id, set up a scope for them. This is mostly + // relevant for Probe items. While we might get away with using the file's id scope + // in the absence of multiplexing, having a proper per-product scope seems cleaner. + QBS_CHECK(productItem->scope()); + QBS_CHECK(productItem->scope() == productContext.project->scope); + for (Item * const child : productItem->children()) { + if (child->id().isEmpty()) + continue; + if (productItem->scope() == productContext.project->scope) { + productItem->setScope(Item::create(&loaderState.itemPool(), ItemType::Scope)); + productItem->scope()->setScope(productContext.project->scope); + } + const ItemValuePtr childValue = ItemValue::create(child); + productItem->scope()->setProperty(child->id(), childValue); + productContext.scope->setProperty(child->id(), childValue); + } + + const bool hasExportItems = mergeExportItems(productContext); + + setScopeForDescendants(productItem, productContext.scope); + + if (!hasExportItems) + return; + + // This "shadow product" exists only to pull in a dependency on the actual product + // and nothing else, thus providing us with the pure environment that we need to + // evaluate the product's exported properties in isolation in the project resolver. + Item * const importer = Item::create(&loaderState.itemPool(), ItemType::Product); + importer->setProperty(QStringLiteral("name"), + VariantValue::create(StringConstants::shadowProductPrefix() + + productContext.name)); + importer->setFile(productItem->file()); + importer->setLocation(productItem->location()); + importer->setScope(projectContext.scope); + importer->setupForBuiltinType(parameters.deprecationWarningMode(), loaderState.logger()); + Item * const dependsItem = Item::create(&loaderState.itemPool(), ItemType::Depends); + dependsItem->setProperty(QStringLiteral("name"), VariantValue::create(productContext.name)); + dependsItem->setProperty(QStringLiteral("required"), VariantValue::create(false)); + dependsItem->setFile(importer->file()); + dependsItem->setLocation(importer->location()); + dependsItem->setupForBuiltinType(parameters.deprecationWarningMode(), loaderState.logger()); + dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), + VariantValue::create(productContext.multiplexConfigurationId)); + Item::addChild(importer, dependsItem); + prepareProduct(projectContext, importer, &productContext); +} + +void ProductsCollector::Private::handleSubProject( + ProjectContext &projectContext, Item *projectItem, const Set<QString> &referencedFilePaths) +{ + const SetupProjectParameters ¶meters = loaderState.parameters(); + Evaluator &evaluator = loaderState.evaluator(); + Logger &logger = loaderState.logger(); + ItemReader &itemReader = loaderState.itemReader(); + TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); + + qCDebug(lcModuleLoader) << "handleSubProject" << projectItem->file()->filePath(); + + Item * const propertiesItem = projectItem->child(ItemType::PropertiesInSubProject); + if (!topLevelProject.checkItemCondition(projectItem, evaluator)) + return; + if (propertiesItem) { + propertiesItem->setScope(projectItem); + if (!topLevelProject.checkItemCondition(propertiesItem, evaluator)) + return; + } + + Item *loadedItem = nullptr; + QString subProjectFilePath; + try { + const QString projectFileDirPath = FileInfo::path(projectItem->file()->filePath()); + const QString relativeFilePath + = evaluator.stringValue(projectItem, StringConstants::filePathProperty()); + subProjectFilePath = FileInfo::resolvePath(projectFileDirPath, relativeFilePath); + if (referencedFilePaths.contains(subProjectFilePath)) + throw ErrorInfo(Tr::tr("Cycle detected while loading subproject file '%1'.") + .arg(relativeFilePath), projectItem->location()); + loadedItem = itemReader.setupItemFromFile(subProjectFilePath, projectItem->location()); + } catch (const ErrorInfo &error) { + if (parameters.productErrorMode() == ErrorHandlingMode::Strict) + throw; + logger.printWarning(error); + return; + } + + loadedItem = itemReader.wrapInProjectIfNecessary(loadedItem); + const bool inheritProperties = evaluator.boolValue( + projectItem, StringConstants::inheritPropertiesProperty()); + + if (inheritProperties) + copyProperties(projectItem->parent(), loadedItem); + if (propertiesItem) { + const Item::PropertyMap &overriddenProperties = propertiesItem->properties(); + for (auto it = overriddenProperties.begin(); it != overriddenProperties.end(); ++it) + loadedItem->setProperty(it.key(), it.value()); + } + + Item::addChild(projectItem, loadedItem); + projectItem->setScope(projectContext.scope); + handleProject(loadedItem, &projectContext, Set<QString>(referencedFilePaths) << subProjectFilePath); +} + +void ProductsCollector::Private::copyProperties(const Item *sourceProject, Item *targetProject) +{ + if (!sourceProject) + return; + const QList<PropertyDeclaration> builtinProjectProperties + = BuiltinDeclarations::instance().declarationsForType(ItemType::Project).properties(); + Set<QString> builtinProjectPropertyNames; + for (const PropertyDeclaration &p : builtinProjectProperties) + builtinProjectPropertyNames << p.name(); + + for (auto it = sourceProject->propertyDeclarations().begin(); + it != sourceProject->propertyDeclarations().end(); ++it) { + + // We must not inherit built-in properties such as "name", + // but there are exceptions. + if (it.key() == StringConstants::qbsSearchPathsProperty() + || it.key() == StringConstants::profileProperty() + || it.key() == StringConstants::buildDirectoryProperty() + || it.key() == StringConstants::sourceDirectoryProperty() + || it.key() == StringConstants::minimumQbsVersionProperty()) { + const JSSourceValueConstPtr &v = targetProject->sourceProperty(it.key()); + QBS_ASSERT(v, continue); + if (v->sourceCode() == StringConstants::undefinedValue()) + sourceProject->copyProperty(it.key(), targetProject); + continue; + } + + if (builtinProjectPropertyNames.contains(it.key())) + continue; + + if (targetProject->hasOwnProperty(it.key())) + continue; // Ignore stuff the target project already has. + + targetProject->setPropertyDeclaration(it.key(), it.value()); + sourceProject->copyProperty(it.key(), targetProject); + } +} + +QList<Item *> ProductsCollector::Private::loadReferencedFile( + const QString &relativePath, const CodeLocation &referencingLocation, + const Set<QString> &referencedFilePaths, ProductContext &dummyContext) +{ + QString absReferencePath = FileInfo::resolvePath(FileInfo::path(referencingLocation.filePath()), + relativePath); + if (FileInfo(absReferencePath).isDir()) { + QString qbsFilePath; + + QDirIterator dit(absReferencePath, StringConstants::qbsFileWildcards()); + while (dit.hasNext()) { + if (!qbsFilePath.isEmpty()) { + throw ErrorInfo(Tr::tr("Referenced directory '%1' contains more than one " + "qbs file.").arg(absReferencePath), referencingLocation); + } + qbsFilePath = dit.next(); + } + if (qbsFilePath.isEmpty()) { + throw ErrorInfo(Tr::tr("Referenced directory '%1' does not contain a qbs file.") + .arg(absReferencePath), referencingLocation); + } + absReferencePath = qbsFilePath; + } + if (referencedFilePaths.contains(absReferencePath)) + throw ErrorInfo(Tr::tr("Cycle detected while referencing file '%1'.").arg(relativePath), + referencingLocation); + Item * const subItem = loaderState.itemReader().setupItemFromFile( + absReferencePath, referencingLocation); + if (subItem->type() != ItemType::Project && subItem->type() != ItemType::Product) { + ErrorInfo error(Tr::tr("Item type should be 'Product' or 'Project', but is '%1'.") + .arg(subItem->typeName())); + error.append(Tr::tr("Item is defined here."), subItem->location()); + error.append(Tr::tr("File is referenced here."), referencingLocation); + throw error; + } + subItem->setScope(dummyContext.project->scope); + subItem->setParent(dummyContext.project->item); + QList<Item *> loadedItems; + loadedItems << subItem; + if (subItem->type() == ItemType::Product) { + collectProfilesFromItems(subItem, dummyContext.project->scope, loaderState); + loadedItems << multiplexProductItem(dummyContext, subItem); + } + return loadedItems; +} + +void ProductsCollector::Private::mergeProperty(Item *dst, const QString &name, + const ValuePtr &value) +{ + if (value->type() == Value::ItemValueType) { + const ItemValueConstPtr itemValue = std::static_pointer_cast<ItemValue>(value); + const Item * const valueItem = itemValue->item(); + Item * const subItem = dst->itemProperty(name, itemValue, loaderState.itemPool())->item(); + for (auto it = valueItem->properties().begin(); it != valueItem->properties().end(); ++it) + mergeProperty(subItem, it.key(), it.value()); + return; + } + + // If the property already exists, set up the base value. + if (value->type() == Value::JSSourceValueType) { + const auto jsValue = static_cast<JSSourceValue *>(value.get()); + if (jsValue->isBuiltinDefaultValue()) + return; + const ValuePtr baseValue = dst->property(name); + if (baseValue) { + QBS_CHECK(baseValue->type() == Value::JSSourceValueType); + const JSSourceValuePtr jsBaseValue = std::static_pointer_cast<JSSourceValue>( + baseValue->clone(loaderState.itemPool())); + jsValue->setBaseValue(jsBaseValue); + std::vector<JSSourceValue::Alternative> alternatives = jsValue->alternatives(); + jsValue->clearAlternatives(); + for (JSSourceValue::Alternative &a : alternatives) { + a.value->setBaseValue(jsBaseValue); + jsValue->addAlternative(a); + } + } + } + dst->setProperty(name, value); +} + +bool ProductsCollector::Private::mergeExportItems(ProductContext &productContext) +{ + const SetupProjectParameters ¶meters = loaderState.parameters(); + Evaluator &evaluator = loaderState.evaluator(); + Logger &logger = loaderState.logger(); + TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); + + std::vector<Item *> exportItems; + QList<Item *> children = productContext.item->children(); + + const auto isExport = [](Item *item) { return item->type() == ItemType::Export; }; + std::copy_if(children.cbegin(), children.cend(), std::back_inserter(exportItems), isExport); + qbs::Internal::removeIf(children, isExport); + + // Note that we do not return if there are no Export items: The "merged" item becomes the + // "product module", which always needs to exist, regardless of whether the product sources + // actually contain an Export item or not. + if (!exportItems.empty()) + productContext.item->setChildren(children); + + Item *merged = Item::create(&loaderState.itemPool(), ItemType::Export); + const QString &nameKey = StringConstants::nameProperty(); + const ValuePtr nameValue = VariantValue::create(productContext.name); + merged->setProperty(nameKey, nameValue); + Set<FileContextConstPtr> filesWithExportItem; + QVariantMap defaultParameters; + for (Item * const exportItem : exportItems) { + topLevelProject.checkCancelation(); + if (Q_UNLIKELY(filesWithExportItem.contains(exportItem->file()))) + throw ErrorInfo(Tr::tr("Multiple Export items in one product are prohibited."), + exportItem->location()); + exportItem->setProperty(nameKey, nameValue); + if (!checkExportItemCondition(exportItem, productContext)) + continue; + filesWithExportItem += exportItem->file(); + for (Item * const child : exportItem->children()) { + if (child->type() == ItemType::Parameters) { + adjustParametersScopes(child, child); + defaultParameters = mergeDependencyParameters(defaultParameters, + getJsVariant(evaluator.engine()->context(), + evaluator.scriptValue(child)).toMap()); + } else { + Item::addChild(merged, child); + } + } + const Item::PropertyDeclarationMap &decls = exportItem->propertyDeclarations(); + for (auto it = decls.constBegin(); it != decls.constEnd(); ++it) { + const PropertyDeclaration &newDecl = it.value(); + const PropertyDeclaration &existingDecl = merged->propertyDeclaration(it.key()); + if (existingDecl.isValid() && existingDecl.type() != newDecl.type()) { + ErrorInfo error(Tr::tr("Export item in inherited item redeclares property " + "'%1' with different type.").arg(it.key()), exportItem->location()); + handlePropertyError(error, parameters, logger); + } + merged->setPropertyDeclaration(newDecl.name(), newDecl); + } + for (QMap<QString, ValuePtr>::const_iterator it = exportItem->properties().constBegin(); + it != exportItem->properties().constEnd(); ++it) { + mergeProperty(merged, it.key(), it.value()); + } + } + merged->setFile(exportItems.empty() + ? productContext.item->file() : exportItems.back()->file()); + merged->setLocation(exportItems.empty() + ? productContext.item->location() : exportItems.back()->location()); + Item::addChild(productContext.item, merged); + merged->setupForBuiltinType(parameters.deprecationWarningMode(), logger); + + productContext.mergedExportItem = merged; + productContext.defaultParameters = defaultParameters; + return !exportItems.empty(); + +} + +// TODO: This seems dubious. Can we merge the conditions instead? +bool ProductsCollector::Private::checkExportItemCondition(Item *exportItem, + const ProductContext &product) +{ + class ScopeHandler { + public: + ScopeHandler(Item *exportItem, const ProductContext &productContext, Item **cachedScopeItem, + ItemPool &itemPool) : m_exportItem(exportItem), m_itemPool(itemPool) + { + if (!*cachedScopeItem) + *cachedScopeItem = Item::create(&m_itemPool, ItemType::Scope); + Item * const scope = *cachedScopeItem; + QBS_CHECK(productContext.item->file()); + scope->setFile(productContext.item->file()); + scope->setScope(productContext.item); + productContext.project->scope->copyProperty(StringConstants::projectVar(), scope); + productContext.scope->copyProperty(StringConstants::productVar(), scope); + QBS_CHECK(!exportItem->scope()); + exportItem->setScope(scope); + } + ~ScopeHandler() { m_exportItem->setScope(nullptr); } + + private: + Item * const m_exportItem; + ItemPool &m_itemPool; + } scopeHandler(exportItem, product, &tempScopeItem, loaderState.itemPool()); + return loaderState.topLevelProject().checkItemCondition(exportItem, loaderState.evaluator()); +} + +void ProductsCollector::Private::initProductProperties(const ProductContext &product) +{ + QString buildDir = ResolvedProduct::deriveBuildDirectoryName(product.name, + product.multiplexConfigurationId); + buildDir = FileInfo::resolvePath(product.project->topLevelProject->buildDirectory(), buildDir); + product.item->setProperty(StringConstants::buildDirectoryProperty(), + VariantValue::create(buildDir)); + const QString sourceDir = QFileInfo(product.item->file()->filePath()).absolutePath(); + product.item->setProperty(StringConstants::sourceDirectoryProperty(), + VariantValue::create(sourceDir)); +} + +void ProductsCollector::Private::checkProjectNamesInOverrides() +{ + for (const QString &projectNameInOverride + : loaderState.topLevelProject().projectNamesUsedInOverrides()) { + if (disabledProjects.contains(projectNameInOverride)) + continue; + if (!any_of(loaderState.topLevelProject().projects(), + [&projectNameInOverride](const ProjectContext *p) { + return p->name == projectNameInOverride; })) { + handlePropertyError(Tr::tr("Unknown project '%1' in property override.") + .arg(projectNameInOverride), + loaderState.parameters(), loaderState.logger()); + } + } +} + +void ProductsCollector::Private::collectProductsByNameAndItem() +{ + TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); + for (ProjectContext * const project : topLevelProject.projects()) { + for (ProductContext &product : project->products) + topLevelProject.addProduct(product); + } +} + +void ProductsCollector::Private::checkProductNamesInOverrides() +{ + TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); + for (const QString &productNameInOverride : topLevelProject.productNamesUsedInOverrides()) { + if (!topLevelProject.productWithNameAndConstraint( + productNameInOverride, [&productNameInOverride](const ProductContext &product) { + // In an override string such as "a.b.c:d, we cannot tell whether we have a product + // "a" and a module "b.c" or a product "a.b" and a module "c", so we need to take + // care not to emit false positives here. + return product.name == productNameInOverride + || product.name.startsWith(productNameInOverride + StringConstants::dot()); + })) { + handlePropertyError(Tr::tr("Unknown product '%1' in property override.") + .arg(productNameInOverride), + loaderState.parameters(), loaderState.logger()); + } + } +} + +ProductsCollector::Private::TempBaseModuleAttacher::TempBaseModuleAttacher( + ProductsCollector::Private *d, ProductContext &product) + : m_productItem(product.item) +{ + const ValuePtr qbsValue = m_productItem->property(StringConstants::qbsModule()); + + // Cloning is necessary because the original value will get "instantiated" now. + if (qbsValue) + m_origQbsValue = qbsValue->clone(d->loaderState.itemPool()); + + m_tempBaseModule = loadBaseModule(product, m_productItem, d->loaderState); +} + +void ProductsCollector::Private::TempBaseModuleAttacher::drop() +{ + if (!m_tempBaseModule) + return; + + // "Unload" the qbs module again. + if (m_origQbsValue) + m_productItem->setProperty(StringConstants::qbsModule(), m_origQbsValue); + else + m_productItem->removeProperty(StringConstants::qbsModule()); + m_productItem->removeModules(); + m_tempBaseModule = nullptr; +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productscollector.h b/src/lib/corelib/loader/productscollector.h new file mode 100644 index 000000000..246361b37 --- /dev/null +++ b/src/lib/corelib/loader/productscollector.h @@ -0,0 +1,63 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +#include <tools/pimpl.h> + +namespace qbs::Internal { +class Item; +class LoaderState; + +// Traverses the root project item and fills the TopLevelProjectContext with all the +// product and sub-project information, including those coming from referenced files. +class ProductsCollector +{ +public: + ProductsCollector(LoaderState &loaderState); + ~ProductsCollector(); + + void run(Item *rootProject); + +private: + class Private; + Pimpl<Private> d; +}; + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productsresolver.cpp b/src/lib/corelib/loader/productsresolver.cpp new file mode 100644 index 000000000..997a4dc57 --- /dev/null +++ b/src/lib/corelib/loader/productsresolver.cpp @@ -0,0 +1,599 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "productsresolver.h" + +#include "itemreader.h" +#include "loaderutils.h" +#include "productresolver.h" + +#include <language/language.h> +#include <language/scriptengine.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/profiling.h> +#include <tools/progressobserver.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <algorithm> +#include <condition_variable> +#include <future> +#include <mutex> +#include <queue> +#include <system_error> +#include <thread> +#include <unordered_map> +#include <vector> + +namespace qbs::Internal { +namespace { +struct ThreadInfo { + ThreadInfo(std::future<void> &&future, LoaderState &loaderState) + : future(std::move(future)), loaderState(loaderState) + {} + std::future<void> future; + LoaderState &loaderState; + bool done = false; +}; + +struct ProductWithLoaderState { + ProductWithLoaderState(ProductContext &product, LoaderState *loaderState) + : product(&product), loaderState(loaderState) {} + ProductContext * const product; + LoaderState *loaderState; +}; + +class ThreadsLocker { +public: + ThreadsLocker(std::launch mode, std::mutex &mutex) { + if (mode == std::launch::async) + lock = std::make_unique<std::unique_lock<std::mutex>>(mutex); + } + std::unique_ptr<std::unique_lock<std::mutex>> lock; +}; +} // namespace + +class ProductsResolver +{ +public: + ProductsResolver(LoaderState &loaderState) : m_loaderState(loaderState) {} + void resolve(); + +private: + void initialize(); + void initializeProductQueue(); + void initializeLoaderStatePool(); + void runScheduler(); + void scheduleNext(); + bool tryToReserveLoaderState(ProductWithLoaderState &product, Deferral deferral); + std::optional<std::pair<ProductContext *, Deferral>> + unblockProductWaitingForLoaderState(LoaderState &loaderState); + void startJob(const ProductWithLoaderState &product, Deferral deferral); + void checkForCancelation(); + void handleFinishedThreads(); + void queueProductForScheduling(const ProductWithLoaderState &product, Deferral deferral); + void waitForSingleDependency(const ProductWithLoaderState &product, ProductContext &dependency); + void waitForBulkDependency(const ProductWithLoaderState &product); + void unblockProductsWaitingForDependency(ProductContext &finishedProduct); + void postProcess(); + void checkForMissedBulkDependencies(const ProductContext &product); + + static int dependsItemCount(const Item *item); + static int dependsItemCount(ProductContext &product); + + LoaderState &m_loaderState; + std::queue<std::pair<ProductWithLoaderState, int>> m_productsToSchedule; + std::vector<ProductContext *> m_finishedProducts; + std::unordered_map<ProductContext *, + std::vector<ProductWithLoaderState>> m_waitingForSingleDependency; + std::vector<ProductWithLoaderState> m_waitingForBulkDependency; + std::unordered_map<LoaderState *, std::queue<std::pair<ProductContext *, Deferral>>> m_waitingForLoaderState; + std::unordered_map<ProductContext *, ThreadInfo> m_runningThreads; + std::mutex m_threadsMutex; + std::condition_variable m_threadsNotifier; + std::vector<std::unique_ptr<ScriptEngine>> m_enginePool; + std::vector<std::unique_ptr<LoaderState>> m_loaderStatePool; + std::vector<LoaderState *> m_availableLoaderStates; + std::mutex m_cancelingMutex; + std::launch m_asyncMode = std::launch::async; + int m_maxJobCount = m_loaderState.parameters().maxJobCount(); + bool m_canceling = false; +}; + +void resolveProducts(LoaderState &loaderState) +{ + ProductsResolver(loaderState).resolve(); +} + +void ProductsResolver::resolve() +{ + initialize(); + try { + runScheduler(); + } catch (const ErrorInfo &e) { + for (auto &thread : m_runningThreads) + thread.second.future.wait(); + throw e; + } + postProcess(); +} + +void ProductsResolver::initialize() +{ + initializeProductQueue(); + initializeLoaderStatePool(); +} + +void ProductsResolver::initializeProductQueue() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + std::vector<ProductContext *> sortedProducts; + for (ProjectContext * const projectContext : topLevelProject.projects()) { + for (ProductContext &product : projectContext->products) { + topLevelProject.addProductToHandle(product); + const auto it = std::lower_bound(sortedProducts.begin(), sortedProducts.end(), product, + [&product](ProductContext *p1, const ProductContext &) { + return dependsItemCount(*p1) < dependsItemCount(product); + }); + sortedProducts.insert(it, &product); + } + } + + for (ProductContext * const product : sortedProducts) { + queueProductForScheduling(ProductWithLoaderState(*product, nullptr), Deferral::Allowed); + if (product->shadowProduct) { + topLevelProject.addProductToHandle(*product->shadowProduct); + queueProductForScheduling(ProductWithLoaderState(*product->shadowProduct, nullptr), + Deferral::Allowed); + } + } +} + +void ProductsResolver::initializeLoaderStatePool() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + + // Adapt max job count: It makes no sense to have it be higher than the number of products + // or what can actually be run concurrently. In both cases, we would simply waste resources. + const int maxConcurrency = std::thread::hardware_concurrency(); + if (maxConcurrency > 0 && maxConcurrency < m_maxJobCount) + m_maxJobCount = maxConcurrency; + if (m_maxJobCount > topLevelProject.productsToHandleCount()) + m_maxJobCount = topLevelProject.productsToHandleCount(); + + // The number of engines and loader states we need to allocate here is one less than the + // total number of concurrent jobs, as we already have one loader state that we can re-use. + if (m_maxJobCount > 1) + m_enginePool.reserve(m_maxJobCount - 1); + m_loaderStatePool.reserve(m_enginePool.size()); + m_availableLoaderStates.reserve(m_enginePool.size() + 1); + m_availableLoaderStates.push_back(&m_loaderState); + for (std::size_t i = 0; i < m_enginePool.capacity(); ++i) { + ScriptEngine &engine = *m_enginePool.emplace_back( + ScriptEngine::create(m_loaderState.logger(), EvalContext::PropertyEvaluation)); + ItemPool &itemPool = topLevelProject.createItemPool(); + engine.setEnvironment(m_loaderState.parameters().adjustedEnvironment()); + auto loaderState = std::make_unique<LoaderState>( + m_loaderState.parameters(), topLevelProject, itemPool, engine, + m_loaderState.logger()); + m_loaderStatePool.push_back(std::move(loaderState)); + m_availableLoaderStates.push_back(m_loaderStatePool.back().get()); + if (topLevelProject.progressObserver()) + topLevelProject.progressObserver()->addScriptEngine(m_enginePool.back().get()); + } + qCDebug(lcLoaderScheduling) << "using" << m_availableLoaderStates.size() << "loader states"; + if (int(m_availableLoaderStates.size()) == 1) + m_asyncMode = std::launch::deferred; +} + +void ProductsResolver::runScheduler() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_loaderState.topLevelProject().timingData().resolvingProducts + : nullptr); + + ThreadsLocker threadsLock(m_asyncMode, m_threadsMutex); + while (true) { + scheduleNext(); + if (m_runningThreads.empty()) + break; + if (m_asyncMode == std::launch::async) { + qCDebug(lcLoaderScheduling()) << "scheduling paused, waiting for threads to finish"; + m_threadsNotifier.wait(*threadsLock.lock); + } + checkForCancelation(); + handleFinishedThreads(); + } + + QBS_CHECK(m_productsToSchedule.empty()); + QBS_CHECK(m_loaderState.topLevelProject().productsToHandleCount() == 0); + QBS_CHECK(m_runningThreads.empty()); + QBS_CHECK(m_waitingForSingleDependency.empty()); + QBS_CHECK(m_waitingForBulkDependency.empty()); +} + +void ProductsResolver::scheduleNext() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &topLevelProject.timingData().schedulingProducts : nullptr); + while (m_maxJobCount > int(m_runningThreads.size()) && !m_productsToSchedule.empty()) { + auto [product, toHandleCountOnInsert] = m_productsToSchedule.front(); + m_productsToSchedule.pop(); + + qCDebug(lcLoaderScheduling) << "potentially scheduling product" + << product.product->displayName() + << "unhandled product count on queue insertion" + << toHandleCountOnInsert << "current unhandled product count" + << topLevelProject.productsToHandleCount(); + + // If the number of unfinished products has shrunk since the last time we tried handling + // this product, there has been forward progress and we can allow a deferral. + const Deferral deferral = toHandleCountOnInsert == -1 + || toHandleCountOnInsert > topLevelProject.productsToHandleCount() + ? Deferral::Allowed : Deferral::NotAllowed; + + if (!tryToReserveLoaderState(product, deferral)) + continue; + + startJob(product, deferral); + } + + // There are jobs running, so forward progress is still possible. + if (!m_runningThreads.empty()) + return; + + QBS_CHECK(m_productsToSchedule.empty()); + + // If we end up here, nothing was scheduled in the loop above, which means that either ... + // a) ... we are done or + // b) ... we finally need to schedule our bulk dependencies or + // c) ... we need to schedule products waiting for an unhandled dependency. + // In the latter case, the project has at least one dependency cycle, and the + // DependencyResolver will emit an error. + + // a) + if (m_waitingForBulkDependency.empty() && m_waitingForSingleDependency.empty()) + return; + + // b) + for (const ProductWithLoaderState &product : m_waitingForBulkDependency) + queueProductForScheduling(product, Deferral::NotAllowed); + if (!m_productsToSchedule.empty()) { + m_waitingForBulkDependency.clear(); + scheduleNext(); + return; + } + + // c) + for (const auto &e : m_waitingForSingleDependency) { + for (const ProductWithLoaderState &p : e.second) + queueProductForScheduling(p, Deferral::NotAllowed); + } + QBS_CHECK(!m_productsToSchedule.empty()); + scheduleNext(); +} + +bool ProductsResolver::tryToReserveLoaderState(ProductWithLoaderState &product, Deferral deferral) +{ + QBS_CHECK(!m_availableLoaderStates.empty()); + if (!product.loaderState) { + product.loaderState = m_availableLoaderStates.back(); + m_availableLoaderStates.pop_back(); + return true; + } + if (const auto it = std::find(m_availableLoaderStates.begin(), m_availableLoaderStates.end(), + product.loaderState); it != m_availableLoaderStates.end()) { + m_availableLoaderStates.erase(it); + return true; + } + qCDebug(lcLoaderScheduling) << "loader state" << product.loaderState << " for product" + << product.product->displayName() + << "not available, adding product to wait queue"; + m_waitingForLoaderState[product.loaderState].push({product.product, deferral}); + return false; +} + +std::optional<std::pair<ProductContext *, Deferral>> +ProductsResolver::unblockProductWaitingForLoaderState(LoaderState &loaderState) +{ + auto &waitingForLoaderState = m_waitingForLoaderState[&loaderState]; + if (waitingForLoaderState.empty()) + return {}; + const auto product = waitingForLoaderState.front(); + waitingForLoaderState.pop(); + qCDebug(lcLoaderScheduling) << "loader state" << &loaderState << "now available for product" + << product.first->displayName(); + return product; +} + +void ProductsResolver::startJob(const ProductWithLoaderState &product, Deferral deferral) +{ + QBS_CHECK(product.loaderState); + qCDebug(lcLoaderScheduling) << "scheduling product" << product.product->displayName() + << "with loader state" << product.loaderState + << "and deferral mode" << int(deferral); + try { + const auto it = m_runningThreads.emplace(product.product, ThreadInfo(std::async(m_asyncMode, + [this, product, deferral] { + product.loaderState->itemReader().setExtraSearchPathsStack( + product.product->project->searchPathsStack); + resolveProduct(*product.product, deferral, *product.loaderState); + + // The search paths stack can change during dependency resolution + // (due to module providers); check that we've rolled back all the changes + QBS_CHECK(product.loaderState->itemReader().extraSearchPathsStack() + == product.product->project->searchPathsStack); + + std::lock_guard cancelingLock(m_cancelingMutex); + if (m_canceling) + return; + ThreadsLocker threadsLock(m_asyncMode, m_threadsMutex); + if (const auto it = m_runningThreads.find(product.product); + it != m_runningThreads.end()) { + it->second.done = true; + qCDebug(lcLoaderScheduling) << "thread for product" + << product.product->displayName() + << "finished, waking up scheduler"; + m_threadsNotifier.notify_one(); + } + }), *product.loaderState)); + + // With just one worker thread, the notify/wait overhead would be excessive, so + // we run the task synchronously. + if (m_asyncMode == std::launch::deferred) { + qCDebug(lcLoaderScheduling) << "blocking on product thread immediately"; + it.first->second.future.wait(); + } + } catch (const std::system_error &e) { + if (e.code() != std::errc::resource_unavailable_try_again) + throw e; + qCWarning(lcLoaderScheduling) << "failed to create thread"; + if (m_maxJobCount >= 2) { + m_maxJobCount /= 2; + qCWarning(lcLoaderScheduling) << "throttling down to" << m_maxJobCount << "jobs"; + } + queueProductForScheduling(product, deferral); + m_availableLoaderStates.push_back(product.loaderState); + } +} + +void ProductsResolver::checkForCancelation() +{ + if (m_loaderState.topLevelProject().isCanceled()) { + m_cancelingMutex.lock(); + m_canceling = true; + m_cancelingMutex.unlock(); + for (auto &thread : m_runningThreads) + thread.second.future.wait(); + throw CancelException(); + } +} + +void ProductsResolver::handleFinishedThreads() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &topLevelProject.timingData().schedulingProducts : nullptr); + + std::vector<std::pair<ProductWithLoaderState, Deferral>> productsToScheduleDirectly; + for (auto it = m_runningThreads.begin(); it != m_runningThreads.end();) { + ThreadInfo &ti = it->second; + if (!ti.done) { + ++it; + continue; + } + ti.future.wait(); + ProductContext &product = *it->first; + LoaderState &loaderState = ti.loaderState; + it = m_runningThreads.erase(it); + + qCDebug(lcLoaderScheduling) << "handling finished thread for product" + << product.displayName() + << "current unhandled product count is" + << topLevelProject.productsToHandleCount(); + + // If there are products waiting for the loader state used in the finished thread, + // we can start a job for one of them right away (but not in the loop, + // because startJob() modifies the thread list we are currently iterating over). + if (const auto productInfo = unblockProductWaitingForLoaderState(loaderState)) { + productsToScheduleDirectly.emplace_back( + ProductWithLoaderState(*productInfo->first, &loaderState), + productInfo->second); + } else { + qCDebug(lcLoaderScheduling) << "making loader state" << &loaderState + << "available again"; + m_availableLoaderStates.push_back(&loaderState); + } + + // If we encountered a dependency to an in-progress product or to a bulk dependency, + // we defer handling this product. + if (product.dependenciesResolvingPending()) { + qCDebug(lcLoaderScheduling) << "dependencies resolving not finished for product" + << product.displayName(); + const auto pending = product.pendingDependency(); + switch (pending.first) { + case ProductDependency::Single: + waitForSingleDependency(ProductWithLoaderState(product, &loaderState), + *pending.second); + break; + case ProductDependency::Bulk: + waitForBulkDependency(ProductWithLoaderState(product, &loaderState)); + break; + case ProductDependency::None: + // This can happen if the dependency has finished in between the check in + // DependencyResolver and the one here. + QBS_CHECK(pending.second); + queueProductForScheduling(ProductWithLoaderState(product, &loaderState), + Deferral::Allowed); + break; + } + topLevelProject.incProductDeferrals(); + } else { + qCDebug(lcLoaderScheduling) << "product" << product.displayName() << "finished"; + topLevelProject.removeProductToHandle(product); + if (!product.name.startsWith(StringConstants::shadowProductPrefix())) + m_finishedProducts.push_back(&product); + topLevelProject.timingData() += product.timingData; + checkForMissedBulkDependencies(product); + topLevelProject.registerBulkDependencies(product); + unblockProductsWaitingForDependency(product); + } + } + + for (const auto &productInfo : productsToScheduleDirectly) + startJob(productInfo.first, productInfo.second); +} + +void ProductsResolver::queueProductForScheduling(const ProductWithLoaderState &product, + Deferral deferral) +{ + qCDebug(lcLoaderScheduling) << "queueing product" << product.product->displayName() + << "with deferral mode" << int(deferral); + m_productsToSchedule.emplace(product, deferral == Deferral::Allowed + ? -1 : m_loaderState.topLevelProject().productsToHandleCount()); +} + +void ProductsResolver::waitForSingleDependency(const ProductWithLoaderState &product, + ProductContext &dependency) +{ + qCDebug(lcLoaderScheduling) << "product" << product.product->displayName() + << "now waiting for single dependency" + << dependency.displayName(); + m_waitingForSingleDependency[&dependency].push_back(product); +} + +void ProductsResolver::waitForBulkDependency(const ProductWithLoaderState &product) +{ + qCDebug(lcLoaderScheduling) << "product" << product.product->displayName() + << "now waiting for bulk dependency"; + m_waitingForBulkDependency.push_back(product); +} + +void ProductsResolver::unblockProductsWaitingForDependency(ProductContext &finishedProduct) +{ + const auto it = m_waitingForSingleDependency.find(&finishedProduct); + if (it == m_waitingForSingleDependency.end()) + return; + + qCDebug(lcLoaderScheduling) << "unblocking all products waiting for now-finished product" << + finishedProduct.displayName(); + for (const ProductWithLoaderState &p : it->second) { + qCDebug(lcLoaderScheduling) << " unblocking product" << p.product->displayName(); + queueProductForScheduling(p, Deferral::Allowed); + } + m_waitingForSingleDependency.erase(it); +} + +void ProductsResolver::postProcess() +{ + for (ProductContext * const product : m_finishedProducts) { + if (product->product) + product->product->project->products.push_back(product->product); + + // This has to be done in post-processing, because we need both product and shadow product + // to be ready, and contrary to what one might assume, there is no proper ordering + // between them regarding dependency resolving. + setupExports(*product, m_loaderState); + } + + for (const auto &engine : m_enginePool) + m_loaderState.topLevelProject().collectDataFromEngine(*engine); + + QBS_CHECK(!m_loaderState.topLevelProject().projects().empty()); + const auto project = std::dynamic_pointer_cast<TopLevelProject>( + m_loaderState.topLevelProject().projects().front()->project); + QBS_CHECK(project); + for (LoaderState * const loaderState : m_availableLoaderStates) + project->warningsEncountered << loaderState->logger().warnings(); +} + +void ProductsResolver::checkForMissedBulkDependencies(const ProductContext &product) +{ + if (!product.product || !product.product->enabled || !product.bulkDependencies.empty()) + return; + for (const FileTag &tag : product.product->fileTags) { + for (const auto &[p, location] + : m_loaderState.topLevelProject().finishedProductsWithBulkDependency(tag)) { + if (!p->product->enabled) + continue; + if (p->name == product.name) + continue; + ErrorInfo e; + e.append(Tr::tr("Cyclic dependencies detected:")); + e.append(Tr::tr("First product is '%1'.") + .arg(product.displayName()), product.item->location()); + e.append(Tr::tr("Second product is '%1'.") + .arg(p->displayName()), p->item->location()); + e.append(Tr::tr("Dependency from %1 to %2 was established via Depends.productTypes.") + .arg(p->displayName(), product.displayName()), location); + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw e; + m_loaderState.logger().printWarning(e); + m_loaderState.logger().printWarning( + ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") + .arg(product.displayName()), product.item->location())); + m_loaderState.logger().printWarning( + ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") + .arg(p->displayName()), p->item->location())); + product.product->enabled = false; + p->product->enabled = false; + } + } +} + +int ProductsResolver::dependsItemCount(const Item *item) +{ + int count = 0; + for (const Item * const child : item->children()) { + if (child->type() == ItemType::Depends) + ++count; + } + return count; +} + +int ProductsResolver::dependsItemCount(ProductContext &product) +{ + if (product.dependsItemCount == -1) + product.dependsItemCount = dependsItemCount(product.item); + return product.dependsItemCount; +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productsresolver.h b/src/lib/corelib/loader/productsresolver.h new file mode 100644 index 000000000..b8b4b7516 --- /dev/null +++ b/src/lib/corelib/loader/productsresolver.h @@ -0,0 +1,47 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs::Internal { +class LoaderState; + +void resolveProducts(LoaderState &loaderState); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/projectresolver.cpp b/src/lib/corelib/loader/projectresolver.cpp new file mode 100644 index 000000000..be99d547d --- /dev/null +++ b/src/lib/corelib/loader/projectresolver.cpp @@ -0,0 +1,562 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "projectresolver.h" + +#include "itemreader.h" +#include "loaderutils.h" +#include "productscollector.h" +#include "productsresolver.h" + +#include <jsextensions/jsextensions.h> +#include <jsextensions/moduleproperties.h> +#include <language/artifactproperties.h> +#include <language/builtindeclarations.h> +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/filetags.h> +#include <language/item.h> +#include <language/itempool.h> +#include <language/language.h> +#include <language/propertymapinternal.h> +#include <language/resolvedfilecontext.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/error.h> +#include <tools/fileinfo.h> +#include <tools/joblimits.h> +#include <tools/jsliterals.h> +#include <tools/profile.h> +#include <tools/profiling.h> +#include <tools/progressobserver.h> +#include <tools/scripttools.h> +#include <tools/qbsassert.h> +#include <tools/qttools.h> +#include <tools/set.h> +#include <tools/settings.h> +#include <tools/setupprojectparameters.h> +#include <tools/stlutils.h> +#include <tools/stringconstants.h> + +#include <quickjs.h> + +#include <QtCore/qdir.h> +#include <QtCore/qregularexpression.h> + +#include <algorithm> +#include <memory> +#include <queue> +#include <utility> +#include <vector> + +namespace qbs { +namespace Internal { + +static SetupProjectParameters finalizedProjectParameters(const SetupProjectParameters &in, + Logger &logger) +{ + SetupProjectParameters params = in; + if (params.topLevelProfile().isEmpty()) { + Settings settings(params.settingsDirectory()); + QString profileName = settings.defaultProfile(); + if (profileName.isEmpty()) { + logger.qbsDebug() << Tr::tr("No profile specified and no default profile exists. " + "Using default property values."); + profileName = Profile::fallbackName(); + } + params.setTopLevelProfile(profileName); + params.expandBuildConfiguration(); + } + params.finalizeProjectFilePath(); + + QBS_CHECK(QFileInfo(params.projectFilePath()).isAbsolute()); + QBS_CHECK(FileInfo::isAbsolute(params.buildRoot())); + + return params; +} + +class ProjectResolver::Private +{ +public: + Private(const SetupProjectParameters ¶meters, ScriptEngine *engine, Logger &&logger) + : setupParams(finalizedProjectParameters(parameters, logger)), engine(engine), + logger(std::move(logger)) {} + + TopLevelProjectPtr resolveTopLevelProject(); + void resolveProject(ProjectContext *projectContext); + void resolveProjectFully(ProjectContext *projectContext); + void resolveSubProject(Item *item, ProjectContext *projectContext); + void checkOverriddenValues(); + void collectNameFromOverride(const QString &overrideString); + void loadTopLevelProjectItem(); + void buildProjectTree(); + + void printProfilingInfo(); + + const SetupProjectParameters setupParams; + ScriptEngine * const engine; + mutable Logger logger; + ItemPool itemPool; + TopLevelProjectContext topLevelProject; + LoaderState state{setupParams, topLevelProject, itemPool, *engine, logger}; + Item *rootProjectItem = nullptr; +}; + +ProjectResolver::ProjectResolver(const SetupProjectParameters ¶meters, ScriptEngine *engine, + Logger logger) + : d(makePimpl<Private>(parameters, engine, std::move(logger))) +{ + d->logger.storeWarnings(); +} + +ProjectResolver::~ProjectResolver() = default; + +void ProjectResolver::setProgressObserver(ProgressObserver *observer) +{ + d->state.topLevelProject().setProgressObserver(observer); +} + +void ProjectResolver::setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes) +{ + d->state.topLevelProject().setOldProjectProbes(oldProbes); +} + +void ProjectResolver::setOldProductProbes( + const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes) +{ + d->state.topLevelProject().setOldProductProbes(oldProbes); +} + +void ProjectResolver::setLastResolveTime(const FileTime &time) +{ + d->state.topLevelProject().setLastResolveTime(time); +} + +void ProjectResolver::setStoredProfiles(const QVariantMap &profiles) +{ + d->state.topLevelProject().setProfileConfigs(profiles); +} + +void ProjectResolver::setStoredModuleProviderInfo(const StoredModuleProviderInfo &providerInfo) +{ + d->state.topLevelProject().setModuleProvidersCache(providerInfo.providers); +} + +static void checkForDuplicateProductNames(const TopLevelProjectConstPtr &project) +{ + const std::vector<ResolvedProductPtr> allProducts = project->allProducts(); + for (size_t i = 0; i < allProducts.size(); ++i) { + const ResolvedProductConstPtr product1 = allProducts.at(i); + const QString productName = product1->uniqueName(); + for (size_t j = i + 1; j < allProducts.size(); ++j) { + const ResolvedProductConstPtr product2 = allProducts.at(j); + if (product2->uniqueName() == productName) { + ErrorInfo error; + error.append(Tr::tr("Duplicate product name '%1'.").arg(product1->name)); + error.append(Tr::tr("First product defined here."), product1->location); + error.append(Tr::tr("Second product defined here."), product2->location); + throw error; + } + } + } +} + +TopLevelProjectPtr ProjectResolver::resolve() +{ + qCDebug(lcProjectResolver) << "resolving" << d->setupParams.projectFilePath(); + + d->engine->setEnvironment(d->setupParams.adjustedEnvironment()); + d->engine->checkAndClearException({}); + d->engine->clearImportsCache(); + d->engine->clearRequestedProperties(); + d->engine->enableProfiling(d->setupParams.logElapsedTime()); + d->logger.clearWarnings(); + EvalContextSwitcher evalContextSwitcher(d->engine, EvalContext::PropertyEvaluation); + + // At this point, we cannot set a sensible total effort, because we know nothing about + // the project yet. That's why we use a placeholder here, so the user at least + // sees that an operation is starting. The real total effort will be set later when + // we have enough information. + if (ProgressObserver * const observer = d->state.topLevelProject().progressObserver()) { + observer->initialize(Tr::tr("Resolving project for configuration %1") + .arg(TopLevelProject::deriveId(d->setupParams.finalBuildConfigurationTree())), 1); + observer->addScriptEngine(d->engine); + } + + const FileTime resolveTime = FileTime::currentTime(); + TopLevelProjectPtr tlp; + try { + d->checkOverriddenValues(); + d->loadTopLevelProjectItem(); + d->buildProjectTree(); + tlp = d->resolveTopLevelProject(); + } catch (const CancelException &) { + throw ErrorInfo(Tr::tr("Project resolving canceled for configuration '%1'.") + .arg(TopLevelProject::deriveId( + d->setupParams.finalBuildConfigurationTree()))); + } + tlp->lastStartResolveTime = resolveTime; + tlp->lastEndResolveTime = FileTime::currentTime(); + + // E.g. if the top-level project is disabled. + if (ProgressObserver * const observer = d->state.topLevelProject().progressObserver()) { + observer->setFinished(); + d->printProfilingInfo(); + } + return tlp; +} + +static void makeSubProjectNamesUniqe(const ResolvedProjectPtr &parentProject) +{ + Set<QString> subProjectNames; + Set<ResolvedProjectPtr> projectsInNeedOfNameChange; + for (const ResolvedProjectPtr &p : std::as_const(parentProject->subProjects)) { + if (!subProjectNames.insert(p->name).second) + projectsInNeedOfNameChange << p; + makeSubProjectNamesUniqe(p); + } + while (!projectsInNeedOfNameChange.empty()) { + auto it = projectsInNeedOfNameChange.begin(); + while (it != projectsInNeedOfNameChange.end()) { + const ResolvedProjectPtr p = *it; + p->name += QLatin1Char('_'); + if (subProjectNames.insert(p->name).second) { + it = projectsInNeedOfNameChange.erase(it); + } else { + ++it; + } + } + } +} + +TopLevelProjectPtr ProjectResolver::Private::resolveTopLevelProject() +{ + if (ProgressObserver * const observer = state.topLevelProject().progressObserver()) + observer->setMaximum(state.topLevelProject().productCount()); + TopLevelProjectPtr project = TopLevelProject::create(); + project->buildDirectory = TopLevelProject::deriveBuildDirectory( + setupParams.buildRoot(), + TopLevelProject::deriveId(setupParams.finalBuildConfigurationTree())); + if (!state.topLevelProject().projects().empty()) { + ProjectContext * const projectContext = state.topLevelProject().projects().front(); + QBS_CHECK(projectContext->item == rootProjectItem); + projectContext->project = project; + resolveProject(projectContext); + } + resolveProducts(state); + ErrorInfo accumulatedErrors; + { + const auto queuedErrors = state.topLevelProject().queuedErrors(); + for (const ErrorInfo &e : queuedErrors.get()) + appendError(accumulatedErrors, e); + } + if (accumulatedErrors.hasError()) + throw accumulatedErrors; + + project->buildSystemFiles.unite(state.topLevelProject().buildSystemFiles()); + project->profileConfigs = state.topLevelProject().profileConfigs(); + project->codeLinks = state.topLevelProject().codeLinks(); + const QVariantMap &profiles = state.topLevelProject().localProfiles(); + for (auto it = profiles.begin(); it != profiles.end(); ++it) + project->profileConfigs.remove(it.key()); + project->probes = state.topLevelProject().projectLevelProbes(); + project->moduleProviderInfo.providers = state.topLevelProject().moduleProvidersCache(); + project->setBuildConfiguration(setupParams.finalBuildConfigurationTree()); + project->overriddenValues = setupParams.overriddenValues(); + state.topLevelProject().collectDataFromEngine(*engine); + makeSubProjectNamesUniqe(project); + checkForDuplicateProductNames(project); + project->warningsEncountered << logger.warnings(); + + return project; +} + +void ProjectResolver::Private::resolveProject(ProjectContext *projectContext) +{ + state.topLevelProject().checkCancelation(); + + if (projectContext->parent) { + projectContext->project = ResolvedProject::create(); + projectContext->parent->project->subProjects.push_back(projectContext->project); + projectContext->project->parentProject = projectContext->parent->project; + projectContext->project->enabled = projectContext->parent->project->enabled; + } + projectContext->project->location = projectContext->item->location(); + + try { + resolveProjectFully(projectContext); + } catch (const ErrorInfo &error) { + if (!projectContext->project->enabled) { + qCDebug(lcProjectResolver) << "error resolving project" + << projectContext->project->location << error.toString(); + return; + } + if (setupParams.productErrorMode() == ErrorHandlingMode::Strict) + throw; + logger.printWarning(error); + } +} + +void ProjectResolver::Private::resolveProjectFully(ProjectContext *projectContext) +{ + Item * const item = projectContext->item; + projectContext->project->enabled = projectContext->project->enabled + && state.evaluator().boolValue(item, StringConstants::conditionProperty()); + projectContext->project->name = state.evaluator().stringValue( + item, StringConstants::nameProperty()); + if (projectContext->project->name.isEmpty()) + projectContext->project->name = FileInfo::baseName(item->location().filePath()); // FIXME: Must also be changed in item? + QVariantMap projectProperties; + if (!projectContext->project->enabled) { + projectProperties.insert(StringConstants::profileProperty(), + state.evaluator().stringValue( + item, StringConstants::profileProperty())); + projectContext->project->setProjectProperties(projectProperties); + return; + } + + projectContext->dummyModule = ResolvedModule::create(); + + for (Item::PropertyDeclarationMap::const_iterator it + = item->propertyDeclarations().constBegin(); + it != item->propertyDeclarations().constEnd(); ++it) { + if (it.value().flags().testFlag(PropertyDeclaration::PropertyNotAvailableInConfig)) + continue; + const ValueConstPtr v = item->property(it.key()); + QBS_ASSERT(v && v->type() != Value::ItemValueType, continue); + const ScopedJsValue sv(engine->context(), state.evaluator().value(item, it.key())); + projectProperties.insert(it.key(), getJsVariant(engine->context(), sv)); + } + projectContext->project->setProjectProperties(projectProperties); + + for (Item * const child : item->children()) { + state.topLevelProject().checkCancelation(); + try { + switch (child->type()) { + case ItemType::SubProject: + resolveSubProject(child, projectContext); + break; + case ItemType::FileTagger: + resolveFileTagger(state, child, projectContext, nullptr); + break; + case ItemType::JobLimit: + resolveJobLimit(state, child, projectContext, nullptr, nullptr); + break; + case ItemType::Rule: + resolveRule(state, child, projectContext, nullptr, nullptr); + break; + default: + break; + } + } catch (const ErrorInfo &e) { + state.topLevelProject().addQueuedError(e); + } + } + + for (ProjectContext * const childContext : projectContext->children) + resolveProject(childContext); +} + +void ProjectResolver::Private::resolveSubProject(Item *item, ProjectContext *projectContext) +{ + // If we added a Project child item in ProductsCollector, then the sub-project will be + // resolved normally via resolveProject(). Otherwise, it was not loaded, for instance + // because its Properties condition was false, and special handling applies. + if (item->child(ItemType::Project)) + return; + + ResolvedProjectPtr project = ResolvedProject::create(); + project->enabled = false; + project->parentProject = projectContext->project; + projectContext->project->subProjects << project; + if (Item * const propertiesItem = item->child(ItemType::PropertiesInSubProject)) { + project->name = state.evaluator().stringValue(propertiesItem, + StringConstants::nameProperty()); + } +} + +void ProjectResolver::Private::checkOverriddenValues() +{ + static const auto matchesPrefix = [](const QString &key) { + static const QStringList prefixes({StringConstants::projectPrefix(), + QStringLiteral("projects"), + QStringLiteral("products"), QStringLiteral("modules"), + StringConstants::moduleProviders(), + StringConstants::qbsModule()}); + return any_of(prefixes, [&key](const QString &prefix) { + return key.startsWith(prefix + QLatin1Char('.')); }); + }; + const QVariantMap &overriddenValues = state.parameters().overriddenValues(); + for (auto it = overriddenValues.begin(); it != overriddenValues.end(); ++it) { + if (matchesPrefix(it.key())) { + collectNameFromOverride(it.key()); + continue; + } + + ErrorInfo e(Tr::tr("Property override key '%1' not understood.").arg(it.key())); + e.append(Tr::tr("Please use one of the following:")); + e.append(QLatin1Char('\t') + Tr::tr("projects.<project-name>.<property-name>:value")); + e.append(QLatin1Char('\t') + Tr::tr("products.<product-name>.<property-name>:value")); + e.append(QLatin1Char('\t') + Tr::tr("modules.<module-name>.<property-name>:value")); + e.append(QLatin1Char('\t') + Tr::tr("products.<product-name>.<module-name>." + "<property-name>:value")); + e.append(QLatin1Char('\t') + Tr::tr("moduleProviders.<provider-name>." + "<property-name>:value")); + handlePropertyError(e, state.parameters(), state.logger()); + } +} + +void ProjectResolver::Private::collectNameFromOverride(const QString &overrideString) +{ + const auto extract = [&overrideString](const QString &prefix) { + if (!overrideString.startsWith(prefix)) + return QString(); + const int startPos = prefix.length(); + const int endPos = overrideString.lastIndexOf(StringConstants::dot()); + if (endPos == -1) + return QString(); + return overrideString.mid(startPos, endPos - startPos); + }; + const QString &projectName = extract(StringConstants::projectsOverridePrefix()); + if (!projectName.isEmpty()) { + state.topLevelProject().addProjectNameUsedInOverrides(projectName); + return; + } + const QString &productName = extract(StringConstants::productsOverridePrefix()); + if (!productName.isEmpty()) { + state.topLevelProject().addProductNameUsedInOverrides(productName.left( + productName.indexOf(StringConstants::dot()))); + return; + } +} + +void ProjectResolver::Private::loadTopLevelProjectItem() +{ + const QStringList topLevelSearchPaths + = state.parameters().finalBuildConfigurationTree() + .value(StringConstants::projectPrefix()).toMap() + .value(StringConstants::qbsSearchPathsProperty()).toStringList(); + SearchPathsManager searchPathsManager(state.itemReader(), topLevelSearchPaths); + Item * const root = state.itemReader().setupItemFromFile( + state.parameters().projectFilePath(), {}); + if (!root) + return; + + switch (root->type()) { + case ItemType::Product: + rootProjectItem = state.itemReader().wrapInProjectIfNecessary(root); + break; + case ItemType::Project: + rootProjectItem = root; + break; + default: + throw ErrorInfo(Tr::tr("The top-level item must be of type 'Project' or 'Product', but it" + " is of type '%1'.").arg(root->typeName()), root->location()); + } +} + +void ProjectResolver::Private::buildProjectTree() +{ + state.topLevelProject().setBuildDirectory(TopLevelProject::deriveBuildDirectory( + state.parameters().buildRoot(), + TopLevelProject::deriveId(state.parameters().finalBuildConfigurationTree()))); + rootProjectItem->setProperty(StringConstants::sourceDirectoryProperty(), + VariantValue::create(QFileInfo(rootProjectItem->file()->filePath()) + .absolutePath())); + rootProjectItem->setProperty(StringConstants::buildDirectoryProperty(), + VariantValue::create(state.topLevelProject().buildDirectory())); + rootProjectItem->setProperty(StringConstants::profileProperty(), + VariantValue::create(state.parameters().topLevelProfile())); + ProductsCollector(state).run(rootProjectItem); + + AccumulatingTimer timer(state.parameters().logElapsedTime() + ? &state.topLevelProject().timingData().propertyChecking : nullptr); + checkPropertyDeclarations(rootProjectItem, state); +} + +void ProjectResolver::Private::printProfilingInfo() +{ + if (!setupParams.logElapsedTime()) + return; + const auto print = [this](int indent, const QString &pattern, qint64 time) { + logger.qbsLog(LoggerInfo, true) << QByteArray(indent, ' ') + << pattern.arg(elapsedTimeString(time)); + }; + print(2, Tr::tr("Project file loading and parsing took %1."), state.itemReader().elapsedTime()); + print(2, Tr::tr("Preparing products took %1."), + state.topLevelProject().timingData().preparingProducts); + print(2, Tr::tr("Setting up Groups took %1."), + state.topLevelProject().timingData().groupsSetup); + print(2, Tr::tr("Scheduling products took %1."), + state.topLevelProject().timingData().schedulingProducts); + print(2, Tr::tr("Resolving products took %1."), + state.topLevelProject().timingData().resolvingProducts); + print(4, Tr::tr("Property evaluation took %1."), + state.topLevelProject().timingData().propertyEvaluation); + print(4, Tr::tr("Resolving groups (without module property evaluation) took %1."), + state.topLevelProject().timingData().groupsResolving); + print(4, Tr::tr("Setting up product dependencies took %1."), + state.topLevelProject().timingData().dependenciesResolving); + print(6, Tr::tr("Running module providers took %1."), + state.topLevelProject().timingData().moduleProviders); + print(6, Tr::tr("Instantiating modules took %1."), + state.topLevelProject().timingData().moduleInstantiation); + print(6, Tr::tr("Merging module property values took %1."), + state.topLevelProject().timingData().propertyMerging); + logger.qbsLog(LoggerInfo, true) << QByteArray(4, ' ') << "There were " + << state.topLevelProject().productDeferrals() + << " product deferrals with a total of " + << state.topLevelProject().productCount() << " products."; + print(2, Tr::tr("Running Probes took %1."), state.topLevelProject().timingData().probes); + state.logger().qbsLog(LoggerInfo, true) + << " " + << Tr::tr("%1 probes encountered, %2 configure scripts executed, " + "%3 re-used from current run, %4 re-used from earlier run.") + .arg(state.topLevelProject().probesEncounteredCount()) + .arg(state.topLevelProject().probesRunCount()) + .arg(state.topLevelProject().reusedCurrentProbesCount()) + .arg(state.topLevelProject().reusedOldProbesCount()); + print(2, Tr::tr("Property checking took %1."), + state.topLevelProject().timingData().propertyChecking); +} + +} // namespace Internal +} // namespace qbs diff --git a/src/lib/corelib/loader/projectresolver.h b/src/lib/corelib/loader/projectresolver.h new file mode 100644 index 000000000..22ad82518 --- /dev/null +++ b/src/lib/corelib/loader/projectresolver.h @@ -0,0 +1,84 @@ +/**************************************************************************** +** +** Copyright (C) 2016 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef PROJECTRESOLVER_H +#define PROJECTRESOLVER_H + +#include <language/forward_decls.h> +#include <logging/logger.h> +#include <tools/pimpl.h> +#include <tools/qbs_export.h> + +#include <QHash> +#include <QVariant> + +#include <vector> + +namespace qbs { +class SetupProjectParameters; +namespace Internal { +class FileTime; +class Logger; +class ProgressObserver; +class ScriptEngine; +class StoredModuleProviderInfo; + +class QBS_AUTOTEST_EXPORT ProjectResolver +{ +public: + ProjectResolver(const SetupProjectParameters ¶meters, ScriptEngine *engine, Logger logger); + ~ProjectResolver(); + + void setProgressObserver(ProgressObserver *observer); + void setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes); + void setOldProductProbes(const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes); + void setLastResolveTime(const FileTime &time); + void setStoredProfiles(const QVariantMap &profiles); + void setStoredModuleProviderInfo(const StoredModuleProviderInfo &providerInfo); + TopLevelProjectPtr resolve(); + +private: + class Private; + Pimpl<Private> d; +}; + +} // namespace Internal +} // namespace qbs + +#endif // PROJECTRESOLVER_H |