diff options
138 files changed, 5821 insertions, 4240 deletions
diff --git a/.github/actions/download-qt/action.yml b/.github/actions/download-qt/action.yml index b6785654b..ff1d31d60 100644 --- a/.github/actions/download-qt/action.yml +++ b/.github/actions/download-qt/action.yml @@ -4,7 +4,7 @@ inputs: version: description: 'Qt version' required: false - default: '6.2.4' + default: '6.5.0' target: description: 'Qt target (desktop, ios, android)' required: false @@ -1 +1 @@ -2.1.2 +2.2.0 diff --git a/doc/appendix/json-api.qdoc b/doc/appendix/json-api.qdoc index b555825f9..28a8a943c 100644 --- a/doc/appendix/json-api.qdoc +++ b/doc/appendix/json-api.qdoc @@ -125,6 +125,7 @@ \row \li force-probe-execution \li bool \li no \row \li log-time \li bool \li no \row \li log-level \li \l LogLevel \li no + \row \li max-job-count \li int \li no \row \li module-properties \li list of strings \li no \row \li overridden-properties \li object \li no \row \li project-file-path \li FilePath \li if resolving from scratch diff --git a/doc/qbs.qdoc b/doc/qbs.qdoc index c77dae824..9ea88763e 100644 --- a/doc/qbs.qdoc +++ b/doc/qbs.qdoc @@ -462,7 +462,7 @@ To build \QBS from the source, you need: \list - \li Qt 5.14, or later + \li Qt 5.15, or later \li Windows: MinGW with GCC 4.9 or Microsoft Visual Studio 2015, or later \li Linux: GCC 4.9, or later, or Clang 3.9.0, or later diff --git a/doc/reference/items/language/moduleprovider.qdoc b/doc/reference/items/language/moduleprovider.qdoc index 81a09a66b..09cabb39b 100644 --- a/doc/reference/items/language/moduleprovider.qdoc +++ b/doc/reference/items/language/moduleprovider.qdoc @@ -67,6 +67,19 @@ */ /*! + \qmlproperty bool ModuleProvider::isEager + + Holds whether provider is eager. + + Eager provider is executed only once and should create multiple modules at once when executed). + A non-eager provider is executed multiple times, one time for each module \QBS tries to + instantiate. + + \sa ModuleProvider::moduleName + \default true +*/ + +/*! \qmlproperty string ModuleProvider::name The name of the module provider. @@ -83,6 +96,17 @@ */ /*! + \qmlproperty string ModuleProvider::moduleName + + This property is set by QBS for non-eager providers and contains the name of the module + that is currently being instantiated by the provider. + + For eager providers, the value of this property is \c undefined. + + \sa ModuleProvider::isEager +*/ + +/*! \qmlproperty string ModuleProvider::outputBaseDir The path under which the new modules should be created when \l relativeSearchPaths diff --git a/doc/reference/module-providers/qbspkgconfig-module-provider.qdoc b/doc/reference/module-providers/qbspkgconfig-module-provider.qdoc index 957e19021..461536fbc 100644 --- a/doc/reference/module-providers/qbspkgconfig-module-provider.qdoc +++ b/doc/reference/module-providers/qbspkgconfig-module-provider.qdoc @@ -111,6 +111,7 @@ /*! \qmlproperty bool qbspkgconfig::mergeDependencies + \obsolete Holds whether dependencies should be merged by pkg-config or \QBS. @@ -119,4 +120,6 @@ may depend on other modules and property merging is done by \QBS. \defaultvalue \c false + + Deprecated in \QBS 2.2.0. */ diff --git a/doc/reference/modules/qt-core-module.qdoc b/doc/reference/modules/qt-core-module.qdoc index 2211c4be6..6aedf2a69 100644 --- a/doc/reference/modules/qt-core-module.qdoc +++ b/doc/reference/modules/qt-core-module.qdoc @@ -510,3 +510,11 @@ \defaultvalue \c{versionParts[2]} */ + +/*! + \qmlproperty bool Qt.core::useRPaths + + Whether to add \l{Qt.core::libPath}{Qt.core.libPath} to \l{cpp::rpaths}{cpp.rpaths}. + + \defaultvalue \c true on Linux, \c false everywhere else. +*/ diff --git a/docker-compose.yml b/docker-compose.yml index 842c2f8de..cc0391faf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -167,12 +167,12 @@ services: QTCREATOR_VERSION: 5.0.3 windows: - image: ${DOCKER_USER:-qbsbuild}/qbsdev:windowsservercore-6.2.4_1.24.0-0 + image: ${DOCKER_USER:-qbsbuild}/qbsdev:windowsservercore-6.5.0_1.24.0-0 build: dockerfile: docker/windowsservercore/Dockerfile context: . args: - QT_VERSION: 6.2.4 + QT_VERSION: 6.5.0 QTCREATOR_VERSION: 9.0.1 working_dir: 'C:/qbs' environment: diff --git a/qbs-resources/imports/QbsProduct.qbs b/qbs-resources/imports/QbsProduct.qbs index 5df694dd8..ea8516736 100644 --- a/qbs-resources/imports/QbsProduct.qbs +++ b/qbs-resources/imports/QbsProduct.qbs @@ -2,7 +2,7 @@ Product { Depends { name: "qbsbuildconfig" } Depends { name: "qbsversion" } Depends { name: "Qt.core"; versionAtLeast: minimumQtVersion } - property string minimumQtVersion: "5.14.0" + property string minimumQtVersion: "5.15.2" property bool install: true property string targetInstallDir cpp.defines: { diff --git a/share/qbs/module-providers/Qt/templates/core.qbs b/share/qbs/module-providers/Qt/templates/core.qbs index ca978eae5..214bd65c2 100644 --- a/share/qbs/module-providers/Qt/templates/core.qbs +++ b/share/qbs/module-providers/Qt/templates/core.qbs @@ -109,6 +109,7 @@ Module { property string libFilePathRelease: @libFilePathRelease@ property string libFilePath: qtBuildVariant === "debug" ? libFilePathDebug : libFilePathRelease + property bool useRPaths: qbs.targetOS.contains("linux") && !qbs.targetOS.contains("android") property stringList coreLibPaths: @libraryPaths@ property bool hasLibrary: true @@ -198,8 +199,7 @@ Module { return undefined; return frameworks; } - cpp.rpaths: qbs.targetOS.contains('linux') && !qbs.targetOS.contains("android") ? [libPath] : - undefined + cpp.rpaths: useRPaths ? libPath : undefined cpp.runtimeLibrary: qbs.toolchain.contains("msvc") ? config.contains("static_runtime") ? "static" : "dynamic" : original diff --git a/share/qbs/module-providers/qbspkgconfig.qbs b/share/qbs/module-providers/qbspkgconfig.qbs index 2736220a8..4eda7cb13 100644 --- a/share/qbs/module-providers/qbspkgconfig.qbs +++ b/share/qbs/module-providers/qbspkgconfig.qbs @@ -58,8 +58,11 @@ ModuleProvider { property path sysroot: qbs.toolchain && qbs.toolchain.includes("xcode") ? undefined : qbs.sysroot - // TODO: deprecate in 2.2, remove in 2.3 property bool mergeDependencies: false + PropertyOptions { + name: "mergeDependencies" + removalVersion: "2.3.0" + } relativeSearchPaths: { diff --git a/share/qbs/modules/bundle/bundle.js b/share/qbs/modules/bundle/bundle.js index da23b7313..da9e2486a 100644 --- a/share/qbs/modules/bundle/bundle.js +++ b/share/qbs/modules/bundle/bundle.js @@ -313,7 +313,8 @@ var XcodeBuildSpecsReader = (function () { }; XcodeBuildSpecsReader.prototype.expandedSetting = function (typeIdentifier, baseSettings, settingName) { - var obj = baseSettings || {}; + var obj = {}; + _assign(obj, baseSettings); // todo: copy recursively obj = _assign(obj, this.settings(typeIdentifier, true)); if (obj) { for (var x in this._additionalSettings) { diff --git a/share/qbs/modules/capnproto/cpp/capnprotocpp.qbs b/share/qbs/modules/capnproto/cpp/capnprotocpp.qbs index f33bc9a48..bccfca192 100644 --- a/share/qbs/modules/capnproto/cpp/capnprotocpp.qbs +++ b/share/qbs/modules/capnproto/cpp/capnprotocpp.qbs @@ -39,6 +39,7 @@ CapnProtoBase { Depends { name: "capnp-rpc"; condition: useRpc } pluginName: "capnpc-c++" + version: capnp.version cpp.systemIncludePaths: outputDir cpp.cxxLanguageVersion: "c++14" diff --git a/share/qbs/modules/protobuf/cpp/protobufcpp.qbs b/share/qbs/modules/protobuf/cpp/protobufcpp.qbs index af1a5d2cd..e9505dfc4 100644 --- a/share/qbs/modules/protobuf/cpp/protobufcpp.qbs +++ b/share/qbs/modules/protobuf/cpp/protobufcpp.qbs @@ -18,7 +18,7 @@ ProtobufBase { property string grpcIncludePath: grpcIncludeProbe.found ? grpcIncludeProbe.path : undefined property string grpcLibraryPath: grpcLibraryProbe.found ? grpcLibraryProbe.path : undefined - property string _cxxLanguageVersion: qbs.targetOS.contains("darwin") ? "c++17" : "c++14" + property string _cxxLanguageVersion: "c++17" readonly property string _libraryName: { var libraryName; diff --git a/src/app/qbs-create-project/create-project-main.cpp b/src/app/qbs-create-project/create-project-main.cpp index bb5d1a6bc..51e7ce514 100644 --- a/src/app/qbs-create-project/create-project-main.cpp +++ b/src/app/qbs-create-project/create-project-main.cpp @@ -80,9 +80,9 @@ int main(int argc, char *argv[]) const ProjectStructure projectStructure = parser.isSet(flatOpt) ? ProjectStructure::Flat : ProjectStructure::Composite; const QStringList whiteList = parser.value(whiteListOpt).split(QLatin1Char(','), - QBS_SKIP_EMPTY_PARTS); + Qt::SkipEmptyParts); const QStringList blackList = parser.value(blackListOpt).split(QLatin1Char(','), - QBS_SKIP_EMPTY_PARTS); + Qt::SkipEmptyParts); try { ProjectCreator().run(QDir::currentPath(), projectStructure, whiteList, blackList); } catch (const ErrorInfo &e) { diff --git a/src/app/qbs-setup-qt/main.cpp b/src/app/qbs-setup-qt/main.cpp index bef95eee2..b739bff2d 100644 --- a/src/app/qbs-setup-qt/main.cpp +++ b/src/app/qbs-setup-qt/main.cpp @@ -82,7 +82,7 @@ int main(int argc, char *argv[]) QString profileName = QLatin1String("qt-") + qtEnvironment.qtVersion.toString(); if (SetupQt::checkIfMoreThanOneQtWithTheSameVersion(qtEnvironment.qtVersion, qtEnvironments)) { QStringList prefixPathParts = QFileInfo(qtEnvironment.qmakeFilePath).path() - .split(QLatin1Char('/'), QBS_SKIP_EMPTY_PARTS); + .split(QLatin1Char('/'), Qt::SkipEmptyParts); if (!prefixPathParts.empty()) profileName += QLatin1String("-") + prefixPathParts.last(); } diff --git a/src/app/qbs-setup-toolchains/xcodeprobe.cpp b/src/app/qbs-setup-toolchains/xcodeprobe.cpp index 9be12d3fc..ea27e4143 100644 --- a/src/app/qbs-setup-toolchains/xcodeprobe.cpp +++ b/src/app/qbs-setup-toolchains/xcodeprobe.cpp @@ -163,7 +163,7 @@ void XcodeProbe::detectDeveloperPaths() qbsInfo() << Tr::tr("Could not detect additional Xcode installations with /usr/bin/mdfind"); } else { const auto paths = QString::fromLocal8Bit(launchServices.readAllStandardOutput()) - .split(QLatin1Char('\n'), QBS_SKIP_EMPTY_PARTS); + .split(QLatin1Char('\n'), Qt::SkipEmptyParts); for (const QString &path : paths) addDeveloperPath(path + QStringLiteral("/Contents/Developer")); } diff --git a/src/app/qbs/commandlinefrontend.cpp b/src/app/qbs/commandlinefrontend.cpp index 3ed5cd1ad..701a49046 100644 --- a/src/app/qbs/commandlinefrontend.cpp +++ b/src/app/qbs/commandlinefrontend.cpp @@ -184,6 +184,7 @@ void CommandLineFrontend::start() params.setConfigurationName(configurationName); params.setBuildRoot(buildDirectory(profileName)); params.setOverriddenValues(userConfig); + params.setMaxJobCount(m_parser.jobCount(profileName)); SetupProjectJob * const job = Project().setupProject(params, ConsoleLogger::instance().logSink(), this); connectJob(job); diff --git a/src/app/qbs/parser/commandlineparser.cpp b/src/app/qbs/parser/commandlineparser.cpp index 14e26ca42..7fbfa28f1 100644 --- a/src/app/qbs/parser/commandlineparser.cpp +++ b/src/app/qbs/parser/commandlineparser.cpp @@ -65,6 +65,7 @@ #include <QtCore/qmap.h> #include <QtCore/qtextstream.h> +#include <algorithm> #include <utility> #ifdef Q_OS_UNIX @@ -155,17 +156,11 @@ QString CommandLineParser::projectBuildDirectory() const BuildOptions CommandLineParser::buildOptions(const QString &profile) const { - Settings settings(settingsDir()); - Preferences preferences(&settings, profile); - - if (d->buildOptions.maxJobCount() <= 0) { - d->buildOptions.setMaxJobCount(preferences.jobs()); - } - + d->buildOptions.setMaxJobCount(jobCount(profile)); if (d->buildOptions.echoMode() < 0) { - d->buildOptions.setEchoMode(preferences.defaultEchoMode()); + Settings settings(settingsDir()); + d->buildOptions.setEchoMode(Preferences(&settings, profile).defaultEchoMode()); } - return d->buildOptions; } @@ -202,6 +197,15 @@ InstallOptions CommandLineParser::installOptions(const QString &profile) const return options; } +int CommandLineParser::jobCount(const QString &profile) const +{ + if (const int explicitJobCount = d->optionPool.jobsOption()->jobCount(); explicitJobCount > 0) + return explicitJobCount; + + Settings settings(settingsDir()); + return Preferences(&settings, profile).jobs(); +} + bool CommandLineParser::forceTimestampCheck() const { return d->optionPool.forceTimestampCheckOption()->enabled(); @@ -334,7 +338,19 @@ void CommandLineParser::CommandLineParserPrivate::doParse() } else { command = commandFromString(commandLine.front()); if (command) { - commandLine.removeFirst(); + const QString commandName = commandLine.takeFirst(); + + // if the command line contains a `<command>` with + // either `-h` or `--help` switch, we transform + // it to corresponding `help <command>` instead + const QStringList helpSwitches = {QStringLiteral("-h"), QStringLiteral("--help")}; + if (auto it = std::find_first_of( + commandLine.begin(), commandLine.end(), + helpSwitches.begin(), helpSwitches.end()); + it != commandLine.end()) { + command = commandPool.getCommand(HelpCommandType); + commandLine = QList{commandName}; // keep only command's name + } } else { // No command given. if (commandLine.front() == QLatin1String("-h") || commandLine.front() == QLatin1String("--help")) { diff --git a/src/app/qbs/parser/commandlineparser.h b/src/app/qbs/parser/commandlineparser.h index 999027006..4ce7756ef 100644 --- a/src/app/qbs/parser/commandlineparser.h +++ b/src/app/qbs/parser/commandlineparser.h @@ -75,6 +75,7 @@ public: CleanOptions cleanOptions(const QString &profile) const; GenerateOptions generateOptions() const; InstallOptions installOptions(const QString &profile) const; + int jobCount(const QString &profile) const; bool forceTimestampCheck() const; bool forceOutputCheck() const; bool dryRun() const; diff --git a/src/app/qbs/parser/parsercommand.cpp b/src/app/qbs/parser/parsercommand.cpp index bbb5db3d1..f11a7b6ac 100644 --- a/src/app/qbs/parser/parsercommand.cpp +++ b/src/app/qbs/parser/parsercommand.cpp @@ -209,6 +209,7 @@ static QList<CommandLineOption::Type> resolveOptions() CommandLineOption::ForceProbesOptionType, CommandLineOption::LogTimeOptionType, CommandLineOption::DeprecationWarningsOptionType, + CommandLineOption::JobsOptionType, CommandLineOption::DisableFallbackProviderType}; } @@ -279,7 +280,6 @@ static QList<CommandLineOption::Type> buildOptions() << CommandLineOption::ForceTimestampCheckOptionType << CommandLineOption::ForceOutputCheckOptionType << CommandLineOption::BuildNonDefaultOptionType - << CommandLineOption::JobsOptionType << CommandLineOption::CommandEchoModeOptionType << CommandLineOption::NoInstallOptionType << CommandLineOption::RemoveFirstOptionType diff --git a/src/app/qbs/sessionpacket.cpp b/src/app/qbs/sessionpacket.cpp index 3830704fa..43dd22aba 100644 --- a/src/app/qbs/sessionpacket.cpp +++ b/src/app/qbs/sessionpacket.cpp @@ -99,7 +99,7 @@ QJsonObject SessionPacket::helloMessage() { return QJsonObject{ {StringConstants::type(), QLatin1String("hello")}, - {QLatin1String("api-level"), 3}, + {QLatin1String("api-level"), 4}, {QLatin1String("api-compat-level"), 2} }; } diff --git a/src/app/qbs/stdinreader.cpp b/src/app/qbs/stdinreader.cpp index 5f00d7de4..4708ff53c 100644 --- a/src/app/qbs/stdinreader.cpp +++ b/src/app/qbs/stdinreader.cpp @@ -43,6 +43,7 @@ #include <QtCore/qfile.h> #include <QtCore/qsocketnotifier.h> +#include <QtCore/qthread.h> #include <QtCore/qtimer.h> #include <cerrno> @@ -111,46 +112,183 @@ public: WindowsStdinReader(QObject *parent) : StdinReader(parent) {} private: - void start() override - { #ifdef Q_OS_WIN32 - m_stdinHandle = GetStdHandle(STD_INPUT_HANDLE); - if (!m_stdinHandle) { - emit errorOccurred(tr("Failed to create handle for standard input.")); - return; + class FileReaderThread : public QThread + { + public: + FileReaderThread(WindowsStdinReader &parent, HANDLE stdInHandle, HANDLE exitEventHandle) + : QThread(&parent), m_stdIn{stdInHandle}, m_exitEvent{exitEventHandle} { } + ~FileReaderThread() + { + wait(); + CloseHandle(m_exitEvent); + } + + void run() override + { + WindowsStdinReader *r = static_cast<WindowsStdinReader *>(parent()); + + char buf[1024]; + while (true) { + DWORD bytesRead = 0; + if (!ReadFile(m_stdIn, buf, sizeof buf, &bytesRead, nullptr)) { + emit r->errorOccurred(tr("Failed to read from input channel.")); + break; + } + if (!bytesRead) + break; + emit r->dataAvailable(QByteArray(buf, bytesRead)); + } + } + private: + HANDLE m_stdIn; + HANDLE m_exitEvent; + }; + + class ConsoleReaderThread : public QThread + { + public: + ConsoleReaderThread(WindowsStdinReader &parent, HANDLE stdInHandle, HANDLE exitEventHandle) + : QThread(&parent), m_stdIn{stdInHandle}, m_exitEvent{exitEventHandle} { } + virtual ~ConsoleReaderThread() override + { + SetEvent(m_exitEvent); + wait(); + CloseHandle(m_exitEvent); } - // A timer seems slightly less awful than to block in a thread - // (how would we abort that one?), but ideally we'd like - // to have a signal-based approach like in the Unix variant. - const auto timer = new QTimer(this); - connect(timer, &QTimer::timeout, this, [this, timer] { + void run() override + { + WindowsStdinReader *r = static_cast<WindowsStdinReader *>(parent()); + + DWORD origConsoleMode; + GetConsoleMode(m_stdIn, &origConsoleMode); + DWORD consoleMode = ENABLE_PROCESSED_INPUT; + SetConsoleMode(m_stdIn, consoleMode); + + HANDLE handles[2] = {m_exitEvent, m_stdIn}; char buf[1024]; - DWORD bytesAvail; - if (!PeekNamedPipe(m_stdinHandle, nullptr, 0, nullptr, &bytesAvail, nullptr)) { - timer->stop(); - emit errorOccurred(tr("Failed to read from input channel.")); + while (true) { + auto result = WaitForMultipleObjects(2, handles, FALSE, INFINITE); + if (result == WAIT_OBJECT_0) + break; + INPUT_RECORD consoleInput; + DWORD inputsRead = 0; + if (!PeekConsoleInputA(m_stdIn, &consoleInput, 1, &inputsRead)) { + emit r->errorOccurred(tr("Failed to read from input channel.")); + break; + } + if (inputsRead) { + if (consoleInput.EventType != KEY_EVENT + || !consoleInput.Event.KeyEvent.bKeyDown + || !consoleInput.Event.KeyEvent.uChar.AsciiChar) { + if (!ReadConsoleInputA(m_stdIn, &consoleInput, 1, &inputsRead)) { + emit r->errorOccurred(tr("Failed to read console input.")); + break; + } + } else { + DWORD bytesRead = 0; + if (!ReadConsoleA(m_stdIn, buf, sizeof buf, &bytesRead, nullptr)) { + emit r->errorOccurred(tr("Failed to read console.")); + break; + } + emit r->dataAvailable(QByteArray(buf, bytesRead)); + } + } + } + SetConsoleMode(m_stdIn, origConsoleMode); + } + private: + HANDLE m_stdIn; + HANDLE m_exitEvent; + }; + + class PipeReaderThread : public QThread + { + public: + PipeReaderThread(WindowsStdinReader &parent, HANDLE stdInHandle, HANDLE exitEventHandle) + : QThread(&parent), m_stdIn{stdInHandle}, m_exitEvent{exitEventHandle} { } + virtual ~PipeReaderThread() override + { + SetEvent(m_exitEvent); + wait(); + CloseHandle(m_exitEvent); + } + + void run() override + { + WindowsStdinReader *r = static_cast<WindowsStdinReader *>(parent()); + + OVERLAPPED overlapped = {}; + overlapped.hEvent = CreateEventA(NULL, TRUE, TRUE, NULL); + if (!overlapped.hEvent) { + emit r->errorOccurred(StdinReader::tr("Failed to create handle for overlapped event.")); return; } - while (bytesAvail > 0) { - DWORD bytesRead; - if (!ReadFile(m_stdinHandle, buf, std::min<DWORD>(bytesAvail, sizeof buf), - &bytesRead, nullptr)) { - timer->stop(); - emit errorOccurred(tr("Failed to read from input channel.")); - return; + + char buf[1024]; + DWORD bytesRead; + HANDLE handles[2] = {m_exitEvent, overlapped.hEvent}; + while (true) { + bytesRead = 0; + auto readResult = ReadFile(m_stdIn, buf, sizeof buf, NULL, &overlapped); + if (!readResult) { + if (GetLastError() != ERROR_IO_PENDING) { + emit r->errorOccurred(StdinReader::tr("ReadFile Failed.")); + break; + } + + auto result = WaitForMultipleObjects(2, handles, FALSE, INFINITE); + if (result == WAIT_OBJECT_0) + break; + } + if (!GetOverlappedResult(m_stdIn, &overlapped, &bytesRead, FALSE)) { + if (GetLastError() != ERROR_HANDLE_EOF) + emit r->errorOccurred(StdinReader::tr("Error GetOverlappedResult.")); + break; } - emit dataAvailable(QByteArray(buf, bytesRead)); - bytesAvail -= bytesRead; + emit r->dataAvailable(QByteArray(buf, bytesRead)); } - }); - timer->start(10); + CancelIo(m_stdIn); + CloseHandle(overlapped.hEvent); + } + private: + HANDLE m_stdIn; + HANDLE m_exitEvent; + }; #endif - } + void start() override + { #ifdef Q_OS_WIN32 - HANDLE m_stdinHandle; + HANDLE stdInHandle = GetStdHandle(STD_INPUT_HANDLE); + if (!stdInHandle) { + emit errorOccurred(StdinReader::tr("Failed to create handle for standard input.")); + return; + } + HANDLE exitEventHandle = CreateEventA(NULL, TRUE, FALSE, NULL); + if (!exitEventHandle) { + emit errorOccurred(StdinReader::tr("Failed to create handle for exit event.")); + return; + } + + auto result = GetFileType(stdInHandle); + switch (result) { + case FILE_TYPE_CHAR: + (new ConsoleReaderThread(*this, stdInHandle, exitEventHandle))->start(); + return; + case FILE_TYPE_PIPE: + (new PipeReaderThread(*this, stdInHandle, exitEventHandle))->start(); + return; + case FILE_TYPE_DISK: + (new FileReaderThread(*this, stdInHandle, exitEventHandle))->start(); + return; + default: + emit errorOccurred(StdinReader::tr("Unable to handle unknown input type")); + return; + } #endif + } }; StdinReader *StdinReader::create(QObject *parent) diff --git a/src/lib/corelib/CMakeLists.txt b/src/lib/corelib/CMakeLists.txt index 4eac9eeab..900d90d99 100644 --- a/src/lib/corelib/CMakeLists.txt +++ b/src/lib/corelib/CMakeLists.txt @@ -266,10 +266,12 @@ set(LOADER_SOURCES probesresolver.h productitemmultiplexer.cpp productitemmultiplexer.h + productresolver.cpp + productresolver.h productscollector.cpp productscollector.h - productshandler.cpp - productshandler.h + productsresolver.cpp + productsresolver.h projectresolver.cpp projectresolver.h ) diff --git a/src/lib/corelib/api/internaljobs.cpp b/src/lib/corelib/api/internaljobs.cpp index 46ee20bbf..35766efa4 100644 --- a/src/lib/corelib/api/internaljobs.cpp +++ b/src/lib/corelib/api/internaljobs.cpp @@ -77,8 +77,8 @@ public: { std::lock_guard<std::mutex> lock(m_cancelMutex); m_canceled = true; - if (scriptEngine()) - scriptEngine()->cancel(); + for (ScriptEngine * const engine : scriptEngines()) + engine->cancel(); } private: diff --git a/src/lib/corelib/api/projectdata.cpp b/src/lib/corelib/api/projectdata.cpp index 9de2d00ee..11469ee18 100644 --- a/src/lib/corelib/api/projectdata.cpp +++ b/src/lib/corelib/api/projectdata.cpp @@ -42,7 +42,7 @@ #include "propertymap_p.h" #include <language/language.h> #include <language/propertymapinternal.h> -#include <loader/productitemmultiplexer.h> +#include <loader/loaderutils.h> #include <tools/fileinfo.h> #include <tools/jsliterals.h> #include <tools/qbsassert.h> @@ -559,7 +559,7 @@ const QString &ProductData::name() const */ QString ProductData::fullDisplayName() const { - return ProductItemMultiplexer::fullProductDisplayName(name(), multiplexConfigurationId()); + return fullProductDisplayName(name(), multiplexConfigurationId()); } /*! diff --git a/src/lib/corelib/api/projectfileupdater.cpp b/src/lib/corelib/api/projectfileupdater.cpp index 5b9f9bced..604600f8e 100644 --- a/src/lib/corelib/api/projectfileupdater.cpp +++ b/src/lib/corelib/api/projectfileupdater.cpp @@ -210,11 +210,7 @@ ProjectFileGroupInserter::ProjectFileGroupInserter(ProductData product, QString static int extractLine(const QString &fileContent, int pos) { -#if QT_VERSION < QT_VERSION_CHECK(5, 15, 2) - return fileContent.leftRef(pos).count(QLatin1Char('\n')); -#else return QStringView{fileContent}.left(pos).count(QLatin1Char('\n')); -#endif } void ProjectFileGroupInserter::doApply(QString &fileContent, UiProgram *ast) diff --git a/src/lib/corelib/api/runenvironment.cpp b/src/lib/corelib/api/runenvironment.cpp index 2544f549e..adf0c4557 100644 --- a/src/lib/corelib/api/runenvironment.cpp +++ b/src/lib/corelib/api/runenvironment.cpp @@ -225,7 +225,7 @@ int RunEnvironment::doRunShell() static QString findExecutable(const QStringList &fileNames) { const QStringList path = QString::fromLocal8Bit(qgetenv("PATH")) - .split(HostOsInfo::pathListSeparator(), QBS_SKIP_EMPTY_PARTS); + .split(HostOsInfo::pathListSeparator(), Qt::SkipEmptyParts); for (const QString &fileName : fileNames) { const QString exeFileName = HostOsInfo::appendExecutableSuffix(fileName); diff --git a/src/lib/corelib/buildgraph/processcommandexecutor.cpp b/src/lib/corelib/buildgraph/processcommandexecutor.cpp index b308e9c47..52a8fe75d 100644 --- a/src/lib/corelib/buildgraph/processcommandexecutor.cpp +++ b/src/lib/corelib/buildgraph/processcommandexecutor.cpp @@ -291,7 +291,7 @@ void ProcessCommandExecutor::getProcessOutput(bool stdOut, ProcessResult &result } else { if (!contentString.isEmpty() && contentString.endsWith(QLatin1Char('\n'))) contentString.chop(1); - *target = contentString.split(QLatin1Char('\n'), QBS_SKIP_EMPTY_PARTS); + *target = contentString.split(QLatin1Char('\n'), Qt::SkipEmptyParts); } } diff --git a/src/lib/corelib/corelib.qbs b/src/lib/corelib/corelib.qbs index b6450821e..bf414ff80 100644 --- a/src/lib/corelib/corelib.qbs +++ b/src/lib/corelib/corelib.qbs @@ -355,10 +355,12 @@ QbsLibrary { "probesresolver.h", "productitemmultiplexer.cpp", "productitemmultiplexer.h", + "productresolver.cpp", + "productresolver.h", "productscollector.cpp", "productscollector.h", - "productshandler.cpp", - "productshandler.h", + "productsresolver.cpp", + "productsresolver.h", "projectresolver.cpp", "projectresolver.h", ] diff --git a/src/lib/corelib/jsextensions/pkgconfigjs.cpp b/src/lib/corelib/jsextensions/pkgconfigjs.cpp index 2d80ec770..d84c42d8e 100644 --- a/src/lib/corelib/jsextensions/pkgconfigjs.cpp +++ b/src/lib/corelib/jsextensions/pkgconfigjs.cpp @@ -121,7 +121,6 @@ QVariantMap packageToMap(const PcPackage &package) case Type::AlwaysMatch: break; } - result[QStringLiteral("comparison")] = QVariant::fromValue(qint32(version.comparison)); return result; }; diff --git a/src/lib/corelib/language/builtindeclarations.cpp b/src/lib/corelib/language/builtindeclarations.cpp index acf50b4f3..a2502685e 100644 --- a/src/lib/corelib/language/builtindeclarations.cpp +++ b/src/lib/corelib/language/builtindeclarations.cpp @@ -326,9 +326,13 @@ void BuiltinDeclarations::addModuleProviderItem() ItemDeclaration item(ItemType::ModuleProvider); item << nameProperty() << PropertyDeclaration(QStringLiteral("outputBaseDir"), PropertyDeclaration::String) + << PropertyDeclaration(StringConstants::isEagerProperty(), + PropertyDeclaration::Boolean, + StringConstants::trueValue()) + << PropertyDeclaration(StringConstants::moduleNameProperty(), PropertyDeclaration::String) << PropertyDeclaration(QStringLiteral("relativeSearchPaths"), PropertyDeclaration::StringList); - item.setAllowedChildTypes({ItemType::Probe}); + item.setAllowedChildTypes({ItemType::PropertyOptions, ItemType::Probe}); insert(item); } diff --git a/src/lib/corelib/language/evaluator.cpp b/src/lib/corelib/language/evaluator.cpp index 9a19828bb..ef9376194 100644 --- a/src/lib/corelib/language/evaluator.cpp +++ b/src/lib/corelib/language/evaluator.cpp @@ -96,6 +96,7 @@ Evaluator::~Evaluator() valuesToFree << data; for (const JSValue cachedValue : evalData->valueCache) JS_FreeValue(m_scriptEngine->context(), cachedValue); + evalData->item->removeObserver(this); delete evalData; } for (const auto &scopes : std::as_const(m_fileContextScopesMap)) { @@ -191,6 +192,15 @@ std::optional<QStringList> Evaluator::optionalStringListValue( return toStringList(m_scriptEngine, v); } +QVariant Evaluator::variantValue(const Item *item, const QString &name, bool *propertySet) +{ + const ScopedJsValue jsValue(m_scriptEngine->context(), property(item, name)); + handleEvaluationError(item, name); + if (propertySet) + *propertySet = isNonDefaultValue(item, name); + return getJsVariant(m_scriptEngine->context(), jsValue); +} + bool Evaluator::isNonDefaultValue(const Item *item, const QString &name) const { const ValueConstPtr v = item->property(name); @@ -215,24 +225,13 @@ JSValue Evaluator::scriptValue(const Item *item) const auto edata = new EvaluationData; edata->evaluator = this; edata->item = item; - edata->item->setObserver(this); + edata->item->addObserver(this); scriptValue = JS_NewObjectClass(m_scriptEngine->context(), m_scriptClass); attachPointerTo(scriptValue, edata); return scriptValue; } -void Evaluator::clearCache(const Item *item) -{ - const auto data = attachedPointer<EvaluationData>(m_scriptValueMap.value(item), - m_scriptEngine->dataWithPtrClass()); - if (data) { - for (const auto value : std::as_const(data->valueCache)) - JS_FreeValue(m_scriptEngine->context(), value); - data->valueCache.clear(); - } -} - void Evaluator::handleEvaluationError(const Item *item, const QString &name) { throwOnEvaluationError(m_scriptEngine, [&item, &name] () { @@ -278,6 +277,42 @@ Evaluator::FileContextScopes Evaluator::fileContextScopes(const FileContextConst return result; } +// This is the only function in this class that can be called from a thread that is not +// the evaluating one. For this reason, we do not clear the cache here, as that would +// incur enourmous synchronization overhead. Instead, we mark the item's cache as invalidated +// and do the actual clearing only at the very few places where the cache is actually accessed. +void Evaluator::invalidateCache(const Item *item) +{ + std::lock_guard lock(m_cacheInvalidationMutex); + m_invalidatedCaches << item; +} + +void Evaluator::clearCache(const Item *item) +{ + std::lock_guard lock(m_cacheInvalidationMutex); + if (const auto data = attachedPointer<EvaluationData>(m_scriptValueMap.value(item), + m_scriptEngine->dataWithPtrClass())) { + clearCache(*data); + m_invalidatedCaches.remove(data->item); + } +} + +void Evaluator::clearCacheIfInvalidated(EvaluationData &edata) +{ + std::lock_guard lock(m_cacheInvalidationMutex); + if (const auto it = m_invalidatedCaches.find(edata.item); it != m_invalidatedCaches.end()) { + clearCache(edata); + m_invalidatedCaches.erase(it); + } +} + +void Evaluator::clearCache(EvaluationData &edata) +{ + for (const auto value : std::as_const(edata.valueCache)) + JS_FreeValue(m_scriptEngine->context(), value); + edata.valueCache.clear(); +} + void throwOnEvaluationError(ScriptEngine *engine, const std::function<CodeLocation()> &provideFallbackCodeLocation) { @@ -773,7 +808,7 @@ private: void handle(VariantValue *variantValue) override { - *result = engine->toScriptValue(variantValue->value()); + *result = engine->toScriptValue(variantValue->value(), variantValue->id()); engine->takeOwnership(*result); } }; @@ -853,10 +888,11 @@ static void collectValuesFromNextChain( struct EvalResult { JSValue v = JS_UNDEFINED; bool found = false; }; static EvalResult getEvalProperty(ScriptEngine *engine, JSValue obj, const Item *item, - const QString &name, const EvaluationData *data) + const QString &name, EvaluationData *data) { Evaluator * const evaluator = data->evaluator; - const bool isModuleInstance = item->type() == ItemType::ModuleInstance; + const bool isModuleInstance = item->type() == ItemType::ModuleInstance + || item->type() == ItemType::ModuleInstancePlaceholder; for (; item; item = item->prototype()) { if (isModuleInstance && (item->type() == ItemType::Module || item->type() == ItemType::Export)) { @@ -871,6 +907,7 @@ static EvalResult getEvalProperty(ScriptEngine *engine, JSValue obj, const Item evaluator->propertyDependencies()); JSValue result; if (evaluator->cachingEnabled()) { + data->evaluator->clearCacheIfInvalidated(*data); const auto result = data->valueCache.constFind(name); if (result != data->valueCache.constEnd()) { if (debugProperties) @@ -893,6 +930,7 @@ static EvalResult getEvalProperty(ScriptEngine *engine, JSValue obj, const Item qDebug() << "[SC] cache miss " << name << ": " << resultToString(engine->context(), result); if (evaluator->cachingEnabled()) { + data->evaluator->clearCacheIfInvalidated(*data); const auto it = data->valueCache.find(name); if (it != data->valueCache.end()) { JS_FreeValue(engine->context(), it.value()); diff --git a/src/lib/corelib/language/evaluator.h b/src/lib/corelib/language/evaluator.h index d791a4c5d..d86e08eb1 100644 --- a/src/lib/corelib/language/evaluator.h +++ b/src/lib/corelib/language/evaluator.h @@ -49,11 +49,13 @@ #include <QtCore/qhash.h> #include <functional> +#include <mutex> #include <optional> #include <stack> namespace qbs { namespace Internal { +class EvaluationData; class FileTags; class Logger; class PropertyDeclaration; @@ -83,6 +85,8 @@ public: std::optional<QStringList> optionalStringListValue(const Item *item, const QString &name, bool *propertyWasSet = nullptr); + QVariant variantValue(const Item *item, const QString &name, bool *propertySet = nullptr); + void convertToPropertyType(const PropertyDeclaration& decl, const CodeLocation &loc, JSValue &v); @@ -99,6 +103,8 @@ public: void setCachingEnabled(bool enabled) { m_valueCacheEnabled = enabled; } bool cachingEnabled() const { return m_valueCacheEnabled; } void clearCache(const Item *item); + void invalidateCache(const Item *item); + void clearCacheIfInvalidated(EvaluationData &edata); PropertyDependencies &propertyDependencies() { return m_propertyDependencies; } void clearPropertyDependencies() { m_propertyDependencies.clear(); } @@ -113,9 +119,10 @@ public: bool isNonDefaultValue(const Item *item, const QString &name) const; private: - void onItemPropertyChanged(Item *item) override { clearCache(item); } + void onItemPropertyChanged(Item *item) override { invalidateCache(item); } bool evaluateProperty(JSValue *result, const Item *item, const QString &name, bool *propertyWasSet); + void clearCache(EvaluationData &edata); ScriptEngine * const m_scriptEngine; const JSClassID m_scriptClass; @@ -124,6 +131,8 @@ private: QString m_pathPropertiesBaseDir; PropertyDependencies m_propertyDependencies; std::stack<QualifiedId> m_requestedProperties; + std::mutex m_cacheInvalidationMutex; + Set<const Item *> m_invalidatedCaches; bool m_valueCacheEnabled = false; }; @@ -139,7 +148,9 @@ public: m_evaluator->setPathPropertiesBaseDir(baseDir); } - ~EvalCacheEnabler() + ~EvalCacheEnabler() { reset(); } + + void reset() { m_evaluator->setCachingEnabled(false); m_evaluator->clearPathPropertiesBaseDir(); diff --git a/src/lib/corelib/language/item.cpp b/src/lib/corelib/language/item.cpp index 647d05aa2..d7ad9f5f5 100644 --- a/src/lib/corelib/language/item.cpp +++ b/src/lib/corelib/language/item.cpp @@ -40,13 +40,13 @@ #include "item.h" #include "builtindeclarations.h" -#include "deprecationinfo.h" #include "filecontext.h" #include "itemobserver.h" #include "itempool.h" #include "value.h" #include <api/languageinfo.h> +#include <loader/loaderutils.h> #include <logging/categories.h> #include <logging/logger.h> #include <logging/translator.h> @@ -60,25 +60,16 @@ namespace qbs { namespace Internal { -Item::Item(ItemPool *pool, ItemType type) - : m_pool(pool) - , m_observer(nullptr) - , m_prototype(nullptr) - , m_scope(nullptr) - , m_outerItem(nullptr) - , m_parent(nullptr) - , m_type(type) -{ -} - Item *Item::create(ItemPool *pool, ItemType type) { return pool->allocateItem(type); } -Item *Item::clone() const +Item *Item::clone(ItemPool &pool) const { - Item *dup = create(pool(), type()); + assertModuleLocked(); + + Item *dup = create(&pool, type()); dup->m_id = m_id; dup->m_location = m_location; dup->m_prototype = m_prototype; @@ -91,14 +82,14 @@ Item *Item::clone() const dup->m_children.reserve(m_children.size()); for (const Item * const child : std::as_const(m_children)) { - Item *clonedChild = child->clone(); + Item *clonedChild = child->clone(pool); clonedChild->m_parent = dup; dup->m_children.push_back(clonedChild); } for (PropertyMap::const_iterator it = m_properties.constBegin(); it != m_properties.constEnd(); ++it) { - dup->m_properties.insert(it.key(), it.value()->clone()); + dup->m_properties.insert(it.key(), it.value()->clone(pool)); } return dup; @@ -128,6 +119,7 @@ QString Item::typeName() const bool Item::hasProperty(const QString &name) const { + assertModuleLocked(); const Item *item = this; do { if (item->m_properties.contains(name)) @@ -139,11 +131,13 @@ bool Item::hasProperty(const QString &name) const bool Item::hasOwnProperty(const QString &name) const { + assertModuleLocked(); return m_properties.contains(name); } ValuePtr Item::property(const QString &name) const { + assertModuleLocked(); ValuePtr value; const Item *item = this; do { @@ -156,21 +150,22 @@ ValuePtr Item::property(const QString &name) const ValuePtr Item::ownProperty(const QString &name) const { + assertModuleLocked(); return m_properties.value(name); } -ItemValuePtr Item::itemProperty(const QString &name, const Item *itemTemplate) +ItemValuePtr Item::itemProperty(const QString &name, ItemPool &pool, const Item *itemTemplate) { - return itemProperty(name, itemTemplate, ItemValueConstPtr()); + return itemProperty(name, itemTemplate, ItemValueConstPtr(), pool); } -ItemValuePtr Item::itemProperty(const QString &name, const ItemValueConstPtr &value) +ItemValuePtr Item::itemProperty(const QString &name, const ItemValueConstPtr &value, ItemPool &pool) { - return itemProperty(name, value->item(), value); + return itemProperty(name, value->item(), value, pool); } ItemValuePtr Item::itemProperty(const QString &name, const Item *itemTemplate, - const ItemValueConstPtr &itemValue) + const ItemValueConstPtr &itemValue, ItemPool &pool) { const ValuePtr v = property(name); if (v && v->type() == Value::ItemValueType) @@ -178,7 +173,7 @@ ItemValuePtr Item::itemProperty(const QString &name, const Item *itemTemplate, if (!itemTemplate) return ItemValuePtr(); const bool createdByPropertiesBlock = itemValue && itemValue->createdByPropertiesBlock(); - ItemValuePtr result = ItemValue::create(Item::create(m_pool, itemTemplate->type()), + ItemValuePtr result = ItemValue::create(Item::create(&pool, itemTemplate->type()), createdByPropertiesBlock); setProperty(name, result); return result; @@ -211,6 +206,28 @@ bool Item::isOfTypeOrhasParentOfType(ItemType type) const return false; } +void Item::addObserver(ItemObserver *observer) const +{ + // Cached Module properties never change. + if (m_type == ItemType::Module) + return; + + std::lock_guard lock(m_observersMutex); + if (!qEnvironmentVariableIsEmpty("QBS_SANITY_CHECKS")) + QBS_CHECK(!contains(m_observers, observer)); + m_observers << observer; +} + +void Item::removeObserver(ItemObserver *observer) const +{ + if (m_type == ItemType::Module) + return; + std::lock_guard lock(m_observersMutex); + const auto it = std::find(m_observers.begin(), m_observers.end(), observer); + QBS_CHECK(it != m_observers.end()); + m_observers.erase(it); +} + PropertyDeclaration Item::propertyDeclaration(const QString &name, bool allowExpired) const { auto it = m_propertyDeclarations.find(name); @@ -230,12 +247,12 @@ void Item::addModule(const Item::Module &module) QBS_CHECK(none_of(m_modules, [&](const Module &m) { if (m.name != module.name) return false; - if (!!module.productInfo != !!m.productInfo) + if (!!module.product != !!m.product) return true; - if (!module.productInfo) + if (!module.product) return true; - if (module.productInfo->multiplexId == m.productInfo->multiplexId - && module.productInfo->profile == m.productInfo->profile) { + if (module.product->multiplexConfigurationId == m.product->multiplexConfigurationId + && module.product->profileName == m.product->profileName) { return true; } return false; @@ -245,17 +262,13 @@ void Item::addModule(const Item::Module &module) m_modules.push_back(module); } -void Item::setObserver(ItemObserver *observer) const -{ - QBS_ASSERT(!observer || !m_observer, return); // warn if accidentally overwritten - m_observer = observer; -} - void Item::setProperty(const QString &name, const ValuePtr &value) { + assertModuleLocked(); m_properties.insert(name, value); - if (m_observer) - m_observer->onItemPropertyChanged(this); + std::lock_guard lock(m_observersMutex); + for (ItemObserver * const observer : m_observers) + observer->onItemPropertyChanged(this); } void Item::dump() const @@ -272,6 +285,7 @@ bool Item::isPresentModule() const void Item::setupForBuiltinType(DeprecationWarningMode deprecationMode, Logger &logger) { + assertModuleLocked(); const BuiltinDeclarations &builtins = BuiltinDeclarations::instance(); const auto properties = builtins.declarationsForType(type()).properties(); for (const PropertyDeclaration &pd : properties) { @@ -353,8 +367,39 @@ void Item::dump(int indentation) const } } +void Item::lockModule() const +{ + QBS_CHECK(m_type == ItemType::Module); + m_moduleMutex.lock(); +#ifndef NDEBUG + QBS_CHECK(!m_moduleLocked); + m_moduleLocked = true; +#endif +} + +void Item::unlockModule() const +{ + QBS_CHECK(m_type == ItemType::Module); +#ifndef NDEBUG + QBS_CHECK(m_moduleLocked); + m_moduleLocked = false; +#endif + m_moduleMutex.unlock(); +} + +// This safeguard verifies that all contexts which access Module properties have really +// acquired the lock via ModuleItemLocker, as they must. +void Item::assertModuleLocked() const +{ +#ifndef NDEBUG + if (m_type == ItemType::Module) + QBS_CHECK(m_moduleLocked); +#endif +} + void Item::removeProperty(const QString &name) { + assertModuleLocked(); m_properties.remove(name); } diff --git a/src/lib/corelib/language/item.h b/src/lib/corelib/language/item.h index 337cad78a..7f81d53b5 100644 --- a/src/lib/corelib/language/item.h +++ b/src/lib/corelib/language/item.h @@ -53,7 +53,8 @@ #include <QtCore/qlist.h> #include <QtCore/qmap.h> -#include <optional> +#include <atomic> +#include <mutex> #include <vector> namespace qbs { @@ -64,28 +65,24 @@ namespace Internal { class ItemObserver; class ItemPool; class Logger; +class ModuleItemLocker; +class ProductContext; class QBS_AUTOTEST_EXPORT Item : public QbsQmlJS::Managed { friend class ASTPropertiesItemHandler; friend class ItemPool; friend class ItemReaderASTVisitor; + friend class ModuleItemLocker; Q_DISABLE_COPY(Item) - Item(ItemPool *pool, ItemType type); + Item(ItemType type) : m_type(type) {} public: struct Module { QualifiedId name; Item *item = nullptr; - struct ProductInfo { - ProductInfo(Item *i, const QString &m, const QString &p) - : item(i), multiplexId(m), profile(p) {} - Item *item = nullptr; - QString multiplexId; - QString profile; - }; - std::optional<ProductInfo> productInfo; // Set if and only if the dep is a product. + ProductContext *product = nullptr; // Set if and only if the dep is a product. // All items that declared an explicit dependency on this module. Can contain any // number of module instances and at most one product. @@ -105,8 +102,7 @@ public: using PropertyMap = QMap<QString, ValuePtr>; static Item *create(ItemPool *pool, ItemType type); - Item *clone() const; - ItemPool *pool() const { return m_pool; } + Item *clone(ItemPool &pool) const; const QString &id() const { return m_id; } const CodeLocation &location() const { return m_location; } @@ -119,8 +115,8 @@ public: const QList<Item *> &children() const { return m_children; } QList<Item *> &children() { return m_children; } Item *child(ItemType type, bool checkForMultiple = true) const; - const PropertyMap &properties() const { return m_properties; } - PropertyMap &properties() { return m_properties; } + const PropertyMap &properties() const { assertModuleLocked(); return m_properties; } + PropertyMap &properties() { assertModuleLocked(); return m_properties; } const PropertyDeclarationMap &propertyDeclarations() const { return m_propertyDeclarations; } PropertyDeclaration propertyDeclaration(const QString &name, bool allowExpired = true) const; @@ -141,14 +137,15 @@ public: bool hasOwnProperty(const QString &name) const; ValuePtr property(const QString &name) const; ValuePtr ownProperty(const QString &name) const; - ItemValuePtr itemProperty(const QString &name, const Item *itemTemplate = nullptr); - ItemValuePtr itemProperty(const QString &name, const ItemValueConstPtr &value); + ItemValuePtr itemProperty(const QString &name, ItemPool &pool, const Item *itemTemplate = nullptr); + ItemValuePtr itemProperty(const QString &name, const ItemValueConstPtr &value, ItemPool &pool); JSSourceValuePtr sourceProperty(const QString &name) const; VariantValuePtr variantProperty(const QString &name) const; bool isOfTypeOrhasParentOfType(ItemType type) const; - void setObserver(ItemObserver *observer) const; + void addObserver(ItemObserver *observer) const; + void removeObserver(ItemObserver *observer) const; void setProperty(const QString &name, const ValuePtr &value); - void setProperties(const PropertyMap &props) { m_properties = props; } + void setProperties(const PropertyMap &props) { assertModuleLocked(); m_properties = props; } void removeProperty(const QString &name); void setPropertyDeclaration(const QString &name, const PropertyDeclaration &declaration); void setPropertyDeclarations(const PropertyDeclarationMap &decls); @@ -180,18 +177,22 @@ public: private: ItemValuePtr itemProperty(const QString &name, const Item *itemTemplate, - const ItemValueConstPtr &itemValue); + const ItemValueConstPtr &itemValue, ItemPool &pool); void dump(int indentation) const; - ItemPool *m_pool; - mutable ItemObserver *m_observer; + void lockModule() const; + void unlockModule() const; + void assertModuleLocked() const; + + mutable std::vector<ItemObserver *> m_observers; + mutable std::mutex m_observersMutex; QString m_id; CodeLocation m_location; - Item *m_prototype; - Item *m_scope; - Item *m_outerItem; - Item *m_parent; + Item *m_prototype = nullptr; + Item *m_scope = nullptr; + Item *m_outerItem = nullptr; + Item *m_parent = nullptr; QList<Item *> m_children; FileContextPtr m_file; PropertyMap m_properties; @@ -199,6 +200,10 @@ private: PropertyDeclarationMap m_expiredPropertyDeclarations; Modules m_modules; ItemType m_type; + mutable std::mutex m_moduleMutex; +#ifndef NDEBUG + mutable std::atomic_bool m_moduleLocked = false; +#endif }; inline bool operator<(const Item::Module &m1, const Item::Module &m2) { return m1.name < m2.name; } @@ -207,6 +212,22 @@ Item *createNonPresentModule(ItemPool &pool, const QString &name, const QString Item *module); void setScopeForDescendants(Item *item, Item *scope); +// This mechanism is needed because Module items are shared between products (not doing so +// would be prohibitively expensive). +// The competing accesses are between +// - Attaching a temporary qbs module for evaluating the Module condition. +// - Cloning the module when creating an instance. +// - Directly accessing Module properties, which happens rarely (as opposed to properties of +// an instance). +class ModuleItemLocker +{ +public: + ModuleItemLocker(const Item &item) : m_item(item) { item.lockModule(); } + ~ModuleItemLocker() { m_item.unlockModule(); } +private: + const Item &m_item; +}; + } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/language/itempool.cpp b/src/lib/corelib/language/itempool.cpp index ccd22fe2e..6552f92ef 100644 --- a/src/lib/corelib/language/itempool.cpp +++ b/src/lib/corelib/language/itempool.cpp @@ -53,7 +53,7 @@ ItemPool::~ItemPool() Item *ItemPool::allocateItem(const ItemType &type) { - const auto item = new (&m_pool) Item(this, type); + const auto item = new (&m_pool) Item(type); m_items.push_back(item); return item; } diff --git a/src/lib/corelib/language/language.cpp b/src/lib/corelib/language/language.cpp index 33fd3c6a4..0d2813447 100644 --- a/src/lib/corelib/language/language.cpp +++ b/src/lib/corelib/language/language.cpp @@ -51,7 +51,8 @@ #include <buildgraph/rulegraph.h> // TODO: Move to language? #include <buildgraph/transformer.h> #include <jsextensions/jsextensions.h> -#include <loader/productitemmultiplexer.h> +#include <language/value.h> +#include <loader/loaderutils.h> #include <logging/categories.h> #include <logging/translator.h> #include <tools/buildgraphlocker.h> @@ -117,6 +118,12 @@ bool Probe::needsReconfigure(const FileTime &referenceTime) const return Internal::any_of(m_importedFilesUsed, criterion); } +void Probe::restoreValues() +{ + for (auto it = m_properties.begin(), end = m_properties.end(); it != end; ++it) { + m_values[it.key()] = VariantValue::createStored(it.value()); + } +} /*! * \class SourceArtifact @@ -427,7 +434,7 @@ QString ResolvedProduct::uniqueName() const QString ResolvedProduct::fullDisplayName() const { - return ProductItemMultiplexer::fullProductDisplayName(name, multiplexConfigurationId); + return fullProductDisplayName(name, multiplexConfigurationId); } QString ResolvedProduct::profile() const @@ -728,7 +735,7 @@ Set<QString> SourceWildCards::expandPatterns(const GroupConstPtr &group, for (QString pattern : patterns) { pattern.prepend(expandedPrefix); pattern.replace(QLatin1Char('\\'), QLatin1Char('/')); - QStringList parts = pattern.split(QLatin1Char('/'), QBS_SKIP_EMPTY_PARTS); + QStringList parts = pattern.split(QLatin1Char('/'), Qt::SkipEmptyParts); if (FileInfo::isAbsolute(pattern)) { QString rootDir; if (HostOsInfo::isWindowsHost() && pattern.at(0) != QLatin1Char('/')) { diff --git a/src/lib/corelib/language/language.h b/src/lib/corelib/language/language.h index 1dae572a1..86f0f86fe 100644 --- a/src/lib/corelib/language/language.h +++ b/src/lib/corelib/language/language.h @@ -116,17 +116,20 @@ public: const QString &configureScript, const QVariantMap &properties, const QVariantMap &initialProperties, + const QMap<QString, VariantValuePtr> &values, const std::vector<QString> &importedFilesUsed) { return ProbeConstPtr(new Probe(globalId, location, condition, configureScript, properties, - initialProperties, importedFilesUsed)); + initialProperties, values, importedFilesUsed)); } const QString &globalId() const { return m_globalId; } bool condition() const { return m_condition; } + const CodeLocation &location() const { return m_location; } const QString &configureScript() const { return m_configureScript; } const QVariantMap &properties() const { return m_properties; } const QVariantMap &initialProperties() const { return m_initialProperties; } + const QMap<QString, VariantValuePtr> &values() const { return m_values; } const std::vector<QString> &importedFilesUsed() const { return m_importedFilesUsed; } bool needsReconfigure(const FileTime &referenceTime) const; @@ -134,6 +137,8 @@ public: { pool.serializationOp<opType>(m_globalId, m_location, m_condition, m_configureScript, m_properties, m_initialProperties, m_importedFilesUsed); + if constexpr (opType == PersistentPool::OpType::Load) + restoreValues(); } private: @@ -144,21 +149,27 @@ private: QString configureScript, QVariantMap properties, QVariantMap initialProperties, + QMap<QString, VariantValuePtr> values, std::vector<QString> importedFilesUsed) : m_globalId(std::move(globalId)) , m_location(location) , m_configureScript(std::move(configureScript)) , m_properties(std::move(properties)) , m_initialProperties(std::move(initialProperties)) + , m_values(std::move(values)) , m_importedFilesUsed(std::move(importedFilesUsed)) , m_condition(condition) - {} + { + } + + void restoreValues(); QString m_globalId; CodeLocation m_location; QString m_configureScript; QVariantMap m_properties; QVariantMap m_initialProperties; + QMap<QString, VariantValuePtr> m_values; std::vector<QString> m_importedFilesUsed; bool m_condition = false; }; diff --git a/src/lib/corelib/language/moduleproviderinfo.h b/src/lib/corelib/language/moduleproviderinfo.h index 500d370cc..c35ed220a 100644 --- a/src/lib/corelib/language/moduleproviderinfo.h +++ b/src/lib/corelib/language/moduleproviderinfo.h @@ -83,24 +83,25 @@ public: QualifiedId name; QVariantMap config; QString providerFile; + bool isEager{true}; QStringList searchPaths; + QHash<QString, QStringList> searchPathsByModule; bool transientOutput = false; // Not to be serialized. }; -using ModuleProviderInfoList = std::vector<ModuleProviderInfo>; +using ModuleProvidersCacheKey = std::tuple< + QString /*name*/, + QString /*moduleName*/, + QVariantMap /*config*/, + QVariantMap /*qbsModule*/, + int /*lookup*/ +>; +using ModuleProvidersCache = QHash<ModuleProvidersCacheKey, ModuleProviderInfo>; // Persistent info stored between sessions class StoredModuleProviderInfo { public: - using CacheKey = std::tuple< - QString /*name*/, - QVariantMap /*config*/, - QVariantMap /*qbsModule*/, - int /*lookup*/ - >; - using ModuleProvidersCache = QHash<CacheKey, ModuleProviderInfo>; - ModuleProvidersCache providers; template<PersistentPool::OpType opType> void completeSerializationOp(PersistentPool &pool) diff --git a/src/lib/corelib/language/propertydeclaration.cpp b/src/lib/corelib/language/propertydeclaration.cpp index 215918462..dcacde954 100644 --- a/src/lib/corelib/language/propertydeclaration.cpp +++ b/src/lib/corelib/language/propertydeclaration.cpp @@ -46,6 +46,7 @@ #include "value.h" #include <api/languageinfo.h> +#include <loader/loaderutils.h> #include <logging/translator.h> #include <tools/error.h> #include <tools/setupprojectparameters.h> @@ -105,7 +106,6 @@ public: DeprecationInfo deprecationInfo; }; - PropertyDeclaration::PropertyDeclaration() : d(new PropertyDeclarationData) { @@ -308,17 +308,53 @@ QVariant PropertyDeclaration::convertToPropertyType(const QVariant &v, Type t, return c; } +void PropertyDeclaration::checkAllowedValues( + const QVariant &value, + const CodeLocation &loc, + const QString &key, + LoaderState &loaderState) const +{ + const auto type = d->type; + if (type != PropertyDeclaration::String && type != PropertyDeclaration::StringList) + return; + + if (value.isNull()) + return; + + const auto &allowedValues = d->allowedValues; + if (allowedValues.isEmpty()) + return; + + const auto checkValue = [&loc, &allowedValues, &key, &loaderState](const QString &value) + { + if (!allowedValues.contains(value)) { + const auto message = Tr::tr("Value '%1' is not allowed for property '%2'.") + .arg(value, key); + ErrorInfo error(message, loc); + handlePropertyError(error, loaderState.parameters(), loaderState.logger()); + } + }; + + if (type == PropertyDeclaration::StringList) { + const auto strings = value.toStringList(); + for (const auto &string: strings) { + checkValue(string); + } + } else if (type == PropertyDeclaration::String) { + checkValue(value.toString()); + } +} + namespace { class PropertyDeclarationCheck : public ValueHandler { public: - PropertyDeclarationCheck(const Set<Item *> &disabledItems, - const SetupProjectParameters ¶ms, Logger &logger) - : m_disabledItems(disabledItems) - , m_params(params) - , m_logger(logger) - { } - void operator()(Item *item) { handleItem(item); } + PropertyDeclarationCheck(LoaderState &loaderState) : m_loaderState(loaderState) {} + void operator()(Item *item) + { + m_checkingProject = item->type() == ItemType::Project; + handleItem(item); + } private: void handle(JSSourceValue *value) override @@ -326,7 +362,7 @@ private: if (!value->createdByPropertiesBlock()) { const ErrorInfo error(Tr::tr("Property '%1' is not declared.") .arg(m_currentName), value->location()); - handlePropertyError(error, m_params, m_logger); + handlePropertyError(error, m_loaderState.parameters(), m_loaderState.logger()); } } void handle(ItemValue *value) override @@ -365,7 +401,7 @@ private: const ErrorInfo error(Tr::tr("Item '%1' is not declared. " "Did you forget to add a Depends item?") .arg(m_currentModuleName.toString()), location); - handlePropertyError(error, m_params, m_logger); + handlePropertyError(error, m_loaderState.parameters(), m_loaderState.logger()); return false; } @@ -373,16 +409,19 @@ private: } void handleItem(Item *item) { + if (m_checkingProject && item->type() == ItemType::Product) + return; if (!m_handledItems.insert(item).second) return; - if (m_disabledItems.contains(item) - || item->type() == ItemType::Module + if (item->type() == ItemType::Module || item->type() == ItemType::Export || (item->type() == ItemType::ModuleInstance && !item->isPresentModule()) || item->type() == ItemType::Properties // The Properties child of a SubProject item is not a regular item. - || item->type() == ItemType::PropertiesInSubProject) { + || item->type() == ItemType::PropertiesInSubProject + + || m_loaderState.topLevelProject().isDisabledItem(item)) { return; } @@ -395,9 +434,12 @@ private: const PropertyDeclaration decl = item->propertyDeclaration(it.key()); if (decl.isValid()) { const ErrorInfo deprecationError = decl.checkForDeprecation( - m_params.deprecationWarningMode(), it.value()->location(), m_logger); - if (deprecationError.hasError()) - handlePropertyError(deprecationError, m_params, m_logger); + m_loaderState.parameters().deprecationWarningMode(), it.value()->location(), + m_loaderState.logger()); + if (deprecationError.hasError()) { + handlePropertyError(deprecationError, m_loaderState.parameters(), + m_loaderState.logger()); + } continue; } m_currentName = it.key(); @@ -429,20 +471,18 @@ private: Item *parentItem() const { return m_parentItems.back(); } - const Set<Item *> &m_disabledItems; + LoaderState &m_loaderState; Set<Item *> m_handledItems; std::vector<Item *> m_parentItems; QualifiedId m_currentModuleName; QString m_currentName; - const SetupProjectParameters &m_params; - Logger &m_logger; + bool m_checkingProject = false; }; } // namespace -void checkPropertyDeclarations(Item *topLevelItem, const Set<Item *> &disabledItems, - const SetupProjectParameters ¶ms, Logger &logger) +void checkPropertyDeclarations(Item *topLevelItem, LoaderState &loaderState) { - PropertyDeclarationCheck(disabledItems, params, logger)(topLevelItem); + (PropertyDeclarationCheck(loaderState))(topLevelItem); } } // namespace Internal diff --git a/src/lib/corelib/language/propertydeclaration.h b/src/lib/corelib/language/propertydeclaration.h index 8c87faedb..65740afd7 100644 --- a/src/lib/corelib/language/propertydeclaration.h +++ b/src/lib/corelib/language/propertydeclaration.h @@ -41,7 +41,6 @@ #define QBS_PROPERTYDECLARATION_H #include <tools/deprecationwarningmode.h> -#include <tools/set.h> #include <QtCore/qshareddata.h> #include <QtCore/qstring.h> @@ -53,11 +52,11 @@ QT_END_NAMESPACE namespace qbs { class CodeLocation; class ErrorInfo; -class SetupProjectParameters; namespace Internal { class DeprecationInfo; class PropertyDeclarationData; class Item; +class LoaderState; class Logger; class PropertyDeclaration @@ -130,12 +129,17 @@ public: static QVariant convertToPropertyType( const QVariant &v, Type t, const QStringList &namePrefix, const QString &key); + void checkAllowedValues( + const QVariant &value, + const CodeLocation &loc, + const QString &key, + LoaderState &loaderState) const; + private: QSharedDataPointer<PropertyDeclarationData> d; }; -void checkPropertyDeclarations(Item *topLevelItem, const Set<Item *> &disabledItems, - const SetupProjectParameters ¶ms, Logger &logger); +void checkPropertyDeclarations(Item *topLevelItem, LoaderState &loaderState); } // namespace Internal diff --git a/src/lib/corelib/language/scriptengine.cpp b/src/lib/corelib/language/scriptengine.cpp index 638813414..d655c0073 100644 --- a/src/lib/corelib/language/scriptengine.cpp +++ b/src/lib/corelib/language/scriptengine.cpp @@ -58,6 +58,7 @@ #include <tools/stlutils.h> #include <tools/stringconstants.h> +#include <QtCore/qdatetime.h> #include <QtCore/qdebug.h> #include <QtCore/qdiriterator.h> #include <QtCore/qfile.h> @@ -196,6 +197,10 @@ void ScriptEngine::reset() JS_FreeValue(m_context, e.second); m_jsFileCache.clear(); + for (const JSValue &s : std::as_const(m_jsValueCache)) + JS_FreeValue(m_context, s); + m_jsValueCache.clear(); + for (auto it = m_evalResults.cbegin(); it != m_evalResults.cend(); ++it) { for (int i = 0; i < it.value(); ++i) JS_FreeValue(m_context, it.key()); @@ -481,6 +486,37 @@ void ScriptEngine::addInternalExtension(const char *name, JSValue ext) m_internalExtensions.insert(QLatin1String(name), JS_DupValue(m_context, ext)); } +JSValue ScriptEngine::asJsValue(const QVariant &v, quintptr id, bool frozen) +{ + switch (static_cast<QMetaType::Type>(v.userType())) { + case QMetaType::QByteArray: + return asJsValue(v.toByteArray()); + case QMetaType::QString: + return asJsValue(v.toString()); + case QMetaType::QStringList: + return asJsValue(v.toStringList()); + case QMetaType::QVariantList: + return asJsValue(v.toList(), id, frozen); + case QMetaType::Int: + case QMetaType::UInt: + return JS_NewInt32(m_context, v.toInt()); + case QMetaType::Long: + case QMetaType::ULong: + case QMetaType::LongLong: + case QMetaType::ULongLong: + return JS_NewInt64(m_context, v.toInt()); + case QMetaType::Bool: + return JS_NewBool(m_context, v.toBool()); + case QMetaType::QDateTime: + return JS_NewDate( + m_context, v.toDateTime().toString(Qt::ISODateWithMs).toUtf8().constData()); + case QMetaType::QVariantMap: + return asJsValue(v.toMap(), id, frozen); + default: + return JS_UNDEFINED; + } +} + JSValue ScriptEngine::asJsValue(const QByteArray &s) { return JS_NewArrayBufferCopy( @@ -506,12 +542,21 @@ JSValue ScriptEngine::asJsValue(const QStringList &l) return array; } -JSValue ScriptEngine::asJsValue(const QVariantMap &m) +JSValue ScriptEngine::asJsValue(const QVariantMap &m, quintptr id, bool frozen) { + const auto it = id ? m_jsValueCache.constFind(id) : m_jsValueCache.constEnd(); + if (it != m_jsValueCache.constEnd()) + return JS_DupValue(m_context, it.value()); + frozen = id || frozen; JSValue obj = JS_NewObject(m_context); for (auto it = m.begin(); it != m.end(); ++it) - setJsProperty(m_context, obj, it.key(), makeJsVariant(m_context, it.value())); - return obj; + setJsProperty(m_context, obj, it.key(), asJsValue(it.value(), 0, frozen)); + if (frozen) + JS_ObjectSeal(m_context, obj, true); + if (!id) + return obj; + m_jsValueCache[id] = obj; + return JS_DupValue(m_context, obj); } void ScriptEngine::setPropertyOnGlobalObject(const QString &property, JSValue value) @@ -520,13 +565,22 @@ void ScriptEngine::setPropertyOnGlobalObject(const QString &property, JSValue va setJsProperty(m_context, globalObject, property, value); } -JSValue ScriptEngine::asJsValue(const QVariantList &l) +JSValue ScriptEngine::asJsValue(const QVariantList &l, quintptr id, bool frozen) { + const auto it = id ? m_jsValueCache.constFind(id) : m_jsValueCache.constEnd(); + if (it != m_jsValueCache.constEnd()) + return JS_DupValue(m_context, it.value()); + frozen = id || frozen; JSValue array = JS_NewArray(m_context); setJsProperty(m_context, array, QLatin1String("length"), JS_NewInt32(m_context, l.size())); for (int i = 0; i < l.size(); ++i) - JS_SetPropertyUint32(m_context, array, i, makeJsVariant(m_context, l.at(i))); - return array; + JS_SetPropertyUint32(m_context, array, i, asJsValue(l.at(i), 0, frozen)); + if (frozen) + JS_ObjectSeal(m_context, array, true); + if (!id) + return array; + m_jsValueCache[id] = array; + return JS_DupValue(m_context, array); } JSValue ScriptEngine::loadInternalExtension(const QString &uri) diff --git a/src/lib/corelib/language/scriptengine.h b/src/lib/corelib/language/scriptengine.h index 4d797dd43..4a55392e3 100644 --- a/src/lib/corelib/language/scriptengine.h +++ b/src/lib/corelib/language/scriptengine.h @@ -58,6 +58,7 @@ #include <QtCore/qprocess.h> #include <QtCore/qstring.h> +#include <atomic> #include <functional> #include <memory> #include <mutex> @@ -204,7 +205,7 @@ public: JSValue newArray(int length, JsValueOwner owner); void takeOwnership(JSValue v); JSValue undefinedValue() const { return JS_UNDEFINED; } - JSValue toScriptValue(const QVariant &v) const { return makeJsVariant(m_context, v); } + JSValue toScriptValue(const QVariant &v, quintptr id = 0) { return asJsValue(v, id); } JSValue evaluate(JsValueOwner resultOwner, const QString &code, const QString &filePath = QString(), int line = 1, const JSValueList &scopeChain = {}); @@ -283,11 +284,12 @@ public: JSValue getInternalExtension(const char *name) const; void addInternalExtension(const char *name, JSValue ext); + JSValue asJsValue(const QVariant &v, quintptr id = 0, bool frozen = false); JSValue asJsValue(const QByteArray &s); JSValue asJsValue(const QString &s); JSValue asJsValue(const QStringList &l); - JSValue asJsValue(const QVariantList &l); - JSValue asJsValue(const QVariantMap &m); + JSValue asJsValue(const QVariantList &l, quintptr id = 0, bool frozen = false); + JSValue asJsValue(const QVariantMap &m, quintptr id = 0, bool frozen = false); QVariant property(const char *name) const { return m_properties.value(QLatin1String(name)); } void setProperty(const char *k, const QVariant &v) { m_properties.insert(QLatin1String(k), v); } @@ -359,7 +361,7 @@ private: std::unordered_map<QString, JSValue> m_jsFileCache; bool m_propertyCacheEnabled = true; bool m_active = false; - bool m_canceling = false; + std::atomic_bool m_canceling = false; QHash<PropertyCacheKey, QVariant> m_propertyCache; PropertySet m_propertiesRequestedInScript; QHash<QString, PropertySet> m_propertiesRequestedFromArtifact; @@ -396,6 +398,7 @@ private: QHash<QString, JSClassID> m_classes; QHash<QString, JSValue> m_internalExtensions; QHash<QString, JSValue> m_stringCache; + QHash<quintptr, JSValue> m_jsValueCache; QHash<JSValue, int> m_evalResults; std::vector<JSValue *> m_externallyCachedValues; QHash<QPair<Artifact *, QString>, JSValue> m_artifactsScriptValues; diff --git a/src/lib/corelib/language/value.cpp b/src/lib/corelib/language/value.cpp index fabc64ccd..634f54faf 100644 --- a/src/lib/corelib/language/value.cpp +++ b/src/lib/corelib/language/value.cpp @@ -54,11 +54,11 @@ Value::Value(Type t, bool createdByPropertiesBlock) : m_type(t) m_flags |= OriginPropertiesBlock; } -Value::Value(const Value &other) +Value::Value(const Value &other, ItemPool &pool) : m_type(other.m_type), m_scope(other.m_scope), m_scopeName(other.m_scopeName), - m_next(other.m_next ? other.m_next->clone() : ValuePtr()), + m_next(other.m_next ? other.m_next->clone(pool) : ValuePtr()), m_candidates(other.m_candidates), m_flags(other.m_flags) { @@ -134,18 +134,18 @@ JSSourceValue::JSSourceValue(bool createdByPropertiesBlock) { } -JSSourceValue::JSSourceValue(const JSSourceValue &other) : Value(other) +JSSourceValue::JSSourceValue(const JSSourceValue &other, ItemPool &pool) : Value(other, pool) { m_sourceCode = other.m_sourceCode; m_line = other.m_line; m_column = other.m_column; m_file = other.m_file; m_baseValue = other.m_baseValue - ? std::static_pointer_cast<JSSourceValue>(other.m_baseValue->clone()) + ? std::static_pointer_cast<JSSourceValue>(other.m_baseValue->clone(pool)) : JSSourceValuePtr(); m_alternatives = transformed<std::vector<Alternative>>( - other.m_alternatives, [](const auto &alternative) { - return alternative.clone(); }); + other.m_alternatives, [&pool](const auto &alternative) { + return alternative.clone(pool); }); } JSSourceValuePtr JSSourceValue::create(bool createdByPropertiesBlock) @@ -155,9 +155,9 @@ JSSourceValuePtr JSSourceValue::create(bool createdByPropertiesBlock) JSSourceValue::~JSSourceValue() = default; -ValuePtr JSSourceValue::clone() const +ValuePtr JSSourceValue::clone(ItemPool &pool) const { - return std::make_shared<JSSourceValue>(*this); + return std::make_shared<JSSourceValue>(*this, pool); } QString JSSourceValue::sourceCodeForEvaluation() const @@ -235,29 +235,51 @@ ItemValuePtr ItemValue::create(Item *item, bool createdByPropertiesBlock) return std::make_shared<ItemValue>(item, createdByPropertiesBlock); } -ValuePtr ItemValue::clone() const +ValuePtr ItemValue::clone(ItemPool &pool) const { - return create(m_item->clone(), createdByPropertiesBlock()); + return create(m_item->clone(pool), createdByPropertiesBlock()); } +class StoredVariantValue : public VariantValue +{ +public: + explicit StoredVariantValue(QVariant v) : VariantValue(std::move(v)) {} + + quintptr id() const override { return quintptr(this); } +}; + VariantValue::VariantValue(QVariant v) : Value(VariantValueType, false) , m_value(std::move(v)) { } -VariantValuePtr VariantValue::create(const QVariant &v) +VariantValue::VariantValue(const VariantValue &other, ItemPool &pool) + : Value(other, pool), m_value(other.m_value) {} + +template<typename T> +VariantValuePtr createImpl(const QVariant &v) { if (!v.isValid()) - return invalidValue(); + return VariantValue::invalidValue(); if (static_cast<QMetaType::Type>(v.userType()) == QMetaType::Bool) return v.toBool() ? VariantValue::trueValue() : VariantValue::falseValue(); - return std::make_shared<VariantValue>(v); + return std::make_shared<T>(v); +} + +VariantValuePtr VariantValue::create(const QVariant &v) +{ + return createImpl<VariantValue>(v); +} + +VariantValuePtr VariantValue::createStored(const QVariant &v) +{ + return createImpl<StoredVariantValue>(v); } -ValuePtr VariantValue::clone() const +ValuePtr VariantValue::clone(ItemPool &pool) const { - return std::make_shared<VariantValue>(*this); + return std::make_shared<VariantValue>(*this, pool); } const VariantValuePtr &VariantValue::falseValue() diff --git a/src/lib/corelib/language/value.h b/src/lib/corelib/language/value.h index 262813841..1a6746e24 100644 --- a/src/lib/corelib/language/value.h +++ b/src/lib/corelib/language/value.h @@ -50,6 +50,7 @@ namespace qbs { namespace Internal { class Item; +class ItemPool; class ValueHandler; class Value @@ -76,12 +77,13 @@ public: Q_DECLARE_FLAGS(Flags, Flag) Value(Type t, bool createdByPropertiesBlock); - Value(const Value &other); + Value(const Value &other) = delete; + Value(const Value &other, ItemPool &pool); virtual ~Value(); Type type() const { return m_type; } virtual void apply(ValueHandler *) = 0; - virtual ValuePtr clone() const = 0; + virtual ValuePtr clone(ItemPool &) const = 0; virtual CodeLocation location() const { return {}; } Item *scope() const { return m_scope; } @@ -144,13 +146,13 @@ class JSSourceValue : public Value public: explicit JSSourceValue(bool createdByPropertiesBlock); - JSSourceValue(const JSSourceValue &other); + JSSourceValue(const JSSourceValue &other, ItemPool &pool); static JSSourceValuePtr QBS_AUTOTEST_EXPORT create(bool createdByPropertiesBlock = false); ~JSSourceValue() override; void apply(ValueHandler *handler) override { handler->handle(this); } - ValuePtr clone() const override; + ValuePtr clone(ItemPool &pool) const override; void setSourceCode(QStringView sourceCode) { m_sourceCode = sourceCode; } QStringView sourceCode() const { return m_sourceCode; } @@ -180,10 +182,10 @@ public: Alternative() = default; Alternative(PropertyData c, PropertyData o, JSSourceValuePtr v) : condition(std::move(c)), overrideListProperties(std::move(o)), value(std::move(v)) {} - Alternative clone() const + Alternative clone(ItemPool &pool) const { return Alternative(condition, overrideListProperties, - std::static_pointer_cast<JSSourceValue>(value->clone())); + std::static_pointer_cast<JSSourceValue>(value->clone(pool))); } PropertyData condition; @@ -222,7 +224,7 @@ public: private: void apply(ValueHandler *handler) override { handler->handle(this); } - ValuePtr clone() const override; + ValuePtr clone(ItemPool &pool) const override; Item *m_item; }; @@ -232,12 +234,15 @@ class VariantValue : public Value { public: explicit VariantValue(QVariant v); + VariantValue(const VariantValue &v, ItemPool &pool); static VariantValuePtr create(const QVariant &v = QVariant()); + static VariantValuePtr createStored(const QVariant &v = QVariant()); void apply(ValueHandler *handler) override { handler->handle(this); } - ValuePtr clone() const override; + ValuePtr clone(ItemPool &pool) const override; const QVariant &value() const { return m_value; } + virtual quintptr id() const { return 0; } static const VariantValuePtr &falseValue(); static const VariantValuePtr &trueValue(); diff --git a/src/lib/corelib/loader/astimportshandler.cpp b/src/lib/corelib/loader/astimportshandler.cpp index 90887e728..c0281ca24 100644 --- a/src/lib/corelib/loader/astimportshandler.cpp +++ b/src/lib/corelib/loader/astimportshandler.cpp @@ -268,21 +268,9 @@ void ASTImportsHandler::checkImportVersion(const QbsQmlJS::AST::SourceLocation & void ASTImportsHandler::collectPrototypes(const QString &path, const QString &as) { QStringList fileNames; // Yes, file *names*. - if (m_visitorState.findDirectoryEntries(path, &fileNames)) { - for (const QString &fileName : std::as_const(fileNames)) - addPrototype(fileName, path + QLatin1Char('/') + fileName, as, false); - return; - } - - QDirIterator dirIter(path, StringConstants::qbsFileWildcards()); - while (dirIter.hasNext()) { - const QString filePath = dirIter.next(); - const QString fileName = dirIter.fileName(); - if (addPrototype(fileName, filePath, as, true)) - fileNames << fileName; - } - m_visitorState.cacheDirectoryEntries(path, fileNames); - + m_visitorState.findDirectoryEntries(path, &fileNames); + for (const QString &fileName : std::as_const(fileNames)) + addPrototype(fileName, path + QLatin1Char('/') + fileName, as, false); } void ASTImportsHandler::collectPrototypesAndJsCollections(const QString &path, const QString &as, diff --git a/src/lib/corelib/loader/astpropertiesitemhandler.cpp b/src/lib/corelib/loader/astpropertiesitemhandler.cpp index fbb8761b5..cd6a32908 100644 --- a/src/lib/corelib/loader/astpropertiesitemhandler.cpp +++ b/src/lib/corelib/loader/astpropertiesitemhandler.cpp @@ -49,7 +49,8 @@ namespace qbs { namespace Internal { -ASTPropertiesItemHandler::ASTPropertiesItemHandler(Item *parentItem) : m_parentItem(parentItem) +ASTPropertiesItemHandler::ASTPropertiesItemHandler(Item *parentItem, ItemPool &itemPool) + : m_parentItem(parentItem), m_itemPool(itemPool) { } @@ -82,9 +83,11 @@ class PropertiesBlockConverter public: PropertiesBlockConverter(const JSSourceValue::AltProperty &condition, const JSSourceValue::AltProperty &overrideListProperties, - Item *propertiesBlockContainer, const Item *propertiesBlock) + Item *propertiesBlockContainer, const Item *propertiesBlock, + ItemPool &pool) : m_propertiesBlockContainer(propertiesBlockContainer) , m_propertiesBlock(propertiesBlock) + , m_itemPool(pool) { m_alternative.condition = condition; m_alternative.overrideListProperties = overrideListProperties; @@ -99,6 +102,7 @@ private: JSSourceValue::Alternative m_alternative; Item * const m_propertiesBlockContainer; const Item * const m_propertiesBlock; + ItemPool &m_itemPool; void doApply(Item *outer, const Item *inner) { @@ -111,9 +115,9 @@ private: } if (it.value()->type() == Value::ItemValueType) { Item * const innerVal = std::static_pointer_cast<ItemValue>(it.value())->item(); - ItemValuePtr outerVal = outer->itemProperty(it.key()); + ItemValuePtr outerVal = outer->itemProperty(it.key(), m_itemPool); if (!outerVal) { - outerVal = ItemValue::create(Item::create(outer->pool(), innerVal->type()), + outerVal = ItemValue::create(Item::create(&m_itemPool, innerVal->type()), true); outer->setProperty(it.key(), outerVal); } @@ -185,7 +189,7 @@ void ASTPropertiesItemHandler::handlePropertiesBlock(const Item *propertiesItem) const auto overrideListProperties = getPropertyData(propertiesItem, StringConstants::overrideListPropertiesProperty()); PropertiesBlockConverter(condition, overrideListProperties, m_parentItem, - propertiesItem).apply(); + propertiesItem, m_itemPool).apply(); } } // namespace Internal diff --git a/src/lib/corelib/loader/astpropertiesitemhandler.h b/src/lib/corelib/loader/astpropertiesitemhandler.h index 413512ee5..804abb8a1 100644 --- a/src/lib/corelib/loader/astpropertiesitemhandler.h +++ b/src/lib/corelib/loader/astpropertiesitemhandler.h @@ -42,11 +42,12 @@ namespace qbs { namespace Internal { class Item; +class ItemPool; class ASTPropertiesItemHandler { public: - ASTPropertiesItemHandler(Item *parentItem); + ASTPropertiesItemHandler(Item *parentItem, ItemPool &itemPool); void handlePropertiesItems(); @@ -55,6 +56,7 @@ private: void handlePropertiesBlock(const Item *propertiesItem); Item * const m_parentItem; + ItemPool &m_itemPool; }; } // namespace Internal diff --git a/src/lib/corelib/loader/dependenciesresolver.cpp b/src/lib/corelib/loader/dependenciesresolver.cpp index 06852afa9..38cba0aa3 100644 --- a/src/lib/corelib/loader/dependenciesresolver.cpp +++ b/src/lib/corelib/loader/dependenciesresolver.cpp @@ -43,8 +43,6 @@ #include "loaderutils.h" #include "moduleinstantiator.h" #include "moduleloader.h" -#include "moduleproviderloader.h" -#include "productitemmultiplexer.h" #include <language/scriptengine.h> #include <language/evaluator.h> @@ -141,21 +139,34 @@ public: std::queue<FullyResolvedDependsItem> pendingResolvedDependencies; bool requiredByLoadingItem = true; }; -} // namespace -static bool haveSameSubProject(const ProductContext &p1, const ProductContext &p2); -static Item::PropertyMap filterItemProperties(const Item::PropertyMap &properties); -static QVariantMap safeToVariant(JSContext *ctx, const JSValue &v); +class DependenciesContextImpl : public DependenciesContext +{ +public: + DependenciesContextImpl(ProductContext &product, LoaderState &loaderState); -class DependenciesResolver::Private + std::list<DependenciesResolvingState> stateStack; + +private: + std::pair<ProductDependency, ProductContext *> pendingDependency() const override; + + void setSearchPathsForProduct(LoaderState &loaderState); + + ProductContext &m_product; +}; + +class DependenciesResolver { public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} + DependenciesResolver(LoaderState &loaderState, ProductContext &product, Deferral deferral) + : m_loaderState(loaderState), m_product(product), m_deferral(deferral) {} - void initializeState(); + void resolve(); + LoadModuleResult loadModule(Item *loadingItem, const FullyResolvedDependsItem &dependency); + +private: void evaluateNextDependsItem(); HandleDependency handleResolvedDependencies(); - LoadModuleResult loadModule(Item *loadingItem, const FullyResolvedDependsItem &dependency); std::pair<Item::Module *, Item *> findExistingModule(const FullyResolvedDependsItem &dependency, Item *item); void updateModule(Item::Module &module, const FullyResolvedDependsItem &dependency); @@ -172,48 +183,64 @@ public: std::optional<EvaluatedDependsItem> evaluateDependsItem(Item *item); std::queue<FullyResolvedDependsItem> multiplexDependency( const EvaluatedDependsItem &dependency); - void setSearchPathsForProduct(); QVariantMap extractParameters(Item *dependsItem) const; - - LoaderState &loaderState; - ModuleLoader moduleLoader{loaderState}; - std::unordered_map<ProductContext *, std::list<DependenciesResolvingState>> statePerProduct; - qint64 elapsedTime = 0; - - ProductContext *product = nullptr; - std::list<DependenciesResolvingState> *stateStack = nullptr; - Deferral deferral = Deferral::Allowed; + void forwardParameterDeclarations(const Item *dependsItem, const Item::Modules &modules); + void forwardParameterDeclarations(const QualifiedId &moduleName, Item *item, + const Item::Modules &modules); + std::list<DependenciesResolvingState> &stateStack(); + + LoaderState &m_loaderState; + ProductContext &m_product; + Deferral m_deferral; }; -DependenciesResolver::DependenciesResolver(LoaderState &loaderState) - : d(makePimpl<Private>(loaderState)) {} -DependenciesResolver::~DependenciesResolver() = default; +static bool haveSameSubProject(const ProductContext &p1, const ProductContext &p2); +static Item::PropertyMap filterItemProperties(const Item::PropertyMap &properties); +static QVariantMap safeToVariant(JSContext *ctx, const JSValue &v); -bool DependenciesResolver::resolveDependencies(ProductContext &product, Deferral deferral) +} // namespace + +void resolveDependencies(ProductContext &product, Deferral deferral, LoaderState &loaderState) { - QBS_CHECK(!product.dependenciesResolved); + DependenciesResolver(loaderState, product, deferral).resolve(); +} - AccumulatingTimer timer(d->loaderState.parameters().logElapsedTime() - ? &d->elapsedTime : nullptr); +Item *loadBaseModule(ProductContext &product, Item *item, LoaderState &loaderState) +{ + const auto baseDependency = FullyResolvedDependsItem::makeBaseDependency(); + Item * const moduleItem = DependenciesResolver(loaderState, product, Deferral::NotAllowed) + .loadModule(item, baseDependency).moduleItem; + if (Q_UNLIKELY(!moduleItem)) + throw ErrorInfo(Tr::tr("Cannot load base qbs module.")); + return moduleItem; +} - d->product = &product; - d->deferral = deferral; - d->stateStack = &d->statePerProduct[&product]; +namespace { - d->initializeState(); - SearchPathsManager searchPathsMgr(d->loaderState.itemReader(), product.searchPaths); +void DependenciesResolver::resolve() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.dependenciesResolving : nullptr); + + if (!m_product.dependenciesContext) { + m_product.dependenciesContext = std::make_unique<DependenciesContextImpl>( + m_product, m_loaderState); + } else { + QBS_CHECK(!m_product.dependenciesContext->dependenciesResolved); + } + SearchPathsManager searchPathsMgr(m_loaderState.itemReader(), m_product.searchPaths); - while (!d->stateStack->empty()) { - auto &state = d->stateStack->front(); + while (!stateStack().empty()) { + auto &state = stateStack().front(); // If we have pending FullyResolvedDependsItems, then these are handled first. - if (d->handleResolvedDependencies() == HandleDependency::Defer) - return false; + if (handleResolvedDependencies() == HandleDependency::Defer) + return; // The above procedure might have pushed another state to the stack due to recursive // dependencies (i.e. Depends items in the newly loaded module), in which case we // continue with that one. - if (&state != &d->stateStack->front()) + if (&state != &stateStack().front()) continue; // If we have a pending EvaluatedDependsItem, we multiplex it and then handle @@ -223,17 +250,19 @@ bool DependenciesResolver::resolveDependencies(ProductContext &product, Deferral // We postpone handling Depends.productTypes for as long as possible, because // the full type of a product becomes available only after its modules have been loaded. - if (!state.currentDependsItem->productTypes.empty() && deferral == Deferral::Allowed) - return false; + if (!state.currentDependsItem->productTypes.empty() && m_deferral == Deferral::Allowed) + return; - state.pendingResolvedDependencies = d->multiplexDependency(*state.currentDependsItem); + state.pendingResolvedDependencies = multiplexDependency(*state.currentDependsItem); state.currentDependsItem.reset(); + m_deferral = Deferral::Allowed; // We made progress. + continue; } // Here we have no resolved/evaluated Depends items of any kind, so we evaluate the next // pending Depends item. - d->evaluateNextDependsItem(); + evaluateNextDependsItem(); if (state.currentDependsItem) continue; @@ -244,9 +273,9 @@ bool DependenciesResolver::resolveDependencies(ProductContext &product, Deferral // This ensures our invariant: A sorted module list in the product // (dependers after dependencies). - if (d->stateStack->size() > 1) { + if (stateStack().size() > 1) { QBS_CHECK(state.loadingItem->type() == ItemType::ModuleInstance); - Item::Modules &modules = product.item->modules(); + Item::Modules &modules = m_product.item->modules(); const auto loadingItemModule = std::find_if(modules.begin(), modules.end(), [&](const Item::Module &m) { return m.item == state.loadingItem; @@ -256,79 +285,14 @@ bool DependenciesResolver::resolveDependencies(ProductContext &product, Deferral modules.erase(loadingItemModule); modules.push_back(tempModule); } - d->stateStack->pop_front(); - } - return true; -} - -void DependenciesResolver::checkDependencyParameterDeclarations( - const Item *productItem, const QString &productName) const -{ - d->moduleLoader.checkDependencyParameterDeclarations(productItem, productName); -} - -void DependenciesResolver::setStoredModuleProviderInfo( - const StoredModuleProviderInfo &moduleProviderInfo) -{ - d->moduleLoader.setStoredModuleProviderInfo(moduleProviderInfo); -} - -StoredModuleProviderInfo DependenciesResolver::storedModuleProviderInfo() const -{ - return d->moduleLoader.storedModuleProviderInfo(); -} - -const Set<QString> &DependenciesResolver::tempQbsFiles() const -{ - return d->moduleLoader.tempQbsFiles(); -} - -void DependenciesResolver::printProfilingInfo(int indent) -{ - if (!d->loaderState.parameters().logElapsedTime()) - return; - const QByteArray prefix(indent, ' '); - d->loaderState.logger().qbsLog(LoggerInfo, true) - << prefix - << Tr::tr("Setting up product dependencies took %1.") - .arg(elapsedTimeString(d->elapsedTime)); - d->moduleLoader.printProfilingInfo(indent + 2); -} - -Item *DependenciesResolver::loadBaseModule(ProductContext &product, Item *item) -{ - d->product = &product; - d->stateStack = &d->statePerProduct[&product]; - d->deferral = Deferral::NotAllowed; - const auto baseDependency = FullyResolvedDependsItem::makeBaseDependency(); - Item * const moduleItem = d->loadModule(item, baseDependency).moduleItem; - if (Q_UNLIKELY(!moduleItem)) - throw ErrorInfo(Tr::tr("Cannot load base qbs module.")); - return moduleItem; -} - -void DependenciesResolver::Private::initializeState() -{ - if (!stateStack->empty()) - return; - - // Initialize the state with the direct Depends items of the product item. - // This is executed once per product, while the function might be entered - // multiple times due to deferrals. - setSearchPathsForProduct(); - DependenciesResolvingState newState{product->item,}; - for (Item * const child : product->item->children()) { - if (child->type() == ItemType::Depends) - newState.pendingDependsItems.push(child); + stateStack().pop_front(); } - stateStack->push_front(std::move(newState)); - stateStack->front().pendingResolvedDependencies.push( - FullyResolvedDependsItem::makeBaseDependency()); + m_product.dependenciesContext->dependenciesResolved = true; } -void DependenciesResolver::Private::evaluateNextDependsItem() +void DependenciesResolver::evaluateNextDependsItem() { - auto &state = stateStack->front(); + auto &state = stateStack().front(); while (!state.pendingDependsItems.empty()) { QBS_CHECK(!state.currentDependsItem); QBS_CHECK(state.pendingResolvedDependencies.empty()); @@ -344,9 +308,9 @@ void DependenciesResolver::Private::evaluateNextDependsItem() } } -HandleDependency DependenciesResolver::Private::handleResolvedDependencies() +HandleDependency DependenciesResolver::handleResolvedDependencies() { - DependenciesResolvingState &state = stateStack->front(); + DependenciesResolvingState &state = stateStack().front(); while (!state.pendingResolvedDependencies.empty()) { QBS_CHECK(!state.currentDependsItem); const FullyResolvedDependsItem dependency = state.pendingResolvedDependencies.front(); @@ -354,7 +318,7 @@ HandleDependency DependenciesResolver::Private::handleResolvedDependencies() const LoadModuleResult res = loadModule(state.loadingItem, dependency); switch (res.handleDependency) { case HandleDependency::Defer: - QBS_CHECK(deferral == Deferral::Allowed); + QBS_CHECK(m_deferral == Deferral::Allowed); // Optimization: We already looked up the product, so let's not do that again // next time. @@ -373,6 +337,7 @@ HandleDependency DependenciesResolver::Private::handleResolvedDependencies() state.pendingResolvedDependencies.pop(); continue; } + m_deferral = Deferral::Allowed; // We made progress. break; } @@ -385,10 +350,10 @@ HandleDependency DependenciesResolver::Private::handleResolvedDependencies() moduleDependsItems.push(child); } state.pendingResolvedDependencies.pop(); - stateStack->push_front( + stateStack().push_front( {res.moduleItem, dependency, moduleDependsItems, {}, {}, dependency.requiredGlobally || state.requiredByLoadingItem}); - stateStack->front().pendingResolvedDependencies.push( + stateStack().front().pendingResolvedDependencies.push( FullyResolvedDependsItem::makeBaseDependency()); break; } catch (const ErrorInfo &e) { @@ -404,23 +369,23 @@ HandleDependency DependenciesResolver::Private::handleResolvedDependencies() // See QBS-1338 for why we do not abort handling the product. state.pendingResolvedDependencies.pop(); - Item::Modules &modules = product->item->modules(); + Item::Modules &modules = m_product.item->modules(); // Unwind. - while (stateStack->size() > 1) { + while (stateStack().size() > 1) { const auto loadingItemModule = std::find_if( modules.begin(), modules.end(), [&](const Item::Module &m) { - return m.item == stateStack->front().loadingItem; + return m.item == stateStack().front().loadingItem; }); for (auto it = loadingItemModule; it != modules.end(); ++it) { - createNonPresentModule(loaderState.itemPool(), it->name.toString(), + createNonPresentModule(m_loaderState.itemPool(), it->name.toString(), QLatin1String("error in Depends chain"), it->item); } modules.erase(loadingItemModule, modules.end()); - stateStack->pop_front(); + stateStack().pop_front(); } - product->handleError(e); + m_product.handleError(e); return HandleDependency::Ignore; } } @@ -434,7 +399,7 @@ HandleDependency DependenciesResolver::Private::handleResolvedDependencies() // created module is added to the module list of the product item and additionally to the // loading item's one, if it is not the product. Its name is also injected into the respective // scopes. -LoadModuleResult DependenciesResolver::Private::loadModule( +LoadModuleResult DependenciesResolver::loadModule( Item *loadingItem, const FullyResolvedDependsItem &dependency) { qCDebug(lcModuleLoader) << "loadModule name:" << dependency.name.toString() @@ -447,7 +412,7 @@ LoadModuleResult DependenciesResolver::Private::loadModule( // The module might already have been loaded for this product (directly or indirectly). const auto &[existingModule, moduleWithSameName] - = findExistingModule(dependency, product->item); + = findExistingModule(dependency, m_product.item); if (existingModule) { // Merge version range and required property. These will be checked again // after probes resolving. @@ -475,7 +440,7 @@ LoadModuleResult DependenciesResolver::Private::loadModule( if (checkResult.first) { QBS_CHECK(productDep->mergedExportItem); - moduleItem = productDep->mergedExportItem->clone(); + moduleItem = productDep->mergedExportItem->clone(m_loaderState.itemPool()); moduleItem->setParent(nullptr); // Needed for isolated Export item evaluation. @@ -496,17 +461,16 @@ LoadModuleResult DependenciesResolver::Private::loadModule( // The loading name is only used to ensure consistent sorting in case of equal // value priorities; see ModulePropertyMerger. QString loadingName; - if (loadingItem == product->item) { - loadingName = product->name; - } else if (!stateStack->empty()) { - const auto &loadingItemOrigin = stateStack->front().loadingItemOrigin; + if (loadingItem == m_product.item) { + loadingName = m_product.name; + } else if (m_product.dependenciesContext && !stateStack().empty()) { + const auto &loadingItemOrigin = stateStack().front().loadingItemOrigin; loadingName = loadingItemOrigin.name.toString() + loadingItemOrigin.multiplexId + loadingItemOrigin.profile; } - loaderState.moduleInstantiator().instantiate({ - product->item, product->name, loadingItem, loadingName, moduleItem, moduleWithSameName, - productDep ? productDep->item : nullptr, product->scope, product->project->scope, - dependency.name, dependency.id(), bool(existingModule)}); + instantiateModule({m_product, loadingItem, loadingName, moduleItem, moduleWithSameName, + productDep ? productDep->item : nullptr, dependency.name, dependency.id(), + bool(existingModule)}, m_loaderState); // At this point, a null module item is only possible for a non-required dependency. // Note that we still needed to to the instantiation above, as that injects the module @@ -532,7 +496,7 @@ LoadModuleResult DependenciesResolver::Private::loadModule( } qCDebug(lcModuleLoader) << "module loaded:" << dependency.name.toString(); - if (product->item) { + if (m_product.item) { Item::Module module = createModule(dependency, moduleItem, productDep); if (module.name.toString() != StringConstants::qbsModule()) { @@ -544,14 +508,14 @@ LoadModuleResult DependenciesResolver::Private::loadModule( } module.required = dependency.requiredGlobally; module.loadingItems.push_back(loadingItem); - module.maxDependsChainLength = stateStack->size(); - product->item->addModule(module); + module.maxDependsChainLength = m_product.dependenciesContext ? stateStack().size() : 1; + m_product.item->addModule(module); addLocalModule(); } return {moduleItem, nullptr, HandleDependency::Use}; } -std::pair<Item::Module *, Item *> DependenciesResolver::Private::findExistingModule( +std::pair<Item::Module *, Item *> DependenciesResolver::findExistingModule( const FullyResolvedDependsItem &dependency, Item *item) { if (!item) // Happens if and only if called via loadBaseModule(). @@ -560,12 +524,12 @@ std::pair<Item::Module *, Item *> DependenciesResolver::Private::findExistingMod for (Item::Module &m : item->modules()) { if (m.name != dependency.name) continue; - if (!m.productInfo) { + if (!m.product) { QBS_CHECK(!dependency.product); return {&m, m.item}; } - if ((dependency.profile.isEmpty() || (m.productInfo->profile == dependency.profile)) - && m.productInfo->multiplexId == dependency.multiplexId) { + if ((dependency.profile.isEmpty() || (m.product->profileName == dependency.profile)) + && m.product->multiplexConfigurationId == dependency.multiplexId) { return {&m, m.item}; } @@ -576,39 +540,38 @@ std::pair<Item::Module *, Item *> DependenciesResolver::Private::findExistingMod return {nullptr, moduleWithSameName}; } -void DependenciesResolver::Private::updateModule( +void DependenciesResolver::updateModule( Item::Module &module, const FullyResolvedDependsItem &dependency) { - moduleLoader.forwardParameterDeclarations(dependency.item, product->item->modules()); + forwardParameterDeclarations(dependency.item, m_product.item->modules()); // TODO: Use priorities like for property values. See QBS-1300. mergeParameters(module.parameters, dependency.parameters); module.versionRange.narrowDown(dependency.versionRange); module.required |= dependency.requiredGlobally; - if (int(stateStack->size()) > module.maxDependsChainLength) - module.maxDependsChainLength = stateStack->size(); + if (int(stateStack().size()) > module.maxDependsChainLength) + module.maxDependsChainLength = stateStack().size(); } -ProductContext *DependenciesResolver::Private::findMatchingProduct( +ProductContext *DependenciesResolver::findMatchingProduct( const FullyResolvedDependsItem &dependency) { - const auto candidates = product->project->topLevelProject - ->productsByName.equal_range(dependency.name.toString()); - for (auto it = candidates.first; it != candidates.second; ++it) { - ProductContext * const candidate = it->second; - if (candidate->multiplexConfigurationId != dependency.multiplexId) - continue; - if (!dependency.profile.isEmpty() && dependency.profile != candidate->profileName) - continue; - if (dependency.limitToSubProject && !haveSameSubProject(*product, *candidate)) - continue; - return candidate; - } - return nullptr; + const auto constraint = [this, &dependency](ProductContext &product) { + if (product.multiplexConfigurationId != dependency.multiplexId) + return false; + if (!dependency.profile.isEmpty() && dependency.profile != product.profileName) + return false; + if (dependency.limitToSubProject && !haveSameSubProject(m_product, product)) + return false; + return true; + + }; + return m_product.project->topLevelProject->productWithNameAndConstraint( + dependency.name.toString(), constraint); } -Item *DependenciesResolver::Private::findMatchingModule( +Item *DependenciesResolver::findMatchingModule( const FullyResolvedDependsItem &dependency) { // If we can tell that this is supposed to be a product dependency, we can skip @@ -618,40 +581,38 @@ Item *DependenciesResolver::Private::findMatchingModule( if (!dependency.profile.isEmpty()) { throw ErrorInfo(Tr::tr("Product '%1' depends on '%2', which does not exist " "for the requested profile '%3'.") - .arg(product->displayName(), dependency.displayName(), + .arg(m_product.displayName(), dependency.displayName(), dependency.profile), - product->item->location()); + m_product.item->location()); } throw ErrorInfo(Tr::tr("Product '%1' depends on '%2', which does not exist.") - .arg(product->displayName(), dependency.displayName()), - product->item->location()); + .arg(m_product.displayName(), dependency.displayName()), + m_product.item->location()); } return nullptr; } - const ModuleLoader::ProductContext loaderContext{ - product->item, product->project->item, product->name, product->uniqueName(), - product->profileName, product->multiplexConfigurationId, product->moduleProperties, - product->profileModuleProperties}; - const ModuleLoader::Result loaderResult = moduleLoader.searchAndLoadModuleFile( - loaderContext, dependency.location(), dependency.name, dependency.fallbackMode, - dependency.requiredGlobally); - - Item *moduleItem = loaderResult.moduleItem; - product->info.probes << loaderResult.providerProbes; - if (moduleItem) { + if (Item *moduleItem = searchAndLoadModuleFile(m_loaderState, m_product, dependency.location(), + dependency.name, dependency.fallbackMode)) { + QBS_CHECK(moduleItem->type() == ItemType::Module); Item * const proto = moduleItem; - moduleItem = moduleItem->clone(); + ModuleItemLocker locker(*moduleItem); + moduleItem = moduleItem->clone(m_loaderState.itemPool()); moduleItem->setPrototype(proto); // For parameter declarations. - } else if (dependency.requiredGlobally) { - throw ErrorInfo(Tr::tr("Dependency '%1' not found for product '%2'.") - .arg(dependency.name.toString(), product->displayName()), - dependency.location()); + return moduleItem; } - return moduleItem; + + if (!dependency.requiredGlobally) { + return createNonPresentModule(m_loaderState.itemPool(), dependency.name.toString(), + QStringLiteral("not found"), nullptr); + } + + throw ErrorInfo(Tr::tr("Dependency '%1' not found for product '%2'.") + .arg(dependency.name.toString(), m_product.displayName()), + dependency.location()); } -std::pair<bool, HandleDependency> DependenciesResolver::Private::checkProductDependency( +std::pair<bool, HandleDependency> DependenciesResolver::checkProductDependency( const FullyResolvedDependsItem &depSpec, const ProductContext &dep) { // Optimization: If we already checked the product earlier and then deferred, we don't @@ -659,20 +620,18 @@ std::pair<bool, HandleDependency> DependenciesResolver::Private::checkProductDep if (!depSpec.checkProduct) return {true, HandleDependency::Use}; - if (&dep == product) { + if (&dep == &m_product) { throw ErrorInfo(Tr::tr("Dependency '%1' refers to itself.").arg(depSpec.name.toString()), depSpec.location()); } - if (any_of(product->project->topLevelProject->productsToHandle, [&dep](const auto &e) { - return e.first == &dep; - })) { - if (deferral == Deferral::Allowed) + if (m_product.project->topLevelProject->isProductQueuedForHandling(dep)) { + if (m_deferral == Deferral::Allowed) return {false, HandleDependency::Defer}; ErrorInfo e; e.append(Tr::tr("Cyclic dependencies detected:")); e.append(Tr::tr("First product is '%1'.") - .arg(product->displayName()), product->item->location()); + .arg(m_product.displayName()), m_product.item->location()); e.append(Tr::tr("Second product is '%1'.") .arg(dep.displayName()), dep.item->location()); e.append(Tr::tr("Requested here."), depSpec.location()); @@ -681,12 +640,12 @@ std::pair<bool, HandleDependency> DependenciesResolver::Private::checkProductDep // This covers both the case of user-disabled products and products with errors. // The latter are force-disabled in ProductContext::handleError(). - if (product->project->topLevelProject->disabledItems.contains(dep.item)) { + if (m_product.project->topLevelProject->isDisabledItem(dep.item)) { if (depSpec.requiredGlobally) { ErrorInfo e; e.append(Tr::tr("Product '%1' depends on '%2',") - .arg(product->displayName(), dep.displayName()), - product->item->location()); + .arg(m_product.displayName(), dep.displayName()), + m_product.item->location()); e.append(Tr::tr("but product '%1' is disabled.").arg(dep.displayName()), dep.item->location()); throw e; @@ -696,10 +655,15 @@ std::pair<bool, HandleDependency> DependenciesResolver::Private::checkProductDep return {true, HandleDependency::Use}; } -void DependenciesResolver::Private::checkModule( +void DependenciesResolver::checkModule( const FullyResolvedDependsItem &dependency, Item *moduleItem, ProductContext *productDep) { - for (auto it = stateStack->begin(); it != stateStack->end(); ++it) { + // When loading a pseudo or temporary qbs module in early setup via loadBaseModule(), + // there is no proper state yet. + if (!m_product.dependenciesContext) + return; + + for (auto it = stateStack().begin(); it != stateStack().end(); ++it) { Item *itemToCheck = moduleItem; if (it->loadingItem != itemToCheck) { if (!productDep) @@ -714,11 +678,11 @@ void DependenciesResolver::Private::checkModule( e.append(it->loadingItemOrigin.name.toString(), it->loadingItemOrigin.location()); if (it->loadingItem->type() == ItemType::ModuleInstance) { - createNonPresentModule(loaderState.itemPool(), + createNonPresentModule(m_loaderState.itemPool(), it->loadingItemOrigin.name.toString(), QLatin1String("cyclic dependency"), it->loadingItem); } - if (it == stateStack->begin()) + if (it == stateStack().begin()) break; --it; } @@ -728,44 +692,30 @@ void DependenciesResolver::Private::checkModule( checkForModuleNamePrefixCollision(dependency); } -void DependenciesResolver::Private::adjustDependsItemForMultiplexing(Item *dependsItem) +void DependenciesResolver::adjustDependsItemForMultiplexing(Item *dependsItem) { - Evaluator &evaluator = loaderState.evaluator(); + if (m_product.name.startsWith(StringConstants::shadowProductPrefix())) + return; + + Evaluator &evaluator = m_loaderState.evaluator(); const QString name = evaluator.stringValue(dependsItem, StringConstants::nameProperty()); - const bool productIsMultiplexed = !product->multiplexConfigurationId.isEmpty(); - if (name == product->name) { + const bool productIsMultiplexed = !m_product.multiplexConfigurationId.isEmpty(); + if (name == m_product.name) { QBS_CHECK(!productIsMultiplexed); // This product must be an aggregator. return; } - const auto productRange = product->project->topLevelProject->productsByName.equal_range(name); - if (productRange.first == productRange.second) - return; // Dependency is a module. Nothing to adjust. - - bool profilesPropertyIsSet; - const QStringList profiles = evaluator.stringListValue( - dependsItem, StringConstants::profilesProperty(), &profilesPropertyIsSet); - std::vector<const ProductContext *> multiplexedDependencies; bool hasNonMultiplexedDependency = false; - for (auto it = productRange.first; it != productRange.second; ++it) { - if (!it->second->multiplexConfigurationId.isEmpty()) - multiplexedDependencies.push_back(it->second); - else + const std::vector<ProductContext *> multiplexedDependencies = m_product.project + ->topLevelProject->productsWithNameAndConstraint(name, [&hasNonMultiplexedDependency] + (const ProductContext &product) { + if (product.multiplexConfigurationId.isEmpty()) { hasNonMultiplexedDependency = true; - } - bool hasMultiplexedDependencies = !multiplexedDependencies.empty(); - - static const auto multiplexConfigurationIntersects = [](const QVariantMap &lhs, - const QVariantMap &rhs) { - QBS_CHECK(!lhs.isEmpty() && !rhs.isEmpty()); - for (auto lhsProperty = lhs.constBegin(); lhsProperty != lhs.constEnd(); lhsProperty++) { - const auto rhsProperty = rhs.find(lhsProperty.key()); - const bool isCommonProperty = rhsProperty != rhs.constEnd(); - if (isCommonProperty && lhsProperty.value() != rhsProperty.value()) - return false; + return false; } return true; - }; + }); + const bool hasMultiplexedDependencies = !multiplexedDependencies.empty(); // These are the allowed cases: // (1) Normal dependency with no multiplexing whatsoever. @@ -783,35 +733,38 @@ void DependenciesResolver::Private::adjustDependsItemForMultiplexing(Item *depen // with a matching profile regardless of whether an aggregator exists or not. // (4) The product is multiplexed, but the dependency is not. We don't have to adapt // any Depends items. - // (5) The product is a "shadow product". In that case, we know which product - // it should have a dependency on, and we make sure we depend on that. - // (1) and (4) if (!hasMultiplexedDependencies) return; + bool profilesPropertyIsSet; + const QStringList profiles = evaluator.stringListValue( + dependsItem, StringConstants::profilesProperty(), &profilesPropertyIsSet); + // (3a) if (!productIsMultiplexed && hasNonMultiplexedDependency && !profilesPropertyIsSet) return; + static const auto multiplexConfigurationIntersects = [](const QVariantMap &lhs, + const QVariantMap &rhs) { + QBS_CHECK(!lhs.isEmpty() && !rhs.isEmpty()); + for (auto lhsProperty = lhs.constBegin(); lhsProperty != lhs.constEnd(); lhsProperty++) { + const auto rhsProperty = rhs.find(lhsProperty.key()); + const bool isCommonProperty = rhsProperty != rhs.constEnd(); + if (isCommonProperty && lhsProperty.value() != rhsProperty.value()) + return false; + } + return true; + }; + QStringList multiplexIds; - const ShadowProductInfo shadowProductInfo = getShadowProductInfo(*product); - const bool isShadowProduct = shadowProductInfo.first && shadowProductInfo.second == name; - const auto productMultiplexConfig - = ProductItemMultiplexer::multiplexIdToVariantMap(product->multiplexConfigurationId); + const auto productMultiplexConfig = m_loaderState.topLevelProject().multiplexConfiguration( + m_product.multiplexConfigurationId); for (const ProductContext *dependency : multiplexedDependencies) { - const bool depMatchesShadowProduct = isShadowProduct - && dependency->item == product->item->parent(); - const QString depMultiplexId = dependency->multiplexConfigurationId; - if (depMatchesShadowProduct) { // (5) - dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), - VariantValue::create(depMultiplexId)); - return; - } if (productIsMultiplexed && !profilesPropertyIsSet) { // 2a - if (dependency->multiplexConfigurationId == product->multiplexConfigurationId) { - const ValuePtr &multiplexId = product->item->property( + if (dependency->multiplexConfigurationId == m_product.multiplexConfigurationId) { + const ValuePtr &multiplexId = m_product.item->property( StringConstants::multiplexConfigurationIdProperty()); dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), multiplexId); @@ -819,8 +772,8 @@ void DependenciesResolver::Private::adjustDependsItemForMultiplexing(Item *depen } // Otherwise collect partial matches and decide later - const auto dependencyMultiplexConfig = ProductItemMultiplexer::multiplexIdToVariantMap( - dependency->multiplexConfigurationId); + const auto dependencyMultiplexConfig = m_loaderState.topLevelProject() + .multiplexConfiguration(dependency->multiplexConfigurationId); if (multiplexConfigurationIntersects(dependencyMultiplexConfig, productMultiplexConfig)) multiplexIds << dependency->multiplexConfigurationId; @@ -829,12 +782,11 @@ void DependenciesResolver::Private::adjustDependsItemForMultiplexing(Item *depen const bool profileMatch = !profilesPropertyIsSet || profiles.empty() || profiles.contains(dependency->profileName); if (profileMatch) - multiplexIds << depMultiplexId; + multiplexIds << dependency->multiplexConfigurationId; } } if (multiplexIds.empty()) { - const QString productName = ProductItemMultiplexer::fullProductDisplayName( - product->name, product->multiplexConfigurationId); + const QString productName = m_product.displayName(); throw ErrorInfo(Tr::tr("Dependency from product '%1' to product '%2' not fulfilled. " "There are no eligible multiplex candidates.").arg(productName, name), @@ -845,11 +797,11 @@ void DependenciesResolver::Private::adjustDependsItemForMultiplexing(Item *depen if (productIsMultiplexed && !profilesPropertyIsSet && multiplexIds.size() > 1) { QStringList candidateNames; for (const auto &id : std::as_const(multiplexIds)) - candidateNames << ProductItemMultiplexer::fullProductDisplayName(name, id); + candidateNames << fullProductDisplayName(name, id); throw ErrorInfo( Tr::tr("Dependency from product '%1' to product '%2' is ambiguous. " "Eligible multiplex candidates: %3.").arg( - product->displayName(), name, candidateNames.join(QLatin1String(", "))), + m_product.displayName(), name, candidateNames.join(QLatin1String(", "))), dependsItem->location()); } @@ -858,10 +810,10 @@ void DependenciesResolver::Private::adjustDependsItemForMultiplexing(Item *depen } -std::optional<EvaluatedDependsItem> DependenciesResolver::Private::evaluateDependsItem(Item *item) +std::optional<EvaluatedDependsItem> DependenciesResolver::evaluateDependsItem(Item *item) { - Evaluator &evaluator = loaderState.evaluator(); - if (!product->project->topLevelProject->checkItemCondition(item, evaluator)) { + Evaluator &evaluator = m_loaderState.evaluator(); + if (!m_product.project->topLevelProject->checkItemCondition(item, evaluator)) { qCDebug(lcModuleLoader) << "Depends item disabled, ignoring."; return {}; } @@ -898,7 +850,7 @@ std::optional<EvaluatedDependsItem> DependenciesResolver::Private::evaluateDepen } const FallbackMode fallbackMode - = loaderState.parameters().fallbackProviderEnabled() + = m_loaderState.parameters().fallbackProviderEnabled() && evaluator.boolValue(item, StringConstants::enableFallbackProperty()) ? FallbackMode::Enabled : FallbackMode::Disabled; @@ -921,11 +873,14 @@ std::optional<EvaluatedDependsItem> DependenciesResolver::Private::evaluateDepen const QStringList multiplexIds = evaluator.stringListValue( item, StringConstants::multiplexConfigurationIdsProperty()); adjustParametersScopes(item, item); - moduleLoader.forwardParameterDeclarations(item, product->item->modules()); + forwardParameterDeclarations(item, m_product.item->modules()); const QVariantMap parameters = extractParameters(item); + const FileTags productTypeTags = FileTags::fromStringList(productTypes); + if (!productTypeTags.empty()) + m_product.bulkDependencies.emplace_back(productTypeTags, item->location()); return EvaluatedDependsItem{ - item, QualifiedId::fromString(name), submodules, FileTags::fromStringList(productTypes), + item, QualifiedId::fromString(name), submodules, productTypeTags, multiplexIds, profiles, {minVersion, maxVersion}, parameters, limitToSubProject, fallbackMode, required}; } @@ -934,22 +889,16 @@ std::optional<EvaluatedDependsItem> DependenciesResolver::Private::evaluateDepen // Depends.profiles, as well as internally set up multiplexing axes. // Each entry in the resulting queue corresponds to exactly one product or module to pull in. std::queue<FullyResolvedDependsItem> -DependenciesResolver::Private::multiplexDependency(const EvaluatedDependsItem &dependency) +DependenciesResolver::multiplexDependency(const EvaluatedDependsItem &dependency) { std::queue<FullyResolvedDependsItem> dependencies; if (!dependency.productTypes.empty()) { - std::vector<ProductContext *> matchingProducts; - for (const FileTag &typeTag : dependency.productTypes) { - const auto range = product->project->topLevelProject->productsByType.equal_range(typeTag); - for (auto it = range.first; it != range.second; ++it) { - if (it->second != product - && it->second->name != product->name - && (!dependency.limitToSubProject - || haveSameSubProject(*product, *it->second))) { - matchingProducts.push_back(it->second); - } - } - } + const auto constraint = [&](const ProductContext &product) { + return &product != &m_product && product.name != m_product.name + && (!dependency.limitToSubProject || haveSameSubProject(m_product, product)); + }; + const std::vector<ProductContext *> matchingProducts = m_product.project->topLevelProject + ->productsWithTypeAndConstraint(dependency.productTypes, constraint); if (matchingProducts.empty()) { qCDebug(lcModuleLoader) << "Depends.productTypes does not match anything." << dependency.item->location(); @@ -979,22 +928,7 @@ DependenciesResolver::Private::multiplexDependency(const EvaluatedDependsItem &d return dependencies; } -void DependenciesResolver::Private::setSearchPathsForProduct() -{ - QBS_CHECK(product->searchPaths.isEmpty()); - - product->searchPaths = loaderState.itemReader().readExtraSearchPaths(product->item); - Settings settings(loaderState.parameters().settingsDirectory()); - const QStringList prefsSearchPaths = Preferences(&settings, product->profileModuleProperties) - .searchPaths(); - const QStringList ¤tSearchPaths = loaderState.itemReader().allSearchPaths(); - for (const QString &p : prefsSearchPaths) { - if (!currentSearchPaths.contains(p) && FileInfo(p).exists()) - product->searchPaths << p; - } -} - -QVariantMap DependenciesResolver::Private::extractParameters(Item *dependsItem) const +QVariantMap DependenciesResolver::extractParameters(Item *dependsItem) const { QVariantMap result; const Item::PropertyMap &itemProperties = filterItemProperties(dependsItem->properties()); @@ -1006,9 +940,9 @@ QVariantMap DependenciesResolver::Private::extractParameters(Item *dependsItem) // way, without allocationg an extra map and exchanging the list of children? dependsItem->setProperties(itemProperties); - JSValue sv = loaderState.evaluator().scriptValue(dependsItem); + JSValue sv = m_loaderState.evaluator().scriptValue(dependsItem); try { - result = safeToVariant(loaderState.evaluator().engine()->context(), sv); + result = safeToVariant(m_loaderState.evaluator().engine()->context(), sv); } catch (const ErrorInfo &exception) { auto ei = exception; ei.prepend(Tr::tr("Error in dependency parameter."), dependsItem->location()); @@ -1018,13 +952,51 @@ QVariantMap DependenciesResolver::Private::extractParameters(Item *dependsItem) return result; } -void DependenciesResolver::Private::checkForModuleNamePrefixCollision( +void DependenciesResolver::forwardParameterDeclarations(const Item *dependsItem, + const Item::Modules &modules) +{ + for (auto it = dependsItem->properties().begin(); it != dependsItem->properties().end(); ++it) { + if (it.value()->type() != Value::ItemValueType) + continue; + forwardParameterDeclarations(it.key(), + std::static_pointer_cast<ItemValue>(it.value())->item(), + modules); + } +} + +void DependenciesResolver::forwardParameterDeclarations( + const QualifiedId &moduleName, Item *item, const Item::Modules &modules) +{ + auto it = std::find_if(modules.begin(), modules.end(), [&moduleName] (const Item::Module &m) { + return m.name == moduleName; + }); + if (it != modules.end()) { + item->setPropertyDeclarations(m_loaderState.topLevelProject().parameterDeclarations( + it->item->rootPrototype())); + } else { + for (auto it = item->properties().begin(); it != item->properties().end(); ++it) { + if (it.value()->type() != Value::ItemValueType) + continue; + forwardParameterDeclarations(QualifiedId(moduleName) << it.key(), + std::static_pointer_cast<ItemValue>(it.value())->item(), + modules); + } + } +} + +std::list<DependenciesResolvingState> &DependenciesResolver::stateStack() +{ + QBS_CHECK(m_product.dependenciesContext); + return static_cast<DependenciesContextImpl *>(m_product.dependenciesContext.get())->stateStack; +} + +void DependenciesResolver::checkForModuleNamePrefixCollision( const FullyResolvedDependsItem &dependency) { - if (!product->item) + if (!m_product.item) return; - for (const Item::Module &m : product->item->modules()) { + for (const Item::Module &m : m_product.item->modules()) { if (m.name.length() == dependency.name.length() || m.name.front() != dependency.name.front()) { continue; @@ -1044,15 +1016,12 @@ void DependenciesResolver::Private::checkForModuleNamePrefixCollision( } } -Item::Module DependenciesResolver::Private::createModule( +Item::Module DependenciesResolver::createModule( const FullyResolvedDependsItem &dependency, Item *item, ProductContext *productDep) { Item::Module m; m.item = item; - if (productDep) { - m.productInfo.emplace(productDep->item, productDep->multiplexConfigurationId, - productDep->profileName); - } + m.product = productDep; m.name = dependency.name; m.required = dependency.requiredLocally; m.versionRange = dependency.versionRange; @@ -1104,7 +1073,7 @@ CodeLocation FullyResolvedDependsItem::location() const QString FullyResolvedDependsItem::displayName() const { - return ProductItemMultiplexer::fullProductDisplayName(name.toString(), multiplexId); + return fullProductDisplayName(name.toString(), multiplexId); } bool haveSameSubProject(const ProductContext &p1, const ProductContext &p2) @@ -1141,4 +1110,65 @@ QVariantMap safeToVariant(JSContext *ctx, const JSValue &v) return result; } +DependenciesContextImpl::DependenciesContextImpl(ProductContext &product, LoaderState &loaderState) + : m_product(product) +{ + setSearchPathsForProduct(loaderState); + + // Initialize the state with the direct Depends items of the product item. + DependenciesResolvingState newState{product.item,}; + for (Item * const child : product.item->children()) { + if (child->type() == ItemType::Depends) + newState.pendingDependsItems.push(child); + } + stateStack.push_front(std::move(newState)); + stateStack.front().pendingResolvedDependencies.push( + FullyResolvedDependsItem::makeBaseDependency()); +} + +std::pair<ProductDependency, ProductContext *> DependenciesContextImpl::pendingDependency() const +{ + QBS_CHECK(!stateStack.empty()); + if (stateStack.front().currentDependsItem + && !stateStack.front().currentDependsItem->productTypes.empty()) { + qCDebug(lcLoaderScheduling) << "product" << m_product.displayName() + << "to be delayed because of bulk dependency"; + return {ProductDependency::Bulk, nullptr}; + } + if (!stateStack.front().pendingResolvedDependencies.empty()) { + if (ProductContext * const dep = stateStack.front().pendingResolvedDependencies + .front().product) { + if (m_product.project->topLevelProject->isProductQueuedForHandling(*dep)) { + qCDebug(lcLoaderScheduling) << "product" << m_product.displayName() + << "to be delayed because of dependency " + "to unfinished product" << dep->displayName(); + return {ProductDependency::Single, dep}; + } else { + qCDebug(lcLoaderScheduling) << "product" << m_product.displayName() + << "to be re-scheduled, as dependency " + << dep->displayName() + << "appears to have finished in the meantime"; + return {ProductDependency::None, dep}; + } + } + } + return {ProductDependency::None, nullptr}; +} + +void DependenciesContextImpl::setSearchPathsForProduct(LoaderState &loaderState) +{ + QBS_CHECK(m_product.searchPaths.isEmpty()); + + m_product.searchPaths = loaderState.itemReader().readExtraSearchPaths(m_product.item); + Settings settings(loaderState.parameters().settingsDirectory()); + const QStringList prefsSearchPaths = Preferences(&settings, m_product.profileModuleProperties) + .searchPaths(); + const QStringList ¤tSearchPaths = loaderState.itemReader().allSearchPaths(); + for (const QString &p : prefsSearchPaths) { + if (!currentSearchPaths.contains(p) && FileInfo(p).exists()) + m_product.searchPaths << p; + } +} + +} // namespace } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/dependenciesresolver.h b/src/lib/corelib/loader/dependenciesresolver.h index 0b6d57023..48502adb7 100644 --- a/src/lib/corelib/loader/dependenciesresolver.h +++ b/src/lib/corelib/loader/dependenciesresolver.h @@ -39,53 +39,19 @@ #pragma once -#include <tools/pimpl.h> -#include <tools/set.h> - -#include <QtGlobal> - -#include <functional> - -QT_BEGIN_NAMESPACE -class QString; -QT_END_NAMESPACE - namespace qbs::Internal { class Item; class LoaderState; class ProductContext; -class StoredModuleProviderInfo; enum class Deferral; // Collects the products' dependencies and builds the list of modules from them. // Actual loading of module files is offloaded to ModuleLoader. -class DependenciesResolver -{ -public: - DependenciesResolver(LoaderState &loaderState); - ~DependenciesResolver(); - - // Returns false if the product has unhandled product dependencies and thus needs - // to be deferred, true otherwise. - bool resolveDependencies(ProductContext &product, Deferral deferral); - - void checkDependencyParameterDeclarations(const Item *productItem, - const QString &productName) const; - - void setStoredModuleProviderInfo(const StoredModuleProviderInfo &moduleProviderInfo); - StoredModuleProviderInfo storedModuleProviderInfo() const; - const Set<QString> &tempQbsFiles() const; - - void printProfilingInfo(int indent); - - // Note: This function is never called for regular loading of the base module into a product, - // but only for the special cases of loading the dummy base module into a project - // and temporarily providing a base module for product multiplexing. - Item *loadBaseModule(ProductContext &product, Item *item); +void resolveDependencies(ProductContext &product, Deferral deferral, LoaderState &loaderState); -private: - class Private; - Pimpl<Private> d; -}; +// Note: This function is never called for regular loading of the base module into a product, +// but only for the special cases of loading the dummy base module into a project +// and temporarily providing a base module for product multiplexing. +Item *loadBaseModule(ProductContext &product, Item *item, LoaderState &loaderState); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/groupshandler.cpp b/src/lib/corelib/loader/groupshandler.cpp index a0cab5367..11af94906 100644 --- a/src/lib/corelib/loader/groupshandler.cpp +++ b/src/lib/corelib/loader/groupshandler.cpp @@ -51,66 +51,47 @@ #include <tools/stringconstants.h> namespace qbs::Internal { -class GroupsHandler::Private +class GroupsHandler { public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} + GroupsHandler(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + void run(); + +private: void gatherAssignedProperties(ItemValue *iv, const QualifiedId &prefix, QualifiedIdSet &properties); void markModuleTargetGroups(Item *group, const Item::Module &module); - void moveGroupsFromModuleToProduct(Item *product, Item *productScope, - const Item::Module &module); - void moveGroupsFromModulesToProduct(Item *product, Item *productScope); + void moveGroupsFromModuleToProduct(const Item::Module &module); + void moveGroupsFromModulesToProduct(); void propagateModulesFromParent(Item *group); - void handleGroup(Item *product, Item *group); + void handleGroup(Item *group); void adjustScopesInGroupModuleInstances(Item *groupItem, const Item::Module &module); QualifiedIdSet gatherModulePropertiesSetInGroup(const Item *group); - LoaderState &loaderState; - std::unordered_map<const Item *, QualifiedIdSet> modulePropsSetInGroups; - Set<Item *> disabledGroups; - qint64 elapsedTime = 0; + ProductContext &m_product; + LoaderState &m_loaderState; }; -GroupsHandler::GroupsHandler(LoaderState &loaderState) : d(makePimpl<Private>(loaderState)) {} -GroupsHandler::~GroupsHandler() = default; - -void GroupsHandler::setupGroups(Item *product, Item *productScope) +void setupGroups(ProductContext &product, LoaderState &loaderState) { - AccumulatingTimer timer(d->loaderState.parameters().logElapsedTime() - ? &d->elapsedTime : nullptr); - - d->modulePropsSetInGroups.clear(); - d->disabledGroups.clear(); - d->moveGroupsFromModulesToProduct(product, productScope); - for (Item * const child : product->children()) { - if (child->type() == ItemType::Group) - d->handleGroup(product, child); - } + GroupsHandler(product, loaderState).run(); } -std::unordered_map<const Item *, QualifiedIdSet> GroupsHandler::modulePropertiesSetInGroups() const +void GroupsHandler::run() { - return d->modulePropsSetInGroups; -} + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.groupsSetup : nullptr); -Set<Item *> GroupsHandler::disabledGroups() const -{ - return d->disabledGroups; -} - -void GroupsHandler::printProfilingInfo(int indent) -{ - if (!d->loaderState.parameters().logElapsedTime()) - return; - d->loaderState.logger().qbsLog(LoggerInfo, true) - << QByteArray(indent, ' ') - << Tr::tr("Setting up Groups took %1.") - .arg(elapsedTimeString(d->elapsedTime)); + moveGroupsFromModulesToProduct(); + for (Item * const child : m_product.item->children()) { + if (child->type() == ItemType::Group) + handleGroup(child); + } } -void GroupsHandler::Private::gatherAssignedProperties(ItemValue *iv, const QualifiedId &prefix, +void GroupsHandler::gatherAssignedProperties(ItemValue *iv, const QualifiedId &prefix, QualifiedIdSet &properties) { const Item::PropertyMap &props = iv->item()->properties(); @@ -131,10 +112,10 @@ void GroupsHandler::Private::gatherAssignedProperties(ItemValue *iv, const Quali } } -void GroupsHandler::Private::markModuleTargetGroups(Item *group, const Item::Module &module) +void GroupsHandler::markModuleTargetGroups(Item *group, const Item::Module &module) { QBS_CHECK(group->type() == ItemType::Group); - if (loaderState.evaluator().boolValue(group, StringConstants::filesAreTargetsProperty())) { + if (m_loaderState.evaluator().boolValue(group, StringConstants::filesAreTargetsProperty())) { group->setProperty(StringConstants::modulePropertyInternal(), VariantValue::create(module.name.toString())); } @@ -142,8 +123,7 @@ void GroupsHandler::Private::markModuleTargetGroups(Item *group, const Item::Mod markModuleTargetGroups(child, module); } -void GroupsHandler::Private::moveGroupsFromModuleToProduct(Item *product, Item *productScope, - const Item::Module &module) +void GroupsHandler::moveGroupsFromModuleToProduct(const Item::Module &module) { if (!module.item->isPresentModule()) return; @@ -153,37 +133,36 @@ void GroupsHandler::Private::moveGroupsFromModuleToProduct(Item *product, Item * ++it; continue; } - child->setScope(productScope); - setScopeForDescendants(child, productScope); - Item::addChild(product, child); + child->setScope(m_product.scope); + setScopeForDescendants(child, m_product.scope); + Item::addChild(m_product.item, child); markModuleTargetGroups(child, module); it = module.item->children().erase(it); } } -void GroupsHandler::Private::moveGroupsFromModulesToProduct(Item *product, Item *productScope) +void GroupsHandler::moveGroupsFromModulesToProduct() { - for (const Item::Module &module : product->modules()) - moveGroupsFromModuleToProduct(product, productScope, module); + for (const Item::Module &module : m_product.item->modules()) + moveGroupsFromModuleToProduct(module); } // TODO: I don't completely understand this function, and I suspect we do both too much // and too little here. In particular, I'm not sure why we should even have to do anything // with groups that don't attach properties. // Set aside a day or two at some point to fully grasp all the details and rewrite accordingly. -void GroupsHandler::Private::propagateModulesFromParent(Item *group) +void GroupsHandler::propagateModulesFromParent(Item *group) { QBS_CHECK(group->type() == ItemType::Group); QHash<QualifiedId, Item *> moduleInstancesForGroup; // Step 1: "Instantiate" the product's modules for the group. for (Item::Module m : group->parent()->modules()) { - Item * const targetItem = loaderState.moduleInstantiator() - .retrieveModuleInstanceItem(group, m.name); + Item * const targetItem = retrieveModuleInstanceItem(group, m.name, m_loaderState); QBS_CHECK(targetItem->type() == ItemType::ModuleInstancePlaceholder); targetItem->setPrototype(m.item); - Item * const moduleScope = Item::create(targetItem->pool(), ItemType::Scope); + Item * const moduleScope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); moduleScope->setFile(group->file()); moduleScope->setProperties(m.item->scope()->properties()); // "project", "product", ids moduleScope->setScope(group); @@ -210,7 +189,8 @@ void GroupsHandler::Private::propagateModulesFromParent(Item *group) adaptedModules << depMod; if (depMod.name.front() == module.name.front()) continue; - const ItemValuePtr &modulePrefix = group->itemProperty(depMod.name.front()); + const ItemValuePtr &modulePrefix = group->itemProperty(depMod.name.front(), + m_loaderState.itemPool()); QBS_CHECK(modulePrefix); module.item->setProperty(depMod.name.front(), modulePrefix); } @@ -220,7 +200,7 @@ void GroupsHandler::Private::propagateModulesFromParent(Item *group) const QualifiedIdSet &propsSetInGroup = gatherModulePropertiesSetInGroup(group); if (propsSetInGroup.empty()) return; - modulePropsSetInGroups.insert(std::make_pair(group, propsSetInGroup)); + m_product.modulePropertiesSetInGroups.insert(std::make_pair(group, propsSetInGroup)); // Step 3: Adapt scopes in values. This is potentially necessary if module properties // get assigned on the group level. @@ -228,18 +208,17 @@ void GroupsHandler::Private::propagateModulesFromParent(Item *group) adjustScopesInGroupModuleInstances(group, module); } -void GroupsHandler::Private::handleGroup(Item *product, Item *group) +void GroupsHandler::handleGroup(Item *group) { propagateModulesFromParent(group); - if (!loaderState.evaluator().boolValue(group, StringConstants::conditionProperty())) - disabledGroups << group; + m_loaderState.topLevelProject().checkItemCondition(group, m_loaderState.evaluator()); for (Item * const child : group->children()) { if (child->type() == ItemType::Group) - handleGroup(product, child); + handleGroup(child); } } -void GroupsHandler::Private::adjustScopesInGroupModuleInstances(Item *groupItem, +void GroupsHandler::adjustScopesInGroupModuleInstances(Item *groupItem, const Item::Module &module) { if (!module.item->isPresentModule()) @@ -279,7 +258,7 @@ void GroupsHandler::Private::adjustScopesInGroupModuleInstances(Item *groupItem, continue; if (propValue->scope()) - module.item->setProperty(propName, propValue->clone()); + module.item->setProperty(propName, propValue->clone(m_loaderState.itemPool())); } for (const ValuePtr &prop : module.item->properties()) { @@ -300,7 +279,7 @@ void GroupsHandler::Private::adjustScopesInGroupModuleInstances(Item *groupItem, } } -QualifiedIdSet GroupsHandler::Private::gatherModulePropertiesSetInGroup(const Item *group) +QualifiedIdSet GroupsHandler::gatherModulePropertiesSetInGroup(const Item *group) { QualifiedIdSet propsSetInGroup; const Item::PropertyMap &props = group->properties(); diff --git a/src/lib/corelib/loader/groupshandler.h b/src/lib/corelib/loader/groupshandler.h index ff6afdad9..5de94ce7c 100644 --- a/src/lib/corelib/loader/groupshandler.h +++ b/src/lib/corelib/loader/groupshandler.h @@ -39,16 +39,9 @@ #pragma once -#include <language/qualifiedid.h> -#include <tools/set.h> -#include <tools/pimpl.h> - -#include <unordered_map> -#include <utility> - namespace qbs::Internal { -class Item; class LoaderState; +class ProductContext; // Sets up Group items for the actual resolving stage. Responsibilities: // - Moving Group items located in modules over to the product. @@ -58,21 +51,6 @@ class LoaderState; // - As a side effect of the above point, collecting all properties set on the Group level // to help the ProjectResolver decide which properties need to be re-evaluated at all, // which is an important optimization (see commit 9cd8653eef). -class GroupsHandler -{ -public: - GroupsHandler(LoaderState &loaderState); - ~GroupsHandler(); - - void setupGroups(Item *product, Item *productScope); - std::unordered_map<const Item *, QualifiedIdSet> modulePropertiesSetInGroups() const; - Set<Item *> disabledGroups() const; - - void printProfilingInfo(int indent); - -private: - class Private; - Pimpl<Private> d; -}; +void setupGroups(ProductContext &product, LoaderState &loaderState); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/itemreader.cpp b/src/lib/corelib/loader/itemreader.cpp index 8d9b2ccd4..0638d1af5 100644 --- a/src/lib/corelib/loader/itemreader.cpp +++ b/src/lib/corelib/loader/itemreader.cpp @@ -73,7 +73,8 @@ ItemReader::ItemReader(LoaderState &loaderState) : m_loaderState(loaderState) {} void ItemReader::init() { - m_visitorState = std::make_unique<ItemReaderVisitorState>(m_loaderState.logger()); + m_visitorState = std::make_unique<ItemReaderVisitorState>( + m_loaderState.topLevelProject().itemReaderCache(), m_loaderState.logger()); m_visitorState->setDeprecationWarningMode(m_loaderState.parameters().deprecationWarningMode()); m_projectFilePath = m_loaderState.parameters().projectFilePath(); setSearchPaths(m_loaderState.parameters().searchPaths()); @@ -150,11 +151,6 @@ Item *ItemReader::readFile(const QString &filePath, const CodeLocation &referenc } } -Set<QString> ItemReader::filesRead() const -{ - return m_visitorState->filesRead(); -} - void ItemReader::handlePropertyOptions(Item *optionsItem) { Evaluator &evaluator = m_loaderState.evaluator(); @@ -218,6 +214,14 @@ void ItemReader::handleAllPropertyOptionsItems(Item *item) Item *ItemReader::setupItemFromFile(const QString &filePath, const CodeLocation &referencingLocation) { Item *item = readFile(filePath, referencingLocation); + + // This is technically not needed, because files are only set up once and then served + // from a cache. But it simplifies the checks in item.cpp if we require the locking invariant + // to always hold. + std::unique_ptr<ModuleItemLocker> locker; + if (item->type() == ItemType::Module) + locker = std::make_unique<ModuleItemLocker>(*item); + handleAllPropertyOptionsItems(item); return item; } @@ -226,7 +230,7 @@ Item *ItemReader::wrapInProjectIfNecessary(Item *item) { if (item->type() == ItemType::Project) return item; - Item *prj = Item::create(item->pool(), ItemType::Project); + Item *prj = Item::create(&m_loaderState.itemPool(), ItemType::Project); Item::addChild(prj, item); prj->setFile(item->file()); prj->setLocation(item->location()); diff --git a/src/lib/corelib/loader/itemreader.h b/src/lib/corelib/loader/itemreader.h index 01e9ae18d..e444db16c 100644 --- a/src/lib/corelib/loader/itemreader.h +++ b/src/lib/corelib/loader/itemreader.h @@ -87,8 +87,6 @@ public: Item *wrapInProjectIfNecessary(Item *item); QStringList readExtraSearchPaths(Item *item, bool *wasSet = nullptr); - Set<QString> filesRead() const; - qint64 elapsedTime() const { return m_elapsedTime; } private: diff --git a/src/lib/corelib/loader/itemreaderastvisitor.cpp b/src/lib/corelib/loader/itemreaderastvisitor.cpp index 48bd0af1b..6932e4f91 100644 --- a/src/lib/corelib/loader/itemreaderastvisitor.cpp +++ b/src/lib/corelib/loader/itemreaderastvisitor.cpp @@ -75,6 +75,8 @@ ItemReaderASTVisitor::ItemReaderASTVisitor(ItemReaderVisitorState &visitorState, { } +ItemReaderASTVisitor::~ItemReaderASTVisitor() = default; + bool ItemReaderASTVisitor::visit(AST::UiProgram *uiProgram) { ASTImportsHandler importsHandler(m_visitorState, m_logger, m_file); @@ -135,10 +137,15 @@ bool ItemReaderASTVisitor::visit(AST::UiObjectDefinition *ast) item->m_type = itemType; - if (m_item) + if (m_item) { Item::addChild(m_item, item); // Add this item to the children of the parent item. - else + } else { m_item = item; // This is the root item. + if (itemType == ItemType::Module) { + QBS_CHECK(!m_moduleItemLocker); + m_moduleItemLocker = std::make_unique<ModuleItemLocker>(*m_item); + } + } if (ast->initializer) { Item *mdi = m_visitorState.mostDerivingItem(); @@ -153,7 +160,7 @@ bool ItemReaderASTVisitor::visit(AST::UiObjectDefinition *ast) m_visitorState.setMostDerivingItem(mdi); } - ASTPropertiesItemHandler(item).handlePropertiesItems(); + ASTPropertiesItemHandler(item, *m_itemPool).handlePropertiesItems(); // Inheritance resolving, part 2 (depends on alternatives having been set up). if (baseItem) { @@ -241,7 +248,7 @@ bool ItemReaderASTVisitor::visit(AST::UiScriptBinding *ast) throw ErrorInfo(Tr::tr("id: must be followed by identifier")); m_item->m_id = idExp->name.toString(); m_file->ensureIdScope(m_itemPool); - ItemValueConstPtr existingId = m_file->idScope()->itemProperty(m_item->id()); + ItemValueConstPtr existingId = m_file->idScope()->itemProperty(m_item->id(), *m_itemPool); if (existingId) { ErrorInfo e(Tr::tr("The id '%1' is not unique.").arg(m_item->id())); e.append(Tr::tr("First occurrence is here."), existingId->item()->location()); @@ -335,6 +342,9 @@ void ItemReaderASTVisitor::inheritItem(Item *dst, const Item *src) dst->setPropertyDeclaration(pd.name(), pd); } + std::unique_ptr<ModuleItemLocker> locker; + if (src->type() == ItemType::Module) + locker = std::make_unique<ModuleItemLocker>(*src); for (auto it = src->properties().constBegin(); it != src->properties().constEnd(); ++it) { ValuePtr &v = dst->m_properties[it.key()]; if (!v) { diff --git a/src/lib/corelib/loader/itemreaderastvisitor.h b/src/lib/corelib/loader/itemreaderastvisitor.h index a102b2821..c47501f2f 100644 --- a/src/lib/corelib/loader/itemreaderastvisitor.h +++ b/src/lib/corelib/loader/itemreaderastvisitor.h @@ -49,6 +49,8 @@ #include <QtCore/qhash.h> #include <QtCore/qstringlist.h> +#include <memory.h> + namespace qbs { class CodeLocation; @@ -56,12 +58,15 @@ namespace Internal { class Item; class ItemPool; class ItemReaderVisitorState; +class ModuleItemLocker; class ItemReaderASTVisitor : public QbsQmlJS::AST::Visitor { public: ItemReaderASTVisitor(ItemReaderVisitorState &visitorState, FileContextPtr file, ItemPool *itemPool, Logger &logger); + ~ItemReaderASTVisitor(); + void checkItemTypes() { doCheckItemTypes(rootItem()); } Item *rootItem() const { return m_item; } @@ -88,6 +93,7 @@ private: Logger &m_logger; QHash<QStringList, QString> m_typeNameToFile; Item *m_item = nullptr; + std::unique_ptr<ModuleItemLocker> m_moduleItemLocker; ItemType m_instanceItemType = ItemType::ModuleInstancePlaceholder; }; diff --git a/src/lib/corelib/loader/itemreadervisitorstate.cpp b/src/lib/corelib/loader/itemreadervisitorstate.cpp index 57484043a..6ea23c021 100644 --- a/src/lib/corelib/loader/itemreadervisitorstate.cpp +++ b/src/lib/corelib/loader/itemreadervisitorstate.cpp @@ -39,6 +39,7 @@ #include "itemreadervisitorstate.h" #include "itemreaderastvisitor.h" +#include "loaderutils.h" #include <language/asttools.h> #include <language/filecontext.h> @@ -47,89 +48,35 @@ #include <parser/qmljslexer_p.h> #include <parser/qmljsparser_p.h> #include <tools/error.h> +#include <tools/stringconstants.h> -#include <QtCore/qshareddata.h> +#include <QtCore/qdiriterator.h> #include <QtCore/qfile.h> #include <QtCore/qfileinfo.h> -#include <QtCore/qshareddata.h> #include <QtCore/qtextstream.h> namespace qbs { namespace Internal { -class ASTCacheValueData : public QSharedData +ItemReaderVisitorState::ItemReaderVisitorState(ItemReaderCache &cache, Logger &logger) + : m_cache(cache), m_logger(logger) { - Q_DISABLE_COPY(ASTCacheValueData) -public: - ASTCacheValueData() - : ast(nullptr) - , processing(false) - { - } - - QString code; - QbsQmlJS::Engine engine; - QbsQmlJS::AST::UiProgram *ast; - bool processing; -}; - -class ASTCacheValue -{ -public: - ASTCacheValue() - : d(new ASTCacheValueData) - { - } - - ASTCacheValue(const ASTCacheValue &other) = default; - - void setProcessingFlag(bool b) { d->processing = b; } - bool isProcessing() const { return d->processing; } - - void setCode(const QString &code) { d->code = code; } - QString code() const { return d->code; } - - QbsQmlJS::Engine *engine() const { return &d->engine; } - - void setAst(QbsQmlJS::AST::UiProgram *ast) { d->ast = ast; } - QbsQmlJS::AST::UiProgram *ast() const { return d->ast; } - bool isValid() const { return d->ast; } - -private: - QExplicitlySharedDataPointer<ASTCacheValueData> d; -}; - -class ItemReaderVisitorState::ASTCache : public std::unordered_map<QString, ASTCacheValue> {}; - - -ItemReaderVisitorState::ItemReaderVisitorState(Logger &logger) - : m_logger(logger) - , m_astCache(std::make_unique<ASTCache>()) -{ - } -ItemReaderVisitorState::~ItemReaderVisitorState() = default; - Item *ItemReaderVisitorState::readFile(const QString &filePath, const QStringList &searchPaths, ItemPool *itemPool) { - ASTCacheValue &cacheValue = (*m_astCache)[filePath]; - if (cacheValue.isValid()) { - if (Q_UNLIKELY(cacheValue.isProcessing())) - throw ErrorInfo(Tr::tr("Loop detected when importing '%1'.").arg(filePath)); - } else { + const auto setupCacheEntry = [&](ItemReaderCache::AstCacheEntry &entry) { QFile file(filePath); if (Q_UNLIKELY(!file.open(QFile::ReadOnly))) throw ErrorInfo(Tr::tr("Cannot open '%1'.").arg(filePath)); - m_filesRead.insert(filePath); QTextStream stream(&file); setupDefaultCodec(stream); const QString &code = stream.readAll(); - QbsQmlJS::Lexer lexer(cacheValue.engine()); + QbsQmlJS::Lexer lexer(&entry.engine); lexer.setCode(code, 1); - QbsQmlJS::Parser parser(cacheValue.engine()); + QbsQmlJS::Parser parser(&entry.engine); file.close(); if (!parser.parse()) { @@ -142,42 +89,49 @@ Item *ItemReaderVisitorState::readFile(const QString &filePath, const QStringLis } } - cacheValue.setCode(code); - cacheValue.setAst(parser.ast()); - } + entry.code = code; + entry.ast = parser.ast(); + }; + ItemReaderCache::AstCacheEntry &cacheEntry = m_cache.retrieveOrSetupCacheEntry( + filePath, setupCacheEntry); const FileContextPtr file = FileContext::create(); file->setFilePath(QFileInfo(filePath).absoluteFilePath()); - file->setContent(cacheValue.code()); + file->setContent(cacheEntry.code); file->setSearchPaths(searchPaths); ItemReaderASTVisitor astVisitor(*this, file, itemPool, m_logger); { class ProcessingFlagManager { public: - ProcessingFlagManager(ASTCacheValue &v) : m_cacheValue(v) { v.setProcessingFlag(true); } - ~ProcessingFlagManager() { m_cacheValue.setProcessingFlag(false); } + ProcessingFlagManager(ItemReaderCache::AstCacheEntry &e, const QString &filePath) + : m_cacheEntry(e) + { + if (!e.addProcessingThread()) + throw ErrorInfo(Tr::tr("Loop detected when importing '%1'.").arg(filePath)); + } + ~ProcessingFlagManager() { m_cacheEntry.removeProcessingThread(); } + private: - ASTCacheValue &m_cacheValue; - } processingFlagManager(cacheValue); - cacheValue.ast()->accept(&astVisitor); + ItemReaderCache::AstCacheEntry &m_cacheEntry; + } processingFlagManager(cacheEntry, filePath); + cacheEntry.ast->accept(&astVisitor); } astVisitor.checkItemTypes(); return astVisitor.rootItem(); } -void ItemReaderVisitorState::cacheDirectoryEntries(const QString &dirPath, const QStringList &entries) -{ - m_directoryEntries.insert(dirPath, entries); -} - -bool ItemReaderVisitorState::findDirectoryEntries(const QString &dirPath, QStringList *entries) const +void ItemReaderVisitorState::findDirectoryEntries(const QString &dirPath, QStringList *entries) const { - const auto it = m_directoryEntries.constFind(dirPath); - if (it == m_directoryEntries.constEnd()) - return false; - *entries = it.value(); - return true; + *entries = m_cache.retrieveOrSetDirectoryEntries(dirPath, [&dirPath] { + QStringList fileNames; + QDirIterator dirIter(dirPath, StringConstants::qbsFileWildcards()); + while (dirIter.hasNext()) { + dirIter.next(); + fileNames << dirIter.fileName(); + } + return fileNames; + }); } Item *ItemReaderVisitorState::mostDerivingItem() const diff --git a/src/lib/corelib/loader/itemreadervisitorstate.h b/src/lib/corelib/loader/itemreadervisitorstate.h index dc22cfb42..cbf1966b6 100644 --- a/src/lib/corelib/loader/itemreadervisitorstate.h +++ b/src/lib/corelib/loader/itemreadervisitorstate.h @@ -50,21 +50,19 @@ namespace qbs { namespace Internal { class Item; class ItemPool; +class ItemReaderCache; class Logger; class ItemReaderVisitorState { public: - ItemReaderVisitorState(Logger &logger); - ~ItemReaderVisitorState(); + ItemReaderVisitorState(ItemReaderCache &cache, Logger &logger); Logger &logger() { return m_logger; } - Set<QString> filesRead() const { return m_filesRead; } Item *readFile(const QString &filePath, const QStringList &searchPaths, ItemPool *itemPool); - void cacheDirectoryEntries(const QString &dirPath, const QStringList &entries); - bool findDirectoryEntries(const QString &dirPath, QStringList *entries) const; + void findDirectoryEntries(const QString &dirPath, QStringList *entries) const; Item *mostDerivingItem() const; void setMostDerivingItem(Item *item); @@ -74,13 +72,9 @@ public: private: DeprecationWarningMode m_deprecationWarningMode = defaultDeprecationWarningMode(); + ItemReaderCache &m_cache; Logger &m_logger; - Set<QString> m_filesRead; - QHash<QString, QStringList> m_directoryEntries; Item *m_mostDerivingItem = nullptr; - - class ASTCache; - const std::unique_ptr<ASTCache> m_astCache; }; } // namespace Internal diff --git a/src/lib/corelib/loader/loaderutils.cpp b/src/lib/corelib/loader/loaderutils.cpp index 98ac10a92..a0949df91 100644 --- a/src/lib/corelib/loader/loaderutils.cpp +++ b/src/lib/corelib/loader/loaderutils.cpp @@ -39,26 +39,35 @@ #include "loaderutils.h" -#include "dependenciesresolver.h" #include "itemreader.h" -#include "localprofiles.h" -#include "moduleinstantiator.h" -#include "modulepropertymerger.h" -#include "probesresolver.h" -#include "productitemmultiplexer.h" #include <language/evaluator.h> -#include <language/item.h> +#include <language/filecontext.h> +#include <language/itempool.h> #include <language/language.h> +#include <language/resolvedfilecontext.h> +#include <language/scriptengine.h> #include <language/value.h> #include <logging/categories.h> #include <logging/translator.h> +#include <tools/fileinfo.h> #include <tools/progressobserver.h> #include <tools/setupprojectparameters.h> #include <tools/stringconstants.h> namespace qbs::Internal { +QString fullProductDisplayName(const QString &name, const QString &multiplexId) +{ + static const auto multiplexIdToString =[](const QString &id) { + return QString::fromUtf8(QByteArray::fromBase64(id.toUtf8())); + }; + QString result = name; + if (!multiplexId.isEmpty()) + result.append(QLatin1Char(' ')).append(multiplexIdToString(multiplexId)); + return result; +} + void mergeParameters(QVariantMap &dst, const QVariantMap &src) { for (auto it = src.begin(); it != src.end(); ++it) { @@ -73,15 +82,6 @@ void mergeParameters(QVariantMap &dst, const QVariantMap &src) } } -ShadowProductInfo getShadowProductInfo(const ProductContext &product) -{ - const bool isShadowProduct = product.name.startsWith(StringConstants::shadowProductPrefix()); - return std::make_pair(isShadowProduct, - isShadowProduct - ? product.name.mid(StringConstants::shadowProductPrefix().size()) - : QString()); -} - void adjustParametersScopes(Item *item, Item *scope) { if (item->type() == ItemType::ModuleParameters) { @@ -102,15 +102,15 @@ QString ProductContext::uniqueName() const QString ProductContext::displayName() const { - return ProductItemMultiplexer::fullProductDisplayName(name, multiplexConfigurationId); + return fullProductDisplayName(name, multiplexConfigurationId); } void ProductContext::handleError(const ErrorInfo &error) { - const bool alreadyHadError = info.delayedError.hasError(); + const bool alreadyHadError = delayedError.hasError(); if (!alreadyHadError) { - info.delayedError.append(Tr::tr("Error while handling product '%1':") - .arg(name), item->location()); + delayedError.append(Tr::tr("Error while handling product '%1':") + .arg(name), item->location()); } if (error.isInternalError()) { if (alreadyHadError) { @@ -121,76 +121,751 @@ void ProductContext::handleError(const ErrorInfo &error) } const auto errorItems = error.items(); for (const ErrorItem &ei : errorItems) - info.delayedError.append(ei.description(), ei.codeLocation()); - project->topLevelProject->productInfos[item] = info; - project->topLevelProject->disabledItems << item; - project->topLevelProject->erroneousProducts.insert(name); + delayedError.append(ei.description(), ei.codeLocation()); + project->topLevelProject->addDisabledItem(item); } +TopLevelProjectContext::~TopLevelProjectContext() { qDeleteAll(m_projects); } + bool TopLevelProjectContext::checkItemCondition(Item *item, Evaluator &evaluator) { if (evaluator.boolValue(item, StringConstants::conditionProperty())) return true; - disabledItems += item; + addDisabledItem(item); return false; } -void TopLevelProjectContext::checkCancelation(const SetupProjectParameters ¶meters) +void TopLevelProjectContext::checkCancelation() +{ + if (m_progressObserver && m_progressObserver->canceled()) + m_canceled = true; + if (m_canceled) + throw CancelException(); +} + +QString TopLevelProjectContext::sourceCodeForEvaluation(const JSSourceValueConstPtr &value) +{ + std::lock_guard lock(m_sourceCode.mutex); + QString &code = m_sourceCode.data[value->sourceCode()]; + if (!code.isNull()) + return code; + code = value->sourceCodeForEvaluation(); + return code; +} + +ScriptFunctionPtr TopLevelProjectContext::scriptFunctionValue(Item *item, const QString &name) +{ + const JSSourceValuePtr value = item->sourceProperty(name); + QBS_CHECK(value); + std::lock_guard lock(m_scriptFunctionMap.mutex); + ScriptFunctionPtr &script = m_scriptFunctionMap.data[value->location()]; + if (!script.get()) { + script = ScriptFunction::create(); + const PropertyDeclaration decl = item->propertyDeclaration(name); + script->sourceCode = sourceCodeAsFunction(value, decl); + script->location = value->location(); + script->fileContext = resolvedFileContext(value->file()); + } + return script; +} + +QString TopLevelProjectContext::sourceCodeAsFunction(const JSSourceValueConstPtr &value, + const PropertyDeclaration &decl) +{ + std::lock_guard lock(m_scriptFunctions.mutex); + QString &scriptFunction = m_scriptFunctions.data[std::make_pair(value->sourceCode(), + decl.functionArgumentNames())]; + if (!scriptFunction.isNull()) + return scriptFunction; + const QString args = decl.functionArgumentNames().join(QLatin1Char(',')); + if (value->hasFunctionForm()) { + // Insert the argument list. + scriptFunction = value->sourceCodeForEvaluation(); + scriptFunction.insert(10, args); + // Remove the function application "()" that has been + // added in ItemReaderASTVisitor::visitStatement. + scriptFunction.chop(2); + } else { + scriptFunction = QLatin1String("(function(") + args + QLatin1String("){return ") + + value->sourceCode().toString() + QLatin1String(";})"); + } + return scriptFunction; +} + +const ResolvedFileContextPtr &TopLevelProjectContext::resolvedFileContext( + const FileContextConstPtr &ctx) +{ + ResolvedFileContextPtr &result = m_fileContextMap[ctx]; + if (!result) + result = ResolvedFileContext::create(*ctx); + return result; +} + +void TopLevelProjectContext::removeProductToHandle(const ProductContext &product) +{ + std::unique_lock lock(m_productsToHandle.mutex); + m_productsToHandle.data.remove(&product); +} + +bool TopLevelProjectContext::isProductQueuedForHandling(const ProductContext &product) const +{ + std::shared_lock lock(m_productsToHandle.mutex); + return m_productsToHandle.data.contains(&product); +} + +void TopLevelProjectContext::addDisabledItem(Item *item) +{ + std::unique_lock lock(m_disabledItems.mutex); + m_disabledItems.data << item; +} + +bool TopLevelProjectContext::isDisabledItem(Item *item) const +{ + std::shared_lock lock(m_disabledItems.mutex); + return m_disabledItems.data.contains(item); +} + +void TopLevelProjectContext::setProgressObserver(ProgressObserver *observer) +{ + m_progressObserver = observer; +} + +ProgressObserver *TopLevelProjectContext::progressObserver() const { return m_progressObserver; } + +void TopLevelProjectContext::addQueuedError(const ErrorInfo &error) +{ + std::lock_guard lock(m_queuedErrors.mutex); + m_queuedErrors.data << error; +} + +void TopLevelProjectContext::addProfileConfig(const QString &profileName, + const QVariantMap &profileConfig) +{ + m_profileConfigs.insert(profileName, profileConfig); +} + +std::optional<QVariantMap> TopLevelProjectContext::profileConfig(const QString &profileName) const +{ + const auto it = m_profileConfigs.constFind(profileName); + if (it == m_profileConfigs.constEnd()) + return {}; + return it.value().toMap(); +} + +void TopLevelProjectContext::addProjectLevelProbe(const ProbeConstPtr &probe) +{ + m_probesInfo.projectLevelProbes << probe; +} + +const std::vector<ProbeConstPtr> TopLevelProjectContext::projectLevelProbes() const +{ + return m_probesInfo.projectLevelProbes; +} + +void TopLevelProjectContext::addProduct(ProductContext &product) +{ + m_productsByName.insert({product.name, &product}); +} + +void TopLevelProjectContext::addProductByType(ProductContext &product, const FileTags &tags) +{ + std::unique_lock lock(m_productsByType.mutex); + for (const FileTag &tag : tags) + m_productsByType.data.insert({tag, &product}); +} + +ProductContext *TopLevelProjectContext::productWithNameAndConstraint( + const QString &name, const std::function<bool (ProductContext &)> &constraint) +{ + const auto candidates = m_productsByName.equal_range(name); + for (auto it = candidates.first; it != candidates.second; ++it) { + ProductContext * const candidate = it->second; + if (constraint(*candidate)) + return candidate; + } + return nullptr; +} + +std::vector<ProductContext *> TopLevelProjectContext::productsWithNameAndConstraint( + const QString &name, const std::function<bool (ProductContext &)> &constraint) +{ + std::vector<ProductContext *> result; + const auto candidates = m_productsByName.equal_range(name); + for (auto it = candidates.first; it != candidates.second; ++it) { + ProductContext * const candidate = it->second; + if (constraint(*candidate)) + result << candidate; + } + return result; +} + +std::vector<ProductContext *> TopLevelProjectContext::productsWithTypeAndConstraint( + const FileTags &tags, const std::function<bool (ProductContext &)> &constraint) +{ + std::shared_lock lock(m_productsByType.mutex); + std::vector<ProductContext *> matchingProducts; + for (const FileTag &typeTag : tags) { + const auto range = m_productsByType.data.equal_range(typeTag); + for (auto it = range.first; it != range.second; ++it) { + if (constraint(*it->second)) + matchingProducts.push_back(it->second); + } + } + return matchingProducts; +} + +std::vector<std::pair<ProductContext *, CodeLocation>> +TopLevelProjectContext::finishedProductsWithBulkDependency(const FileTag &tag) const +{ + return m_reverseBulkDependencies.value(tag); +} + +void TopLevelProjectContext::registerBulkDependencies(ProductContext &product) +{ + for (const auto &tagAndLoc : product.bulkDependencies) { + for (const FileTag &tag : tagAndLoc.first) + m_reverseBulkDependencies[tag].emplace_back(&product, tagAndLoc.second); + } +} + +void TopLevelProjectContext::addProjectNameUsedInOverrides(const QString &name) +{ + m_projectNamesUsedInOverrides << name; +} + +const Set<QString> &TopLevelProjectContext::projectNamesUsedInOverrides() const +{ + return m_projectNamesUsedInOverrides; +} + +void TopLevelProjectContext::addProductNameUsedInOverrides(const QString &name) +{ + m_productNamesUsedInOverrides << name; +} + +const Set<QString> &TopLevelProjectContext::productNamesUsedInOverrides() const +{ + return m_productNamesUsedInOverrides; +} + +void TopLevelProjectContext::addMultiplexConfiguration(const QString &id, const QVariantMap &config) { - if (progressObserver && progressObserver->canceled()) { - throw ErrorInfo(Tr::tr("Project resolving canceled for configuration %1.") - .arg(TopLevelProject::deriveId( - parameters.finalBuildConfigurationTree()))); + m_multiplexConfigsById.insert(std::make_pair(id, config)); +} + +QVariantMap TopLevelProjectContext::multiplexConfiguration(const QString &id) const +{ + if (id.isEmpty()) + return {}; + const auto it = m_multiplexConfigsById.find(id); + QBS_CHECK(it != m_multiplexConfigsById.end() && !it->second.isEmpty()); + return it->second; +} + +std::lock_guard<std::mutex> TopLevelProjectContext::moduleProvidersCacheLock() +{ + return std::lock_guard<std::mutex>(m_moduleProvidersCacheMutex); +} + +void TopLevelProjectContext::setModuleProvidersCache(const ModuleProvidersCache &cache) +{ + m_moduleProvidersCache = cache; +} + +ModuleProviderInfo *TopLevelProjectContext::moduleProvider(const ModuleProvidersCacheKey &key) +{ + if (const auto it = m_moduleProvidersCache.find(key); it != m_moduleProvidersCache.end()) + return &(*it); + return nullptr; +} + +ModuleProviderInfo &TopLevelProjectContext::addModuleProvider(const ModuleProvidersCacheKey &key, + const ModuleProviderInfo &provider) +{ + return m_moduleProvidersCache[key] = provider; +} + +void TopLevelProjectContext::addParameterDeclarations(const Item *moduleProto, + const Item::PropertyDeclarationMap &decls) +{ + std::unique_lock lock(m_parameterDeclarations.mutex); + m_parameterDeclarations.data.insert({moduleProto, decls}); +} + +Item::PropertyDeclarationMap TopLevelProjectContext::parameterDeclarations(Item *moduleProto) const +{ + std::shared_lock lock(m_parameterDeclarations.mutex); + if (const auto it = m_parameterDeclarations.data.find(moduleProto); + it != m_parameterDeclarations.data.end()) { + return it->second; } + return {}; +} + +QString TopLevelProjectContext::findModuleDirectory( + const QualifiedId &module, const QString &searchPath, + const std::function<QString()> &findOnDisk) +{ + std::lock_guard lock(m_modulePathCache.mutex); + auto &path = m_modulePathCache.data[{searchPath, module}]; + if (!path) + path = findOnDisk(); + return *path; +} + +QStringList TopLevelProjectContext::getModuleFilesForDirectory( + const QString &dir, const std::function<QStringList ()> &findOnDisk) +{ + std::lock_guard lock(m_moduleFilesPerDirectory.mutex); + auto &list = m_moduleFilesPerDirectory.data[dir]; + if (!list) + list = findOnDisk(); + return *list; +} + +void TopLevelProjectContext::removeModuleFileFromDirectoryCache(const QString &filePath) +{ + std::lock_guard lock(m_moduleFilesPerDirectory.mutex); + const auto it = m_moduleFilesPerDirectory.data.find(FileInfo::path(filePath)); + QBS_CHECK(it != m_moduleFilesPerDirectory.data.end()); + it->second->removeOne(filePath); +} + +void TopLevelProjectContext::addUnknownProfilePropertyError(const Item *moduleProto, + const ErrorInfo &error) +{ + std::unique_lock lock(m_unknownProfilePropertyErrors.mutex); + m_unknownProfilePropertyErrors.data[moduleProto].push_back(error); +} + +const std::vector<ErrorInfo> &TopLevelProjectContext::unknownProfilePropertyErrors( + const Item *moduleProto) const +{ + std::shared_lock lock(m_unknownProfilePropertyErrors.mutex); + if (const auto it = m_unknownProfilePropertyErrors.data.find(moduleProto); + it != m_unknownProfilePropertyErrors.data.end()) { + return it->second; + } + static const std::vector<ErrorInfo> empty; + return empty; +} + +Item *TopLevelProjectContext::getModulePrototype(const QString &filePath, const QString &profile, + const std::function<Item *()> &produce) +{ + std::lock_guard lock(m_modulePrototypes.mutex); + auto &prototypeList = m_modulePrototypes.data[filePath]; + for (const auto &prototype : prototypeList) { + if (prototype.second == profile) + return prototype.first; + } + Item * const module = produce(); + if (module) + prototypeList.emplace_back(module, profile); + return module; +} + +void TopLevelProjectContext::addLocalProfile(const QString &name, const QVariantMap &values, + const CodeLocation &location) +{ + if (m_localProfiles.contains(name)) + throw ErrorInfo(Tr::tr("Local profile '%1' redefined.").arg(name), location); + m_localProfiles.insert(name, values); +} + +void TopLevelProjectContext::checkForLocalProfileAsTopLevelProfile(const QString &topLevelProfile) +{ + for (auto it = m_localProfiles.cbegin(); it != m_localProfiles.cend(); ++it) { + if (it.key() != topLevelProfile) + continue; + + // This covers the edge case that a locally defined profile was specified as the + // top-level profile, in which case we must invalidate the qbs module prototype that was + // created in early setup before local profiles were handled. + QBS_CHECK(m_modulePrototypes.data.size() == 1); + m_modulePrototypes.data.clear(); + break; + } +} + +std::lock_guard<std::mutex> TopLevelProjectContext::probesCacheLock() +{ + return std::lock_guard<std::mutex>(m_probesMutex); +} + +void TopLevelProjectContext::setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes) +{ + for (const ProbeConstPtr& probe : oldProbes) + m_probesInfo.oldProjectProbes[probe->globalId()] << probe; +} + +ProbeConstPtr TopLevelProjectContext::findOldProjectProbe(const QString &id, + const ProbeFilter &filter) const +{ + for (const ProbeConstPtr &oldProbe : m_probesInfo.oldProjectProbes.value(id)) { + if (filter(oldProbe)) + return oldProbe; + } + return {}; +} + +void TopLevelProjectContext::setOldProductProbes( + const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes) +{ + m_probesInfo.oldProductProbes = oldProbes; +} + +ProbeConstPtr TopLevelProjectContext::findOldProductProbe(const QString &productName, + const ProbeFilter &filter) const +{ + for (const ProbeConstPtr &oldProbe : m_probesInfo.oldProductProbes.value(productName)) { + if (filter(oldProbe)) + return oldProbe; + } + return {}; +} + +void TopLevelProjectContext::addNewlyResolvedProbe(const ProbeConstPtr &probe) +{ + m_probesInfo.currentProbes[probe->location()] << probe; +} + +ProbeConstPtr TopLevelProjectContext::findCurrentProbe(const CodeLocation &location, + const ProbeFilter &filter) const +{ + for (const ProbeConstPtr &probe : m_probesInfo.currentProbes.value(location)) { + if (filter(probe)) + return probe; + } + return {}; +} + +void TopLevelProjectContext::collectDataFromEngine(const ScriptEngine &engine) +{ + const auto project = dynamic_cast<TopLevelProject *>(m_projects.front()->project.get()); + QBS_CHECK(project); + project->canonicalFilePathResults.insert(engine.canonicalFilePathResults()); + project->fileExistsResults.insert(engine.fileExistsResults()); + project->directoryEntriesResults.insert(engine.directoryEntriesResults()); + project->fileLastModifiedResults.insert(engine.fileLastModifiedResults()); + project->environment.insert(engine.environment()); + project->buildSystemFiles.unite(engine.imports()); +} + +ItemPool &TopLevelProjectContext::createItemPool() +{ + m_itemPools.push_back(std::make_unique<ItemPool>()); + return *m_itemPools.back(); } class LoaderState::Private { public: - Private(LoaderState &q, const SetupProjectParameters ¶meters, ItemPool &itemPool, - Evaluator &evaluator, Logger &logger) - : parameters(parameters), itemPool(itemPool), evaluator(evaluator), logger(logger), - itemReader(q), probesResolver(q), propertyMerger(q), localProfiles(q), - moduleInstantiator(q), dependenciesResolver(q), - multiplexer(q, [this](Item *productItem) { - return moduleInstantiator.retrieveQbsItem(productItem); - }) - {} + Private(LoaderState &q, const SetupProjectParameters ¶meters, + TopLevelProjectContext &topLevelProject, ItemPool &itemPool, ScriptEngine &engine, + Logger &&logger) + : parameters(parameters), topLevelProject(topLevelProject), itemPool(itemPool), + logger(std::move(logger)), itemReader(q), evaluator(&engine) + { + this->logger.clearWarnings(); + this->logger.storeWarnings(); + } const SetupProjectParameters ¶meters; + TopLevelProjectContext &topLevelProject; ItemPool &itemPool; - Evaluator &evaluator; - Logger &logger; - TopLevelProjectContext topLevelProject; + Logger logger; ItemReader itemReader; - ProbesResolver probesResolver; - ModulePropertyMerger propertyMerger; - LocalProfiles localProfiles; - ModuleInstantiator moduleInstantiator; - DependenciesResolver dependenciesResolver; - ProductItemMultiplexer multiplexer; + Evaluator evaluator; }; -LoaderState::LoaderState(const SetupProjectParameters ¶meters, ItemPool &itemPool, - Evaluator &evaluator, Logger &logger) - : d(makePimpl<Private>(*this, parameters, itemPool, evaluator, logger)) +LoaderState::LoaderState(const SetupProjectParameters ¶meters, + TopLevelProjectContext &topLevelProject, ItemPool &itemPool, + ScriptEngine &engine, Logger logger) + : d(makePimpl<Private>(*this, parameters, topLevelProject, itemPool, engine, std::move(logger))) { d->itemReader.init(); } LoaderState::~LoaderState() = default; const SetupProjectParameters &LoaderState::parameters() const { return d->parameters; } -DependenciesResolver &LoaderState::dependenciesResolver() { return d->dependenciesResolver; } ItemPool &LoaderState::itemPool() { return d->itemPool; } Evaluator &LoaderState::evaluator() { return d->evaluator; } Logger &LoaderState::logger() { return d->logger; } -ModuleInstantiator &LoaderState::moduleInstantiator() { return d->moduleInstantiator; } -ProductItemMultiplexer &LoaderState::multiplexer() { return d->multiplexer; } ItemReader &LoaderState::itemReader() { return d->itemReader; } -LocalProfiles &LoaderState::localProfiles() { return d->localProfiles; } -ProbesResolver &LoaderState::probesResolver() { return d->probesResolver; } -ModulePropertyMerger &LoaderState::propertyMerger() { return d->propertyMerger; } TopLevelProjectContext &LoaderState::topLevelProject() { return d->topLevelProject; } +static QString verbatimValue(LoaderState &state, const ValueConstPtr &value) +{ + QString result; + if (value && value->type() == Value::JSSourceValueType) { + const JSSourceValueConstPtr sourceValue = std::static_pointer_cast<const JSSourceValue>( + value); + result = state.topLevelProject().sourceCodeForEvaluation(sourceValue); + } + return result; +} + +static void resolveRuleArtifactBinding( + LoaderState &state, const RuleArtifactPtr &ruleArtifact, Item *item, + const QStringList &namePrefix, QualifiedIdSet *seenBindings) +{ + for (auto it = item->properties().constBegin(); it != item->properties().constEnd(); ++it) { + const QStringList name = QStringList(namePrefix) << it.key(); + if (it.value()->type() == Value::ItemValueType) { + resolveRuleArtifactBinding(state, ruleArtifact, + std::static_pointer_cast<ItemValue>(it.value())->item(), + name, seenBindings); + } else if (it.value()->type() == Value::JSSourceValueType) { + const auto insertResult = seenBindings->insert(name); + if (!insertResult.second) + continue; + JSSourceValuePtr sourceValue = std::static_pointer_cast<JSSourceValue>(it.value()); + RuleArtifact::Binding rab; + rab.name = name; + rab.code = state.topLevelProject().sourceCodeForEvaluation(sourceValue); + rab.location = sourceValue->location(); + ruleArtifact->bindings.push_back(rab); + } else { + QBS_ASSERT(!"unexpected value type", continue); + } + } +} + +static void resolveRuleArtifact(LoaderState &state, const RulePtr &rule, Item *item) +{ + RuleArtifactPtr artifact = RuleArtifact::create(); + rule->artifacts.push_back(artifact); + artifact->location = item->location(); + + if (const auto sourceProperty = item->sourceProperty(StringConstants::filePathProperty())) + artifact->filePathLocation = sourceProperty->location(); + + artifact->filePath = verbatimValue(state, item->property(StringConstants::filePathProperty())); + artifact->fileTags = state.evaluator().fileTagsValue(item, StringConstants::fileTagsProperty()); + artifact->alwaysUpdated = state.evaluator().boolValue( + item, StringConstants::alwaysUpdatedProperty()); + + QualifiedIdSet seenBindings; + for (Item *obj = item; obj; obj = obj->prototype()) { + for (QMap<QString, ValuePtr>::const_iterator it = obj->properties().constBegin(); + it != obj->properties().constEnd(); ++it) + { + if (it.value()->type() != Value::ItemValueType) + continue; + resolveRuleArtifactBinding( + state, artifact, std::static_pointer_cast<ItemValue>(it.value())->item(), + QStringList(it.key()), &seenBindings); + } + } +} + +void resolveRule(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext) +{ + Evaluator &evaluator = state.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) + return; + + RulePtr rule = Rule::create(); + + // read artifacts + bool hasArtifactChildren = false; + for (Item * const child : item->children()) { + if (Q_UNLIKELY(child->type() != ItemType::Artifact)) { + throw ErrorInfo(Tr::tr("'Rule' can only have children of type 'Artifact'."), + child->location()); + } + hasArtifactChildren = true; + resolveRuleArtifact(state, rule, child); + } + + rule->name = evaluator.stringValue(item, StringConstants::nameProperty()); + rule->prepareScript.initialize(state.topLevelProject().scriptFunctionValue( + item, StringConstants::prepareProperty())); + rule->outputArtifactsScript.initialize(state.topLevelProject().scriptFunctionValue( + item, StringConstants::outputArtifactsProperty())); + rule->outputFileTags = evaluator.fileTagsValue(item, StringConstants::outputFileTagsProperty()); + if (rule->outputArtifactsScript.isValid()) { + if (hasArtifactChildren) + throw ErrorInfo(Tr::tr("The Rule.outputArtifacts script is not allowed in rules " + "that contain Artifact items."), + item->location()); + } + if (!hasArtifactChildren && rule->outputFileTags.empty()) { + throw ErrorInfo(Tr::tr("A rule needs to have Artifact items or a non-empty " + "outputFileTags property."), item->location()); + } + rule->multiplex = evaluator.boolValue(item, StringConstants::multiplexProperty()); + rule->alwaysRun = evaluator.boolValue(item, StringConstants::alwaysRunProperty()); + rule->inputs = evaluator.fileTagsValue(item, StringConstants::inputsProperty()); + rule->inputsFromDependencies + = evaluator.fileTagsValue(item, StringConstants::inputsFromDependenciesProperty()); + bool requiresInputsSet = false; + rule->requiresInputs = evaluator.boolValue(item, StringConstants::requiresInputsProperty(), + &requiresInputsSet); + if (!requiresInputsSet) + rule->requiresInputs = rule->declaresInputs(); + rule->auxiliaryInputs + = evaluator.fileTagsValue(item, StringConstants::auxiliaryInputsProperty()); + rule->excludedInputs + = evaluator.fileTagsValue(item, StringConstants::excludedInputsProperty()); + if (rule->excludedInputs.empty()) { + rule->excludedInputs = evaluator.fileTagsValue( + item, StringConstants::excludedAuxiliaryInputsProperty()); + } + rule->explicitlyDependsOn + = evaluator.fileTagsValue(item, StringConstants::explicitlyDependsOnProperty()); + rule->explicitlyDependsOnFromDependencies = evaluator.fileTagsValue( + item, StringConstants::explicitlyDependsOnFromDependenciesProperty()); + rule->module = moduleContext ? moduleContext->module : projectContext->dummyModule; + if (!rule->multiplex && !rule->declaresInputs()) { + throw ErrorInfo(Tr::tr("Rule has no inputs, but is not a multiplex rule."), + item->location()); + } + if (!rule->multiplex && !rule->requiresInputs) { + throw ErrorInfo(Tr::tr("Rule.requiresInputs is false for non-multiplex rule."), + item->location()); + } + if (!rule->declaresInputs() && rule->requiresInputs) { + throw ErrorInfo(Tr::tr("Rule.requiresInputs is true, but the rule " + "does not declare any input tags."), item->location()); + } + if (productContext) { + rule->product = productContext->product.get(); + productContext->product->rules.push_back(rule); + } else { + projectContext->rules.push_back(rule); + } +} + +void resolveFileTagger(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext) +{ + Evaluator &evaluator = state.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) + return; + std::vector<FileTaggerConstPtr> &fileTaggers = productContext + ? productContext->product->fileTaggers + : projectContext->fileTaggers; + const QStringList patterns = evaluator.stringListValue(item, + StringConstants::patternsProperty()); + if (patterns.empty()) + throw ErrorInfo(Tr::tr("FileTagger.patterns must be a non-empty list."), item->location()); + + const FileTags fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty()); + if (fileTags.empty()) + throw ErrorInfo(Tr::tr("FileTagger.fileTags must not be empty."), item->location()); + + for (const QString &pattern : patterns) { + if (pattern.isEmpty()) + throw ErrorInfo(Tr::tr("A FileTagger pattern must not be empty."), item->location()); + } + + const int priority = evaluator.intValue(item, StringConstants::priorityProperty()); + fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority)); +} + +void resolveJobLimit(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext) +{ + Evaluator &evaluator = state.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) + return; + const QString jobPool = evaluator.stringValue(item, StringConstants::jobPoolProperty()); + if (jobPool.isEmpty()) + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.") + .arg(StringConstants::jobPoolProperty()), item->location()); + bool jobCountWasSet; + const int jobCount = evaluator.intValue(item, StringConstants::jobCountProperty(), -1, + &jobCountWasSet); + if (!jobCountWasSet) { + throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + if (jobCount < 0) { + throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.") + .arg(StringConstants::jobCountProperty()), item->location()); + } + JobLimits &jobLimits = moduleContext + ? moduleContext->jobLimits + : productContext ? productContext->product->jobLimits + : projectContext->jobLimits; + JobLimit jobLimit(jobPool, jobCount); + const int oldLimit = jobLimits.getLimit(jobPool); + if (oldLimit == -1 || oldLimit > jobCount) + jobLimits.setJobLimit(jobLimit); +} + +const FileTag unknownFileTag() +{ + static const FileTag tag("unknown-file-tag"); + return tag; +} + +bool ProductContext::dependenciesResolvingPending() const +{ + return (!dependenciesContext || !dependenciesContext->dependenciesResolved) + && !product && !delayedError.hasError(); +} + +std::pair<ProductDependency, ProductContext *> ProductContext::pendingDependency() const +{ + return dependenciesContext ? dependenciesContext->pendingDependency() + : std::make_pair(ProductDependency::None, nullptr); +} + +TimingData &TimingData::operator+=(const TimingData &other) +{ + dependenciesResolving += other.dependenciesResolving; + moduleProviders += other.moduleProviders; + moduleInstantiation += other.moduleInstantiation; + propertyMerging += other.propertyMerging; + groupsSetup += other.groupsSetup; + groupsResolving += other.groupsResolving; + preparingProducts += other.preparingProducts; + resolvingProducts += other.resolvingProducts; + probes += other.probes; + propertyEvaluation += other.propertyEvaluation; + propertyChecking += other.propertyChecking; + return *this; +} + +DependenciesContext::~DependenciesContext() = default; + +ItemReaderCache::AstCacheEntry &ItemReaderCache::retrieveOrSetupCacheEntry( + const QString &filePath, const std::function<void (AstCacheEntry &)> &setup) +{ + std::lock_guard lock(m_astCache.mutex); + AstCacheEntry &entry = m_astCache.data[filePath]; + if (!entry.ast) { + setup(entry); + m_filesRead << filePath; + } + return entry; +} + +const QStringList &ItemReaderCache::retrieveOrSetDirectoryEntries( + const QString &dir, const std::function<QStringList ()> &findOnDisk) +{ + std::lock_guard lock(m_directoryEntries.mutex); + auto &entries = m_directoryEntries.data[dir]; + if (!entries) + entries = findOnDisk(); + return *entries; +} + +bool ItemReaderCache::AstCacheEntry::addProcessingThread() +{ + std::lock_guard lock(m_processingThreads.mutex); + return m_processingThreads.data.insert(std::this_thread::get_id()).second; +} + +void ItemReaderCache::AstCacheEntry::removeProcessingThread() +{ + std::lock_guard lock(m_processingThreads.mutex); + m_processingThreads.data.remove(std::this_thread::get_id()); +} + } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/loaderutils.h b/src/lib/corelib/loader/loaderutils.h index db05698a5..679f60e27 100644 --- a/src/lib/corelib/loader/loaderutils.h +++ b/src/lib/corelib/loader/loaderutils.h @@ -41,46 +41,113 @@ #include <language/filetags.h> #include <language/forward_decls.h> +#include <language/item.h> +#include <language/moduleproviderinfo.h> +#include <language/propertydeclaration.h> #include <language/qualifiedid.h> +#include <parser/qmljsengine_p.h> +#include <tools/filetime.h> +#include <tools/joblimits.h> #include <tools/pimpl.h> #include <tools/set.h> #include <tools/version.h> +#include <QHash> #include <QStringList> #include <QVariant> +#include <atomic> +#include <functional> +#include <memory> +#include <mutex> +#include <optional> +#include <shared_mutex> +#include <thread> +#include <utility> #include <vector> namespace qbs { class SetupProjectParameters; namespace Internal { -class DependenciesResolver; class Evaluator; -class Item; class ItemPool; class ItemReader; -class LocalProfiles; class Logger; -class ModuleInstantiator; -class ModulePropertyMerger; -class ProbesResolver; class ProductContext; -class ProductItemMultiplexer; class ProgressObserver; class ProjectContext; +class ScriptEngine; using ModulePropertiesPerGroup = std::unordered_map<const Item *, QualifiedIdSet>; -using ShadowProductInfo = std::pair<bool, QString>; +using FileLocations = QHash<std::pair<QString, QString>, CodeLocation>; enum class FallbackMode { Enabled, Disabled }; enum class Deferral { Allowed, NotAllowed }; +enum class ProductDependency { None, Single, Bulk }; -class ProductInfo +class CancelException { }; + +template<typename DataType, typename MutexType = std::shared_mutex> +struct GuardedData { + DataType data; + mutable MutexType mutex; +}; + +class TimingData { public: - std::vector<ProbeConstPtr> probes; - ModulePropertiesPerGroup modulePropertiesSetInGroups; - ErrorInfo delayedError; + TimingData &operator+=(const TimingData &other); + + qint64 dependenciesResolving = 0; + qint64 moduleProviders = 0; + qint64 moduleInstantiation = 0; + qint64 propertyMerging = 0; + qint64 groupsSetup = 0; + qint64 groupsResolving = 0; + qint64 preparingProducts = 0; + qint64 resolvingProducts = 0; + qint64 schedulingProducts = 0; + qint64 probes = 0; + qint64 propertyEvaluation = 0; + qint64 propertyChecking = 0; +}; + +class ItemReaderCache +{ +public: + class AstCacheEntry + { + public: + QString code; + QbsQmlJS::Engine engine; + QbsQmlJS::AST::UiProgram *ast = nullptr; + + bool addProcessingThread(); + void removeProcessingThread(); + + private: + GuardedData<Set<std::thread::id>, std::recursive_mutex> m_processingThreads; + }; + + const Set<QString> &filesRead() const { return m_filesRead; } + AstCacheEntry &retrieveOrSetupCacheEntry(const QString &filePath, + const std::function<void(AstCacheEntry &)> &setup); + const QStringList &retrieveOrSetDirectoryEntries( + const QString &dir, const std::function<QStringList()> &findOnDisk); + +private: + Set<QString> m_filesRead; + GuardedData<std::unordered_map<QString, std::optional<QStringList>>, std::mutex> m_directoryEntries; // TODO: Merge with module dir entries cache? + GuardedData<std::unordered_map<QString, AstCacheEntry>, std::mutex> m_astCache; +}; + +class DependenciesContext +{ +public: + virtual ~DependenciesContext(); + virtual std::pair<ProductDependency, ProductContext *> pendingDependency() const = 0; + + bool dependenciesResolved = false; }; class ProductContext @@ -89,21 +156,39 @@ public: QString uniqueName() const; QString displayName() const; void handleError(const ErrorInfo &error); + bool dependenciesResolvingPending() const; + std::pair<ProductDependency, ProductContext *> pendingDependency() const; QString name; + QString buildDirectory; Item *item = nullptr; Item *scope = nullptr; ProjectContext *project = nullptr; + std::unique_ptr<ProductContext> shadowProduct; Item *mergedExportItem = nullptr; - ProductInfo info; + std::vector<ProbeConstPtr> probes; + ModulePropertiesPerGroup modulePropertiesSetInGroups; + ErrorInfo delayedError; QString profileName; QString multiplexConfigurationId; QVariantMap profileModuleProperties; // Tree-ified module properties from profile. QVariantMap moduleProperties; // Tree-ified module properties from profile + overridden values. + std::optional<QVariantMap> providerConfig; QVariantMap defaultParameters; // In Export item. QStringList searchPaths; + ResolvedProductPtr product; + TimingData timingData; + std::unique_ptr<DependenciesContext> dependenciesContext; - bool dependenciesResolved = false; + // This is needed because complex cyclic dependencies that involve Depends.productTypes + // may only be detected after a product has already been fully resolved. + std::vector<std::pair<FileTags, CodeLocation>> bulkDependencies; + + // The keys are module prototypes, the values specify whether the module's + // condition is true for this product. + std::unordered_map<Item *, bool> modulePrototypeEnabledInfo; + + int dependsItemCount = -1; }; class TopLevelProjectContext @@ -112,27 +197,196 @@ public: TopLevelProjectContext() = default; TopLevelProjectContext(const TopLevelProjectContext &) = delete; TopLevelProjectContext &operator=(const TopLevelProjectContext &) = delete; - ~TopLevelProjectContext() { qDeleteAll(projects); } + ~TopLevelProjectContext(); bool checkItemCondition(Item *item, Evaluator &evaluator); - void checkCancelation(const SetupProjectParameters ¶meters); - - std::vector<ProjectContext *> projects; - std::list<std::pair<ProductContext *, int>> productsToHandle; - std::multimap<QString, ProductContext *> productsByName; - std::unordered_map<Item *, ProductInfo> productInfos; - Set<QString> projectNamesUsedInOverrides; - Set<QString> productNamesUsedInOverrides; - Set<Item *> disabledItems; - Set<QString> erroneousProducts; - std::vector<ProbeConstPtr> probes; - QString buildDirectory; - QVariantMap profileConfigs; - ProgressObserver *progressObserver = nullptr; + QString sourceCodeForEvaluation(const JSSourceValueConstPtr &value); + ScriptFunctionPtr scriptFunctionValue(Item *item, const QString &name); + QString sourceCodeAsFunction(const JSSourceValueConstPtr &value, + const PropertyDeclaration &decl); + + void setCanceled() { m_canceled = true; } + void checkCancelation(); + bool isCanceled() const { return m_canceled; } + + int productCount() const { return m_productsByName.size(); } + + void addProductToHandle(const ProductContext &product) { m_productsToHandle.data << &product; } + void removeProductToHandle(const ProductContext &product); + bool isProductQueuedForHandling(const ProductContext &product) const; + int productsToHandleCount() const { return m_productsToHandle.data.size(); } + + void addDisabledItem(Item *item); + bool isDisabledItem(Item *item) const; + + void setProgressObserver(ProgressObserver *observer); + ProgressObserver *progressObserver() const; + + void addProject(ProjectContext *project) { m_projects.push_back(project); } + const std::vector<ProjectContext *> &projects() const { return m_projects; } + + void addQueuedError(const ErrorInfo &error); + const std::vector<ErrorInfo> &queuedErrors() const { return m_queuedErrors.data; } + + void setProfileConfigs(const QVariantMap &profileConfigs) { m_profileConfigs = profileConfigs; } + void addProfileConfig(const QString &profileName, const QVariantMap &profileConfig); + const QVariantMap &profileConfigs() const { return m_profileConfigs; } + std::optional<QVariantMap> profileConfig(const QString &profileName) const; + + void addProduct(ProductContext &product); + void addProductByType(ProductContext &product, const FileTags &tags); + ProductContext *productWithNameAndConstraint( + const QString &name, const std::function<bool(ProductContext &)> &constraint); + std::vector<ProductContext *> productsWithNameAndConstraint( + const QString &name, const std::function<bool(ProductContext &)> &constraint); + std::vector<ProductContext *> productsWithTypeAndConstraint( + const FileTags &tags, const std::function<bool(ProductContext &)> &constraint); + std::vector<std::pair<ProductContext *, CodeLocation>> + finishedProductsWithBulkDependency(const FileTag &tag) const; + void registerBulkDependencies(ProductContext &product); + + void addProjectNameUsedInOverrides(const QString &name); + const Set<QString> &projectNamesUsedInOverrides() const; + + void addProductNameUsedInOverrides(const QString &name); + const Set<QString> &productNamesUsedInOverrides() const; + + void setBuildDirectory(const QString &buildDir) { m_buildDirectory = buildDir; } + const QString &buildDirectory() const { return m_buildDirectory; } + + void addMultiplexConfiguration(const QString &id, const QVariantMap &config); + QVariantMap multiplexConfiguration(const QString &id) const; + + void setLastResolveTime(const FileTime &time) { m_lastResolveTime = time; } + const FileTime &lastResolveTime() const { return m_lastResolveTime; } + + Set<QString> buildSystemFiles() const { return m_itemReaderCache.filesRead(); } + + std::lock_guard<std::mutex> moduleProvidersCacheLock(); + void setModuleProvidersCache(const ModuleProvidersCache &cache); + const ModuleProvidersCache &moduleProvidersCache() const { return m_moduleProvidersCache; } + ModuleProviderInfo *moduleProvider(const ModuleProvidersCacheKey &key); + ModuleProviderInfo &addModuleProvider(const ModuleProvidersCacheKey &key, + const ModuleProviderInfo &provider); + + void addParameterDeclarations(const Item *moduleProto, + const Item::PropertyDeclarationMap &decls); + Item::PropertyDeclarationMap parameterDeclarations(Item *moduleProto) const; + + // An empty string means no matching module directory was found. + QString findModuleDirectory(const QualifiedId &module, const QString &searchPath, + const std::function<QString()> &findOnDisk); + + QStringList getModuleFilesForDirectory(const QString &dir, + const std::function<QStringList()> &findOnDisk); + void removeModuleFileFromDirectoryCache(const QString &filePath); + + void addUnknownProfilePropertyError(const Item *moduleProto, const ErrorInfo &error); + const std::vector<ErrorInfo> &unknownProfilePropertyErrors(const Item *moduleProto) const; + + Item *getModulePrototype(const QString &filePath, const QString &profile, + const std::function<Item *()> &produce); + + void addLocalProfile(const QString &name, const QVariantMap &values, + const CodeLocation &location); + const QVariantMap localProfiles() { return m_localProfiles; } + void checkForLocalProfileAsTopLevelProfile(const QString &topLevelProfile); + + using ProbeFilter = std::function<bool(const ProbeConstPtr &)>; + std::lock_guard<std::mutex> probesCacheLock(); + void setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes); + void setOldProductProbes(const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes); + void addNewlyResolvedProbe(const ProbeConstPtr &probe); + void addProjectLevelProbe(const ProbeConstPtr &probe); + const std::vector<ProbeConstPtr> projectLevelProbes() const; + ProbeConstPtr findOldProjectProbe(const QString &id, const ProbeFilter &filter) const; + ProbeConstPtr findOldProductProbe(const QString &productName, const ProbeFilter &filter) const; + ProbeConstPtr findCurrentProbe(const CodeLocation &location, const ProbeFilter &filter) const; + void incrementProbesCount() { ++m_probesInfo.probesEncountered; } + void incrementReusedCurrentProbesCount() { ++m_probesInfo.probesCachedCurrent; } + void incrementReusedOldProbesCount() { ++m_probesInfo.probesCachedOld; } + void incrementRunProbesCount() { ++m_probesInfo.probesRun; } + int probesEncounteredCount() const { return m_probesInfo.probesEncountered; } + int probesRunCount() const { return m_probesInfo.probesRun; } + int reusedOldProbesCount() const { return m_probesInfo.probesCachedOld; } + int reusedCurrentProbesCount() const { return m_probesInfo.probesCachedCurrent; } + + TimingData &timingData() { return m_timingData; } + ItemReaderCache &itemReaderCache() { return m_itemReaderCache; } + + void incProductDeferrals() { ++m_productDeferrals; } + int productDeferrals() const { return m_productDeferrals; } + + void collectDataFromEngine(const ScriptEngine &engine); + + ItemPool &createItemPool(); + +private: + const ResolvedFileContextPtr &resolvedFileContext(const FileContextConstPtr &ctx); + + std::vector<ProjectContext *> m_projects; + GuardedData<Set<const ProductContext *>> m_productsToHandle; + std::multimap<QString, ProductContext *> m_productsByName; + GuardedData<std::unordered_map<QStringView, QString>, std::mutex> m_sourceCode; + std::unordered_map<QString, QVariantMap> m_multiplexConfigsById; + GuardedData<QHash<CodeLocation, ScriptFunctionPtr>, std::mutex> m_scriptFunctionMap; + GuardedData<std::unordered_map<std::pair<QStringView, QStringList>, QString>, + std::mutex> m_scriptFunctions; + std::unordered_map<FileContextConstPtr, ResolvedFileContextPtr> m_fileContextMap; + Set<QString> m_projectNamesUsedInOverrides; + Set<QString> m_productNamesUsedInOverrides; + GuardedData<Set<Item *>> m_disabledItems; + GuardedData<std::vector<ErrorInfo>, std::mutex> m_queuedErrors; + QString m_buildDirectory; + QVariantMap m_profileConfigs; + ProgressObserver *m_progressObserver = nullptr; + TimingData m_timingData; + ModuleProvidersCache m_moduleProvidersCache; + std::mutex m_moduleProvidersCacheMutex; + QVariantMap m_localProfiles; + ItemReaderCache m_itemReaderCache; + QHash<FileTag, std::vector<std::pair<ProductContext *, CodeLocation>>> m_reverseBulkDependencies; // For fast look-up when resolving Depends.productTypes. // The contract is that it contains fully handled, error-free, enabled products. - std::multimap<FileTag, ProductContext *> productsByType; + GuardedData<std::multimap<FileTag, ProductContext *>> m_productsByType; + + // The keys are module prototypes. + GuardedData<std::unordered_map<const Item *, + Item::PropertyDeclarationMap>> m_parameterDeclarations; + GuardedData<std::unordered_map<const Item *, + std::vector<ErrorInfo>>> m_unknownProfilePropertyErrors; + + // The keys are search path + module name, the values are directories. + GuardedData<QHash<std::pair<QString, QualifiedId>, std::optional<QString>>, + std::mutex> m_modulePathCache; + + // The keys are file paths, the values are module prototype items accompanied by a profile. + GuardedData<std::unordered_map<QString, std::vector<std::pair<Item *, QString>>>, + std::mutex> m_modulePrototypes; + + GuardedData<std::map<QString, std::optional<QStringList>>, + std::mutex> m_moduleFilesPerDirectory; + + struct { + QHash<QString, std::vector<ProbeConstPtr>> oldProjectProbes; + QHash<QString, std::vector<ProbeConstPtr>> oldProductProbes; + QHash<CodeLocation, std::vector<ProbeConstPtr>> currentProbes; + std::vector<ProbeConstPtr> projectLevelProbes; + + quint64 probesEncountered = 0; + quint64 probesRun = 0; + quint64 probesCachedCurrent = 0; + quint64 probesCachedOld = 0; + } m_probesInfo; + std::mutex m_probesMutex; + + std::vector<std::unique_ptr<ItemPool>> m_itemPools; + + FileTime m_lastResolveTime; + + std::atomic_bool m_canceled = false; + int m_productDeferrals = 0; }; class ProjectContext @@ -142,28 +396,36 @@ public: Item *item = nullptr; Item *scope = nullptr; TopLevelProjectContext *topLevelProject = nullptr; + ProjectContext *parent = nullptr; + std::vector<ProjectContext *> children; std::vector<ProductContext> products; std::vector<QStringList> searchPathsStack; + ResolvedProjectPtr project; + std::vector<FileTaggerConstPtr> fileTaggers; + std::vector<RulePtr> rules; + JobLimits jobLimits; + ResolvedModulePtr dummyModule; +}; + +class ModuleContext +{ +public: + ResolvedModulePtr module; + JobLimits jobLimits; }; class LoaderState { public: - LoaderState(const SetupProjectParameters ¶meters, ItemPool &itemPool, Evaluator &evaluator, - Logger &logger); + LoaderState(const SetupProjectParameters ¶meters, TopLevelProjectContext &topLevelProject, + ItemPool &itemPool, ScriptEngine &engine, Logger logger); ~LoaderState(); - DependenciesResolver &dependenciesResolver(); Evaluator &evaluator(); ItemPool &itemPool(); ItemReader &itemReader(); - LocalProfiles &localProfiles(); Logger &logger(); - ModuleInstantiator &moduleInstantiator(); - ProductItemMultiplexer &multiplexer(); const SetupProjectParameters ¶meters() const; - ProbesResolver &probesResolver(); - ModulePropertyMerger &propertyMerger(); TopLevelProjectContext &topLevelProject(); private: @@ -171,9 +433,16 @@ private: Pimpl<Private> d; }; +QString fullProductDisplayName(const QString &name, const QString &multiplexId); void mergeParameters(QVariantMap &dst, const QVariantMap &src); -ShadowProductInfo getShadowProductInfo(const ProductContext &product); void adjustParametersScopes(Item *item, Item *scope); +void resolveRule(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext); +void resolveJobLimit(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext, ModuleContext *moduleContext); +void resolveFileTagger(LoaderState &state, Item *item, ProjectContext *projectContext, + ProductContext *productContext); +const FileTag unknownFileTag(); } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/loader/localprofiles.cpp b/src/lib/corelib/loader/localprofiles.cpp index b59204492..e72128fff 100644 --- a/src/lib/corelib/loader/localprofiles.cpp +++ b/src/lib/corelib/loader/localprofiles.cpp @@ -52,34 +52,26 @@ #include <tools/stringconstants.h> namespace qbs::Internal { -class LocalProfiles::Private +class LocalProfiles { public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} + LocalProfiles(LoaderState &loaderState) : m_loaderState(loaderState) {} + void collectProfiles(Item *productOrProject, Item *projectScope); + +private: void handleProfile(Item *profileItem); void evaluateProfileValues(const QualifiedId &namePrefix, Item *item, Item *profileItem, QVariantMap &values); - void collectProfiles(Item *productOrProject, Item *projectScope); - - LoaderState &loaderState; - QVariantMap profiles; + LoaderState &m_loaderState; }; -LocalProfiles::LocalProfiles(LoaderState &loaderState) : d(makePimpl<Private>(loaderState)) {} -LocalProfiles::~LocalProfiles() = default; - -void LocalProfiles::collectProfilesFromItems(Item *productOrProject, Item *projectScope) +void collectProfilesFromItems(Item *productOrProject, Item *projectScope, LoaderState &loaderState) { - d->collectProfiles(productOrProject, projectScope); + LocalProfiles(loaderState).collectProfiles(productOrProject, projectScope); } -const QVariantMap &LocalProfiles::profiles() const -{ - return d->profiles; -} - -void LocalProfiles::Private::handleProfile(Item *profileItem) +void LocalProfiles::handleProfile(Item *profileItem) { QVariantMap values; evaluateProfileValues(QualifiedId(), profileItem, profileItem, values); @@ -93,15 +85,11 @@ void LocalProfiles::Private::handleProfile(Item *profileItem) throw ErrorInfo(Tr::tr("Reserved name '%1' cannot be used for an actual profile.") .arg(profileName), profileItem->location()); } - if (profiles.contains(profileName)) { - throw ErrorInfo(Tr::tr("Local profile '%1' redefined.").arg(profileName), - profileItem->location()); - } - profiles.insert(profileName, values); + m_loaderState.topLevelProject().addLocalProfile(profileName, values, profileItem->location()); } -void LocalProfiles::Private::evaluateProfileValues(const QualifiedId &namePrefix, Item *item, - Item *profileItem, QVariantMap &values) +void LocalProfiles::evaluateProfileValues(const QualifiedId &namePrefix, Item *item, + Item *profileItem, QVariantMap &values) { const Item::PropertyMap &props = item->properties(); for (auto it = props.begin(); it != props.end(); ++it) { @@ -119,16 +107,16 @@ void LocalProfiles::Private::evaluateProfileValues(const QualifiedId &namePrefix case Value::JSSourceValueType: if (item != profileItem) item->setScope(profileItem); - const ScopedJsValue sv(loaderState.evaluator().engine()->context(), - loaderState.evaluator().value(item, it.key())); + const ScopedJsValue sv(m_loaderState.evaluator().engine()->context(), + m_loaderState.evaluator().value(item, it.key())); values.insert(name.join(QLatin1Char('.')), - getJsVariant(loaderState.evaluator().engine()->context(), sv)); + getJsVariant(m_loaderState.evaluator().engine()->context(), sv)); break; } } } -void LocalProfiles::Private::collectProfiles(Item *productOrProject, Item *projectScope) +void LocalProfiles::collectProfiles(Item *productOrProject, Item *projectScope) { Item * scope = productOrProject->type() == ItemType::Project ? projectScope : nullptr; for (auto it = productOrProject->children().begin(); @@ -137,7 +125,7 @@ void LocalProfiles::Private::collectProfiles(Item *productOrProject, Item *proje if (childItem->type() == ItemType::Profile) { if (!scope) { const ItemValuePtr itemValue = ItemValue::create(productOrProject); - scope = Item::create(productOrProject->pool(), ItemType::Scope); + scope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); scope->setProperty(StringConstants::productVar(), itemValue); scope->setFile(productOrProject->file()); scope->setScope(projectScope); @@ -146,7 +134,7 @@ void LocalProfiles::Private::collectProfiles(Item *productOrProject, Item *proje try { handleProfile(childItem); } catch (const ErrorInfo &e) { - handlePropertyError(e, loaderState.parameters(), loaderState.logger()); + handlePropertyError(e, m_loaderState.parameters(), m_loaderState.logger()); } it = productOrProject->children().erase(it); // TODO: delete item and scope } else { diff --git a/src/lib/corelib/loader/localprofiles.h b/src/lib/corelib/loader/localprofiles.h index e31c8c81b..7d9eee88e 100644 --- a/src/lib/corelib/loader/localprofiles.h +++ b/src/lib/corelib/loader/localprofiles.h @@ -39,27 +39,11 @@ #pragma once -#include <tools/pimpl.h> -#include <QVariantMap> - namespace qbs::Internal { class Item; class LoaderState; -// This class evaluates all Profile items encountered in the project tree and holds the results. -class LocalProfiles -{ -public: - LocalProfiles(LoaderState &loaderState); - ~LocalProfiles(); - - void collectProfilesFromItems(Item *productOrProject, Item *projectScope); - const QVariantMap &profiles() const; - -private: - class Private; - Pimpl<Private> d; -}; +void collectProfilesFromItems(Item *productOrProject, Item *projectScope, LoaderState &loaderState); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleinstantiator.cpp b/src/lib/corelib/loader/moduleinstantiator.cpp index 73b676be4..ab67bc270 100644 --- a/src/lib/corelib/loader/moduleinstantiator.cpp +++ b/src/lib/corelib/loader/moduleinstantiator.cpp @@ -55,39 +55,39 @@ #include <utility> namespace qbs::Internal { -class ModuleInstantiator::Private + +static std::pair<const Item *, Item *> +getOrSetModuleInstanceItem(Item *container, Item *moduleItem, const QualifiedId &moduleName, + const QString &id, bool replace, LoaderState &loaderState); + +class ModuleInstantiator { public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} + ModuleInstantiator(const InstantiationContext &context, LoaderState &loaderState) + : context(context), loaderState(loaderState) {} + + void instantiate(); - void overrideProperties(const Context &context); - void setupScope(const Context &context); - void exchangePlaceholderItem(Item *product, Item *loadingItem, const QString &loadingName, - Item *moduleItemForItemValues, const QualifiedId &moduleName, const QString &id, - bool isProductDependency, bool alreadyLoaded); - std::pair<const Item *, Item *> - getOrSetModuleInstanceItem(Item *container, Item *moduleItem, const QualifiedId &moduleName, - const QString &id, bool replace); +private: + void overrideProperties(); + void setupScope(); + void exchangePlaceholderItem(Item *loadingItem, Item *moduleItemForItemValues); + const InstantiationContext &context; LoaderState &loaderState; - qint64 elapsedTime = 0; }; -ModuleInstantiator::ModuleInstantiator(LoaderState &loaderState) - : d(makePimpl<Private>(loaderState)) {} -ModuleInstantiator::~ModuleInstantiator() = default; - -void ModuleInstantiator::instantiate(const Context &context) +void ModuleInstantiator::instantiate() { - AccumulatingTimer timer(d->loaderState.parameters().logElapsedTime() - ? &d->elapsedTime : nullptr); + AccumulatingTimer timer(loaderState.parameters().logElapsedTime() + ? &context.product.timingData.moduleInstantiation : nullptr); // This part needs to be done only once per module and product, and only if the module // was successfully loaded. if (context.module && !context.alreadyLoaded) { context.module->setType(ItemType::ModuleInstance); - d->overrideProperties(context); - d->setupScope(context); + overrideProperties(); + setupScope(); } // This strange-looking code deals with the fact that our syntax cannot properly handle @@ -109,33 +109,28 @@ void ModuleInstantiator::instantiate(const Context &context) // } // It's debatable whether that's a good feature, but it has been working (accidentally?) // for a long time, and removing it now would break a lot of existing projects. - d->exchangePlaceholderItem( - context.product, context.loadingItem, context.loadingName, moduleItemForItemValues, - context.moduleName, context.id, context.exportingProduct, context.alreadyLoaded); - - if (!context.alreadyLoaded && context.product && context.product != context.loadingItem) { - d->exchangePlaceholderItem( - context.product, context.product, context.productName, moduleItemForItemValues, - context.moduleName, context.id, context.exportingProduct, context.alreadyLoaded); + exchangePlaceholderItem(context.loadingItem, moduleItemForItemValues); + + if (!context.alreadyLoaded && context.product.item + && context.product.item != context.loadingItem) { + exchangePlaceholderItem(context.product.item, moduleItemForItemValues); } } -void ModuleInstantiator::Private::exchangePlaceholderItem( - Item *product, Item *loadingItem, const QString &loadingName, Item *moduleItemForItemValues, - const QualifiedId &moduleName, const QString &id, bool isProductModule, bool alreadyLoaded) +void ModuleInstantiator::exchangePlaceholderItem(Item *loadingItem, Item *moduleItemForItemValues) { // If we have a module item, set an item value pointing to it as a property on the loading item. // Evict a possibly existing placeholder item, and return it to us, so we can merge its values // into the instance. const auto &[oldItem, newItem] = getOrSetModuleInstanceItem( - loadingItem, moduleItemForItemValues, moduleName, id, true); + loadingItem, moduleItemForItemValues, context.moduleName, context.id, true, loaderState); // The new item always exists, even if we don't have a module item. In that case, the // function created a placeholder item for us, which we then have to turn into a // non-present module. QBS_CHECK(newItem); if (!moduleItemForItemValues) { - createNonPresentModule(loaderState.itemPool(), moduleName.toString(), + createNonPresentModule(loaderState.itemPool(), context.moduleName.toString(), QLatin1String("not found"), newItem); return; } @@ -148,7 +143,7 @@ void ModuleInstantiator::Private::exchangePlaceholderItem( // (see getOrSetModuleInstanceItem() below for details). if (oldItem == newItem) { QBS_CHECK(oldItem->type() == ItemType::ModuleInstance); - QBS_CHECK(alreadyLoaded || isProductModule); + QBS_CHECK(context.alreadyLoaded || context.exportingProduct); return; } @@ -172,22 +167,96 @@ void ModuleInstantiator::Private::exchangePlaceholderItem( } // Now merge the locally attached values into the actual module instance. - loaderState.propertyMerger().mergeFromLocalInstance(product, loadingItem, loadingName, - oldItem, moduleItemForItemValues); + mergeFromLocalInstance(context.product, loadingItem, context.loadingName, oldItem, + moduleItemForItemValues, loaderState); // TODO: We'd like to delete the placeholder item here, because it's not // being referenced anymore and there's a lot of them. However, this // is not supported by ItemPool. Investigate the use of std::pmr. } -Item *ModuleInstantiator::retrieveModuleInstanceItem(Item *containerItem, const QualifiedId &name) +Item *retrieveModuleInstanceItem(Item *containerItem, const QualifiedId &name, + LoaderState &loaderState) +{ + return getOrSetModuleInstanceItem(containerItem, nullptr, name, {}, false, loaderState).second; +} + +Item *retrieveQbsItem(Item *containerItem, LoaderState &loaderState) +{ + return retrieveModuleInstanceItem(containerItem, StringConstants::qbsModule(), loaderState); +} + +void ModuleInstantiator::overrideProperties() +{ + // Users can override module properties on the command line with the + // modules.<module-name>.<property-name>:<value> syntax. + // For simplicity and backwards compatibility, qbs properties can also be given without + // the "modules." prefix, i.e. just qbs.<property-name>:<value>. + // In addition, users can override module properties just for certain products + // using the products.<product-name>.<module-name>.<property-name>:<value> syntax. + // Such product-specific overrides have higher precedence. + const QString fullName = context.moduleName.toString(); + const QString generalOverrideKey = QStringLiteral("modules.") + fullName; + const QString perProductOverrideKey = StringConstants::productsOverridePrefix() + + context.product.name + QLatin1Char('.') + fullName; + const SetupProjectParameters ¶meters = loaderState.parameters(); + Logger &logger = loaderState.logger(); + context.module->overrideProperties(parameters.overriddenValuesTree(), generalOverrideKey, + parameters, logger); + if (fullName == StringConstants::qbsModule()) { + context.module->overrideProperties(parameters.overriddenValuesTree(), fullName, parameters, + logger); + } + context.module->overrideProperties(parameters.overriddenValuesTree(), perProductOverrideKey, + parameters, logger); +} + +void ModuleInstantiator::setupScope() { - return d->getOrSetModuleInstanceItem(containerItem, nullptr, name, {}, false).second; + Item * const scope = Item::create(&loaderState.itemPool(), ItemType::Scope); + QBS_CHECK(context.module->file()); + scope->setFile(context.module->file()); + QBS_CHECK(context.product.project->scope); + context.product.project->scope->copyProperty(StringConstants::projectVar(), scope); + if (context.product.scope) + context.product.scope->copyProperty(StringConstants::productVar(), scope); + else + QBS_CHECK(context.moduleName.toString() == StringConstants::qbsModule()); // Dummy product. + + if (!context.module->id().isEmpty()) + scope->setProperty(context.module->id(), ItemValue::create(context.module)); + for (Item * const child : context.module->children()) { + child->setScope(scope); + if (!child->id().isEmpty()) + scope->setProperty(child->id(), ItemValue::create(child)); + } + context.module->setScope(scope); + + if (context.exportingProduct) { + QBS_CHECK(context.exportingProduct->type() == ItemType::Product); + + const auto exportingProductItemValue = ItemValue::create(context.exportingProduct); + scope->setProperty(QStringLiteral("exportingProduct"), exportingProductItemValue); + + const auto importingProductItemValue = ItemValue::create(context.product.item); + scope->setProperty(QStringLiteral("importingProduct"), importingProductItemValue); + + // FIXME: This looks wrong. Introduce exportingProject variable? + scope->setProperty(StringConstants::projectVar(), + ItemValue::create(context.exportingProduct->parent())); + + PropertyDeclaration pd(StringConstants::qbsSourceDirPropertyInternal(), + PropertyDeclaration::String, QString(), + PropertyDeclaration::PropertyNotAvailableInConfig); + context.module->setPropertyDeclaration(pd.name(), pd); + context.module->setProperty(pd.name(), context.exportingProduct->property( + StringConstants::sourceDirectoryProperty())); + } } -Item *ModuleInstantiator::retrieveQbsItem(Item *containerItem) +void instantiateModule(const InstantiationContext &context, LoaderState &loaderState) { - return retrieveModuleInstanceItem(containerItem, StringConstants::qbsModule()); + ModuleInstantiator(context, loaderState).instantiate(); } // This important function deals with retrieving and setting (pseudo-)module instances from/on @@ -220,9 +289,9 @@ Item *ModuleInstantiator::retrieveQbsItem(Item *containerItem) // Use case 4: Module propagation to the the Group level. // In all cases, the first returned item is the existing one, and the second returned item // is the new one. Depending on the use case, they might be null and might also be the same item. -std::pair<const Item *, Item *> ModuleInstantiator::Private::getOrSetModuleInstanceItem( +std::pair<const Item *, Item *> getOrSetModuleInstanceItem( Item *container, Item *moduleItem, const QualifiedId &moduleName, const QString &id, - bool replace) + bool replace, LoaderState &loaderState) { Item *instance = container; const QualifiedId itemValueName @@ -248,9 +317,9 @@ std::pair<const Item *, Item *> ModuleInstantiator::Private::getOrSetModuleInsta Item *newItem = i < itemValueName.size() - 1 ? Item::create(&loaderState.itemPool(), ItemType::ModulePrefix) : moduleItem - ? moduleItem - : Item::create(&loaderState.itemPool(), - ItemType::ModuleInstancePlaceholder); + ? moduleItem + : Item::create(&loaderState.itemPool(), + ItemType::ModuleInstancePlaceholder); instance->setProperty(moduleNameSegment, ItemValue::create(newItem)); instance = newItem; } @@ -258,82 +327,4 @@ std::pair<const Item *, Item *> ModuleInstantiator::Private::getOrSetModuleInsta return {nullptr, instance}; } -void ModuleInstantiator::printProfilingInfo(int indent) -{ - if (!d->loaderState.parameters().logElapsedTime()) - return; - d->loaderState.logger().qbsLog(LoggerInfo, true) - << QByteArray(indent, ' ') - << Tr::tr("Instantiating modules took %1.") - .arg(elapsedTimeString(d->elapsedTime)); -} - -void ModuleInstantiator::Private::overrideProperties(const ModuleInstantiator::Context &context) -{ - // Users can override module properties on the command line with the - // modules.<module-name>.<property-name>:<value> syntax. - // For simplicity and backwards compatibility, qbs properties can also be given without - // the "modules." prefix, i.e. just qbs.<property-name>:<value>. - // In addition, users can override module properties just for certain products - // using the products.<product-name>.<module-name>.<property-name>:<value> syntax. - // Such product-specific overrides have higher precedence. - const QString fullName = context.moduleName.toString(); - const QString generalOverrideKey = QStringLiteral("modules.") + fullName; - const QString perProductOverrideKey = StringConstants::productsOverridePrefix() - + context.productName + QLatin1Char('.') + fullName; - const SetupProjectParameters ¶meters = loaderState.parameters(); - Logger &logger = loaderState.logger(); - context.module->overrideProperties(parameters.overriddenValuesTree(), generalOverrideKey, - parameters, logger); - if (fullName == StringConstants::qbsModule()) { - context.module->overrideProperties(parameters.overriddenValuesTree(), fullName, parameters, - logger); - } - context.module->overrideProperties(parameters.overriddenValuesTree(), perProductOverrideKey, - parameters, logger); -} - -void ModuleInstantiator::Private::setupScope(const ModuleInstantiator::Context &context) -{ - Item * const scope = Item::create(&loaderState.itemPool(), ItemType::Scope); - QBS_CHECK(context.module->file()); - scope->setFile(context.module->file()); - QBS_CHECK(context.projectScope); - context.projectScope->copyProperty(StringConstants::projectVar(), scope); - if (context.productScope) - context.productScope->copyProperty(StringConstants::productVar(), scope); - else - QBS_CHECK(context.moduleName.toString() == StringConstants::qbsModule()); // Dummy product. - - if (!context.module->id().isEmpty()) - scope->setProperty(context.module->id(), ItemValue::create(context.module)); - for (Item * const child : context.module->children()) { - child->setScope(scope); - if (!child->id().isEmpty()) - scope->setProperty(child->id(), ItemValue::create(child)); - } - context.module->setScope(scope); - - if (context.exportingProduct) { - QBS_CHECK(context.exportingProduct->type() == ItemType::Product); - - const auto exportingProductItemValue = ItemValue::create(context.exportingProduct); - scope->setProperty(QStringLiteral("exportingProduct"), exportingProductItemValue); - - const auto importingProductItemValue = ItemValue::create(context.product); - scope->setProperty(QStringLiteral("importingProduct"), importingProductItemValue); - - // FIXME: This looks wrong. Introduce exportingProject variable? - scope->setProperty(StringConstants::projectVar(), - ItemValue::create(context.exportingProduct->parent())); - - PropertyDeclaration pd(StringConstants::qbsSourceDirPropertyInternal(), - PropertyDeclaration::String, QString(), - PropertyDeclaration::PropertyNotAvailableInConfig); - context.module->setPropertyDeclaration(pd.name(), pd); - context.module->setProperty(pd.name(), context.exportingProduct->property( - StringConstants::sourceDirectoryProperty())); - } -} - } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleinstantiator.h b/src/lib/corelib/loader/moduleinstantiator.h index 7be886869..5e9ad2f44 100644 --- a/src/lib/corelib/loader/moduleinstantiator.h +++ b/src/lib/corelib/loader/moduleinstantiator.h @@ -39,7 +39,6 @@ #pragma once -#include <tools/pimpl.h> #include <QtGlobal> QT_BEGIN_NAMESPACE @@ -49,48 +48,35 @@ QT_END_NAMESPACE namespace qbs::Internal { class Item; class LoaderState; +class ProductContext; class QualifiedId; -// This class is responsible for setting up a proper module instance from a bunch of items: +class InstantiationContext { +public: + ProductContext &product; + Item * const loadingItem; + const QString &loadingName; + Item * const module; + Item * const moduleWithSameName; + Item * const exportingProduct; + const QualifiedId &moduleName; + const QString &id; + const bool alreadyLoaded; +}; + +// This function is responsible for setting up a proper module instance from a bunch of items: // - Set the item type to ItemType::ModuleInstance (from Module or Export). // - Apply possible command-line overrides for module properties. // - Replace a possible module instance placeholder in the loading item with the actual instance // and merge their values employing the ModulePropertyMerger. // - Setting up the module instance scope. -// In addition, it also provides helper functions for retrieving/setting module instance items -// for special purposes. -class ModuleInstantiator -{ -public: - ModuleInstantiator(LoaderState &loaderState); - ~ModuleInstantiator(); +void instantiateModule(const InstantiationContext &context, LoaderState &loaderState); - struct Context { - Item * const product; - const QString &productName; - Item * const loadingItem; - const QString &loadingName; - Item * const module; - Item * const moduleWithSameName; - Item * const exportingProduct; - Item * const productScope; - Item * const projectScope; - const QualifiedId &moduleName; - const QString &id; - const bool alreadyLoaded; - }; - void instantiate(const Context &context); - - // Note that these will also create the respective item value if it does not exist yet. - Item *retrieveModuleInstanceItem(Item *containerItem, const QualifiedId &name); - Item *retrieveQbsItem(Item *containerItem); - - void printProfilingInfo(int indent); - -private: - class Private; - Pimpl<Private> d; -}; +// Helper functions for retrieving/setting module instance items for special purposes. +// Note that these will also create the respective item value if it does not exist yet. +Item *retrieveModuleInstanceItem(Item *containerItem, const QualifiedId &name, + LoaderState &loaderState); +Item *retrieveQbsItem(Item *containerItem, LoaderState &loaderState); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleloader.cpp b/src/lib/corelib/loader/moduleloader.cpp index 98548278f..c937e8b22 100644 --- a/src/lib/corelib/loader/moduleloader.cpp +++ b/src/lib/corelib/loader/moduleloader.cpp @@ -42,13 +42,13 @@ #include "itemreader.h" #include "loaderutils.h" #include "moduleproviderloader.h" -#include "productitemmultiplexer.h" #include <api/languageinfo.h> #include <language/evaluator.h> #include <language/value.h> #include <logging/categories.h> #include <logging/translator.h> +#include <tools/codelocation.h> #include <tools/error.h> #include <tools/fileinfo.h> #include <tools/hostosinfo.h> @@ -64,42 +64,35 @@ namespace qbs::Internal { -class ModuleLoader::Private +class ModuleLoader { public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} - - std::pair<Item *, bool> loadModuleFile(const ProductContext &product, - const QString &moduleName, const QString &filePath); - std::pair<Item *, bool> getModulePrototype(const ModuleLoader::ProductContext &product, - const QString &moduleName, const QString &filePath); - bool evaluateModuleCondition(const ModuleLoader::ProductContext &product, Item *module, - const QString &fullModuleName); - void forwardParameterDeclarations(const QualifiedId &moduleName, Item *item, - const Item::Modules &modules); - - LoaderState &loaderState; - ModuleProviderLoader providerLoader{loaderState}; - - // The keys are file paths, the values are module prototype items accompanied by a profile. - std::unordered_map<QString, std::vector<std::pair<Item *, QString>>> modulePrototypes; - - // The keys are module prototypes and products, the values specify whether the module's - // condition is true for that product. - std::unordered_map<std::pair<Item *, const Item *>, bool> modulePrototypeEnabledInfo; - - std::unordered_map<const Item *, std::vector<ErrorInfo>> unknownProfilePropertyErrors; - std::unordered_map<const Item *, Item::PropertyDeclarationMap> parameterDeclarations; - std::unordered_map<const Item *, std::optional<QVariantMap>> providerConfigsPerProduct; - QHash<std::pair<QString, QualifiedId>, std::optional<QString>> existingModulePathCache; - std::map<QString, QStringList> moduleDirListCache; - - qint64 elapsedTimeModuleProviders = 0; -}; + ModuleLoader(LoaderState &loaderState, ProductContext &product, + const CodeLocation &dependsItemLocation, const QualifiedId &moduleName, + FallbackMode fallbackMode) + : m_loaderState(loaderState), m_product(product), + m_dependsItemLocation(dependsItemLocation), m_moduleName(moduleName), + m_fallbackMode(fallbackMode) + {} -ModuleLoader::ModuleLoader(LoaderState &loaderState) : d(makePimpl<Private>(loaderState)) { } + Item *load(); -ModuleLoader::~ModuleLoader() = default; +private: + std::pair<Item *, bool> loadModuleFile(const QString &moduleName, const QString &filePath); + Item *getModulePrototype(const QString &moduleName, const QString &filePath); + Item *createAndInitModuleItem(const QString &moduleName, const QString &filePath); + bool evaluateModuleCondition(Item *module, const QString &fullModuleName); + void checkForUnknownProfileProperties(const Item *module); + QString findModuleDirectory(const QString &searchPath); + QStringList findModuleDirectories(); + QStringList getModuleFilePaths(const QString &dir); + + LoaderState &m_loaderState; + ProductContext &m_product; + const CodeLocation &m_dependsItemLocation; + const QualifiedId &m_moduleName; + const FallbackMode m_fallbackMode; +}; struct PrioritizedItem { @@ -148,180 +141,127 @@ static Item *chooseModuleCandidate(const std::vector<PrioritizedItem> &candidate return maxIt->item; } -ModuleLoader::Result ModuleLoader::searchAndLoadModuleFile( - const ProductContext &productContext, const CodeLocation &dependsItemLocation, - const QualifiedId &moduleName, FallbackMode fallbackMode, bool isRequired) +Item *searchAndLoadModuleFile(LoaderState &loaderState, ProductContext &product, + const CodeLocation &dependsItemLocation, const QualifiedId &moduleName, + FallbackMode fallbackMode) { - const auto findExistingModulePath = [&](const QString &searchPath) { - // isFileCaseCorrect is a very expensive call on macOS, so we cache the value for the - // modules and search paths we've already processed - auto &moduleInfo = d->existingModulePathCache[{searchPath, moduleName}]; - if (moduleInfo) - return *moduleInfo; - - QString dirPath = searchPath + QStringLiteral("/modules"); - for (const QString &moduleNamePart : moduleName) { - dirPath = FileInfo::resolvePath(dirPath, moduleNamePart); - if (!FileInfo::exists(dirPath) || !FileInfo::isFileCaseCorrect(dirPath)) { - return *(moduleInfo = QString()); - } - } - - return *(moduleInfo = dirPath); - }; - const auto findExistingModulePaths = [&] { - const QStringList &searchPaths = d->loaderState.itemReader().allSearchPaths(); - QStringList result; - result.reserve(searchPaths.size()); - for (const auto &path: searchPaths) { - const QString dirPath = findExistingModulePath(path); - if (!dirPath.isEmpty()) - result.append(dirPath); - } - return result; - }; + return ModuleLoader(loaderState, product, dependsItemLocation, moduleName, fallbackMode).load(); +} - SearchPathsManager searchPathsManager(d->loaderState.itemReader()); +Item *ModuleLoader::load() +{ + SearchPathsManager searchPathsManager(m_loaderState.itemReader()); - Result loadResult; - auto existingPaths = findExistingModulePaths(); + QStringList existingPaths = findModuleDirectories(); if (existingPaths.isEmpty()) { // no suitable names found, try to use providers - AccumulatingTimer providersTimer(d->loaderState.parameters().logElapsedTime() - ? &d->elapsedTimeModuleProviders : nullptr); - std::optional<QVariantMap> &providerConfig - = d->providerConfigsPerProduct[productContext.productItem]; - auto result = d->providerLoader.executeModuleProviders( - {productContext.productItem, productContext.projectItem, productContext.name, - productContext.uniqueName, productContext.moduleProperties, providerConfig}, - dependsItemLocation, - moduleName, - fallbackMode); - loadResult.providerProbes << result.probes; - if (!providerConfig) - providerConfig = result.providerConfig; + AccumulatingTimer providersTimer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.moduleProviders : nullptr); + auto result = ModuleProviderLoader(m_loaderState).executeModuleProviders( + m_product, m_dependsItemLocation, m_moduleName, m_fallbackMode); if (result.searchPaths) { - qCDebug(lcModuleLoader) << "Re-checking for module" << moduleName.toString() + qCDebug(lcModuleLoader) << "Re-checking for module" << m_moduleName.toString() << "with newly added search paths from module provider"; - d->loaderState.itemReader().pushExtraSearchPaths(*result.searchPaths); - existingPaths = findExistingModulePaths(); + m_loaderState.itemReader().pushExtraSearchPaths(*result.searchPaths); + existingPaths = findModuleDirectories(); } } - const auto getModuleFileNames = [&](const QString &dirPath) -> QStringList & { - QStringList &moduleFileNames = d->moduleDirListCache[dirPath]; - if (moduleFileNames.empty()) { - QDirIterator dirIter(dirPath, StringConstants::qbsFileWildcards()); - while (dirIter.hasNext()) - moduleFileNames += dirIter.next(); - } - return moduleFileNames; - }; - - const QString fullName = moduleName.toString(); + const QString fullName = m_moduleName.toString(); bool triedToLoadModule = false; std::vector<PrioritizedItem> candidates; candidates.reserve(size_t(existingPaths.size())); for (int i = 0; i < existingPaths.size(); ++i) { - const QString &dirPath = existingPaths.at(i); - QStringList &moduleFileNames = getModuleFileNames(dirPath); - for (auto it = moduleFileNames.begin(); it != moduleFileNames.end(); ) { - const QString &filePath = *it; - const auto [module, triedToLoad] = d->loadModuleFile(productContext, fullName, - filePath); + const QStringList &moduleFileNames = getModuleFilePaths(existingPaths.at(i)); + for (const QString &filePath : moduleFileNames) { + const auto [module, triedToLoad] = loadModuleFile(fullName, filePath); if (module) candidates.emplace_back(module, 0, i); if (!triedToLoad) - it = moduleFileNames.erase(it); - else - ++it; + m_loaderState.topLevelProject().removeModuleFileFromDirectoryCache(filePath); triedToLoadModule = triedToLoadModule || triedToLoad; } } - if (candidates.empty()) { - if (!isRequired) { - loadResult.moduleItem = createNonPresentModule( - *productContext.projectItem->pool(), fullName, QStringLiteral("not found"), - nullptr); - return loadResult; - } - if (Q_UNLIKELY(triedToLoadModule)) { - throw ErrorInfo(Tr::tr("Module %1 could not be loaded.").arg(fullName), - dependsItemLocation); - } - return loadResult; - } + if (candidates.empty()) + return nullptr; + Item *moduleItem = nullptr; if (candidates.size() == 1) { - loadResult.moduleItem = candidates.at(0).item; + moduleItem = candidates.at(0).item; } else { for (auto &candidate : candidates) { - candidate.priority = d->loaderState.evaluator() + ModuleItemLocker lock(*candidate.item); + candidate.priority = m_loaderState.evaluator() .intValue(candidate.item, StringConstants::priorityProperty(), candidate.priority); } - loadResult.moduleItem = chooseModuleCandidate(candidates, fullName); - } - - const QString fullProductName = ProductItemMultiplexer::fullProductDisplayName( - productContext.name, productContext.multiplexId); - const auto it = d->unknownProfilePropertyErrors.find(loadResult.moduleItem); - if (it != d->unknownProfilePropertyErrors.cend()) { - ErrorInfo error(Tr::tr("Loading module '%1' for product '%2' failed due to invalid values " - "in profile '%3':") - .arg(moduleName.toString(), fullProductName, productContext.profile)); - for (const ErrorInfo &e : it->second) - error.append(e.toString()); - handlePropertyError(error, d->loaderState.parameters(), d->loaderState.logger()); + moduleItem = chooseModuleCandidate(candidates, fullName); } - return loadResult; -} - -void ModuleLoader::setStoredModuleProviderInfo(const StoredModuleProviderInfo &moduleProviderInfo) -{ - d->providerLoader.setStoredModuleProviderInfo(moduleProviderInfo); -} - -StoredModuleProviderInfo ModuleLoader::storedModuleProviderInfo() const -{ - return d->providerLoader.storedModuleProviderInfo(); -} - -const Set<QString> &ModuleLoader::tempQbsFiles() const -{ - return d->providerLoader.tempQbsFiles(); + checkForUnknownProfileProperties(moduleItem); + return moduleItem; } -std::pair<Item *, bool> ModuleLoader::Private::loadModuleFile( - const ProductContext &product, const QString &moduleName, const QString &filePath) +std::pair<Item *, bool> ModuleLoader::loadModuleFile(const QString &moduleName, + const QString &filePath) { qCDebug(lcModuleLoader) << "loadModuleFile" << moduleName << "from" << filePath; - const auto [module, triedToLoad] = getModulePrototype(product, moduleName, filePath); + Item * const module = getModulePrototype(moduleName, filePath); if (!module) - return {nullptr, triedToLoad}; + return {nullptr, false}; - const auto key = std::make_pair(module, product.productItem); - const auto it = modulePrototypeEnabledInfo.find(key); - if (it != modulePrototypeEnabledInfo.end()) { + const auto it = m_product.modulePrototypeEnabledInfo.find(module); + if (it != m_product.modulePrototypeEnabledInfo.end()) { qCDebug(lcModuleLoader) << "prototype cache hit (level 2)"; - return {it->second ? module : nullptr, triedToLoad}; + return {it->second ? module : nullptr, true}; } - if (!evaluateModuleCondition(product, module, moduleName)) { + if (!evaluateModuleCondition(module, moduleName)) { qCDebug(lcModuleLoader) << "condition of module" << moduleName << "is false"; - modulePrototypeEnabledInfo.insert({key, false}); - return {nullptr, triedToLoad}; + m_product.modulePrototypeEnabledInfo.insert({module, false}); + return {nullptr, true}; + } + + m_product.modulePrototypeEnabledInfo.insert({module, true}); + return {module, true}; +} + +Item * ModuleLoader::getModulePrototype(const QString &moduleName, const QString &filePath) +{ + bool fromCache = true; + Item * const module = m_loaderState.topLevelProject().getModulePrototype( + filePath, m_product.profileName, [&] { + fromCache = false; + return createAndInitModuleItem(moduleName, filePath); + }); + + if (fromCache) + qCDebug(lcModuleLoader) << "prototype cache hit (level 1)"; + return module; +} + +Item *ModuleLoader::createAndInitModuleItem(const QString &moduleName, const QString &filePath) +{ + Item * const module = m_loaderState.itemReader().setupItemFromFile(filePath, {}); + if (module->type() != ItemType::Module) { + qCDebug(lcModuleLoader).nospace() + << "Alleged module " << moduleName << " has type '" + << module->typeName() << "', so it's not a module after all."; + return nullptr; } + // Not technically needed, but we want to keep the invariant in item.cpp. + ModuleItemLocker locker(*module); + + module->setProperty(StringConstants::nameProperty(), VariantValue::create(moduleName)); if (moduleName == StringConstants::qbsModule()) { module->setProperty(QStringLiteral("hostPlatform"), VariantValue::create(HostOsInfo::hostOSIdentifier())); module->setProperty(QStringLiteral("hostArchitecture"), VariantValue::create(HostOsInfo::hostOSArchitecture())); module->setProperty(QStringLiteral("libexecPath"), - VariantValue::create(loaderState.parameters().libexecPath())); + VariantValue::create(m_loaderState.parameters().libexecPath())); const Version qbsVersion = LanguageInfo::qbsVersion(); module->setProperty(QStringLiteral("versionMajor"), @@ -340,41 +280,17 @@ std::pair<Item *, bool> ModuleLoader::Private::loadModuleFile( for (auto it = paramDecls.begin(); it != paramDecls.end(); ++it) decls.insert(it.key(), it.value()); } - parameterDeclarations.insert({module, decls}); - } - - modulePrototypeEnabledInfo.insert({key, true}); - return {module, triedToLoad}; -} - -std::pair<Item *, bool> ModuleLoader::Private::getModulePrototype( - const ProductContext &product, const QString &moduleName, const QString &filePath) -{ - auto &prototypeList = modulePrototypes[filePath]; - for (const auto &prototype : prototypeList) { - if (prototype.second == product.profile) { - qCDebug(lcModuleLoader) << "prototype cache hit (level 1)"; - return {prototype.first, true}; - } + m_loaderState.topLevelProject().addParameterDeclarations(module, decls); } - Item * const module = loaderState.itemReader().setupItemFromFile(filePath, CodeLocation()); - if (module->type() != ItemType::Module) { - qCDebug(lcModuleLoader).nospace() - << "Alleged module " << moduleName << " has type '" - << module->typeName() << "', so it's not a module after all."; - return {nullptr, false}; - } - prototypeList.emplace_back(module, product.profile); - // Module properties that are defined in the profile are used as default values. // This is the reason we need to have different items per profile. const QVariantMap profileModuleProperties - = product.profileModuleProperties.value(moduleName).toMap(); + = m_product.profileModuleProperties.value(moduleName).toMap(); for (auto it = profileModuleProperties.cbegin(); it != profileModuleProperties.cend(); ++it) { if (Q_UNLIKELY(!module->hasProperty(it.key()))) { - unknownProfilePropertyErrors[module].emplace_back(Tr::tr("Unknown property: %1.%2") - .arg(moduleName, it.key())); + m_loaderState.topLevelProject().addUnknownProfilePropertyError( + module, {Tr::tr("Unknown property: %1.%2").arg(moduleName, it.key())}); continue; } const PropertyDeclaration decl = module->propertyDeclaration(it.key()); @@ -385,14 +301,12 @@ std::pair<Item *, bool> ModuleLoader::Private::getModulePrototype( module->setProperty(it.key(), v); } - return {module, true}; + return module; } -bool ModuleLoader::Private::evaluateModuleCondition(const ProductContext &product, - Item *module, const QString &fullModuleName) +bool ModuleLoader::evaluateModuleCondition(Item *module, const QString &fullModuleName) { - // Evaluator reqires module name to be set. - module->setProperty(StringConstants::nameProperty(), VariantValue::create(fullModuleName)); + ModuleItemLocker locker(*module); // Temporarily make the product's qbs module instance available, so the condition // can use qbs.targetOS etc. @@ -405,7 +319,7 @@ bool ModuleLoader::Private::evaluateModuleCondition(const ProductContext &produc if (m_needsQbsItem) { m_prevQbsItemValue = module->property(StringConstants::qbsModule()); module->setProperty(StringConstants::qbsModule(), - product.productItem->property(StringConstants::qbsModule())); + product.item->property(StringConstants::qbsModule())); } } ~TempQbsModuleProvider() @@ -423,114 +337,63 @@ bool ModuleLoader::Private::evaluateModuleCondition(const ProductContext &produc const bool m_needsQbsItem; }; - const TempQbsModuleProvider tempQbs(product, module, fullModuleName); - return loaderState.evaluator().boolValue(module, StringConstants::conditionProperty()); + const TempQbsModuleProvider tempQbs(m_product, module, fullModuleName); + return m_loaderState.evaluator().boolValue(module, StringConstants::conditionProperty()); } -class DependencyParameterDeclarationCheck +void ModuleLoader::checkForUnknownProfileProperties(const Item *module) { -public: - DependencyParameterDeclarationCheck( - const QString &productName, const Item *productItem, - const std::unordered_map<const Item *, Item::PropertyDeclarationMap> &decls) - : m_productName(productName), m_productItem(productItem), m_parameterDeclarations(decls) - {} - - void operator()(const QVariantMap ¶meters) const { check(parameters, QualifiedId()); } - -private: - void check(const QVariantMap ¶meters, const QualifiedId &moduleName) const - { - for (auto it = parameters.begin(); it != parameters.end(); ++it) { - if (it.value().userType() == QMetaType::QVariantMap) { - check(it.value().toMap(), QualifiedId(moduleName) << it.key()); - } else { - const auto &deps = m_productItem->modules(); - auto m = std::find_if(deps.begin(), deps.end(), - [&moduleName] (const Item::Module &module) { - return module.name == moduleName; - }); - - if (m == deps.end()) { - const QualifiedId fullName = QualifiedId(moduleName) << it.key(); - throw ErrorInfo(Tr::tr("Cannot set parameter '%1', " - "because '%2' does not have a dependency on '%3'.") - .arg(fullName.toString(), m_productName, moduleName.toString()), - m_productItem->location()); - } - - const auto decls = m_parameterDeclarations.find(m->item->rootPrototype()); - if (decls == m_parameterDeclarations.end() || !decls->second.contains(it.key())) { - const QualifiedId fullName = QualifiedId(moduleName) << it.key(); - throw ErrorInfo(Tr::tr("Parameter '%1' is not declared.") - .arg(fullName.toString()), m_productItem->location()); - } - } - } - } - - bool moduleExists(const QualifiedId &name) const - { - const auto &deps = m_productItem->modules(); - return any_of(deps, [&name](const Item::Module &module) { - return module.name == name; - }); - } - - const QString &m_productName; - const Item * const m_productItem; - const std::unordered_map<const Item *, Item::PropertyDeclarationMap> &m_parameterDeclarations; -}; + const std::vector<ErrorInfo> &errors + = m_loaderState.topLevelProject().unknownProfilePropertyErrors(module); + if (errors.empty()) + return; -void ModuleLoader::checkDependencyParameterDeclarations(const Item *productItem, - const QString &productName) const -{ - DependencyParameterDeclarationCheck dpdc(productName, productItem, d->parameterDeclarations); - for (const Item::Module &dep : productItem->modules()) { - if (!dep.parameters.empty()) - dpdc(dep.parameters); - } + ErrorInfo error(Tr::tr("Loading module '%1' for product '%2' failed due to invalid values " + "in profile '%3':") + .arg(m_moduleName.toString(), m_product.displayName(), m_product.profileName)); + for (const ErrorInfo &e : errors) + error.append(e.toString()); + handlePropertyError(error, m_loaderState.parameters(), m_loaderState.logger()); } -void ModuleLoader::forwardParameterDeclarations(const Item *dependsItem, - const Item::Modules &modules) +QString ModuleLoader::findModuleDirectory(const QString &searchPath) { - for (auto it = dependsItem->properties().begin(); it != dependsItem->properties().end(); ++it) { - if (it.value()->type() != Value::ItemValueType) - continue; - d->forwardParameterDeclarations(it.key(), - std::static_pointer_cast<ItemValue>(it.value())->item(), - modules); - } + // isFileCaseCorrect is a very expensive call on macOS, so we cache the value for the + // modules and search paths we've already processed + return m_loaderState.topLevelProject().findModuleDirectory(m_moduleName, searchPath, + [&] { + QString dirPath = searchPath + QStringLiteral("/modules"); + for (const QString &moduleNamePart : m_moduleName) { + dirPath = FileInfo::resolvePath(dirPath, moduleNamePart); + if (!FileInfo::exists(dirPath) || !FileInfo::isFileCaseCorrect(dirPath)) + return QString(); + } + return dirPath; + }); } -void ModuleLoader::Private::forwardParameterDeclarations(const QualifiedId &moduleName, Item *item, - const Item::Modules &modules) +QStringList ModuleLoader::findModuleDirectories() { - auto it = std::find_if(modules.begin(), modules.end(), [&moduleName] (const Item::Module &m) { - return m.name == moduleName; - }); - if (it != modules.end()) { - item->setPropertyDeclarations(parameterDeclarations[it->item->rootPrototype()]); - } else { - for (auto it = item->properties().begin(); it != item->properties().end(); ++it) { - if (it.value()->type() != Value::ItemValueType) - continue; - forwardParameterDeclarations(QualifiedId(moduleName) << it.key(), - std::static_pointer_cast<ItemValue>(it.value())->item(), - modules); - } + const QStringList &searchPaths = m_loaderState.itemReader().allSearchPaths(); + QStringList result; + result.reserve(searchPaths.size()); + for (const auto &path: searchPaths) { + const QString dirPath = findModuleDirectory(path); + if (!dirPath.isEmpty()) + result.append(dirPath); } + return result; } -void ModuleLoader::printProfilingInfo(int indent) +QStringList ModuleLoader::getModuleFilePaths(const QString &dir) { - if (!d->loaderState.parameters().logElapsedTime()) - return; - d->loaderState.logger().qbsLog(LoggerInfo, true) - << QByteArray(indent, ' ') - << Tr::tr("Running module providers took %1.") - .arg(elapsedTimeString(d->elapsedTimeModuleProviders)); + return m_loaderState.topLevelProject().getModuleFilesForDirectory(dir, [&] { + QStringList moduleFiles; + QDirIterator dirIter(dir, StringConstants::qbsFileWildcards()); + while (dirIter.hasNext()) + moduleFiles += dirIter.next(); + return moduleFiles; + }); } } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleloader.h b/src/lib/corelib/loader/moduleloader.h index af5a8c0e9..60567e60e 100644 --- a/src/lib/corelib/loader/moduleloader.h +++ b/src/lib/corelib/loader/moduleloader.h @@ -39,62 +39,18 @@ #pragma once - -#include <language/forward_decls.h> -#include <language/item.h> -#include <tools/pimpl.h> -#include <tools/set.h> - -#include <QString> -#include <QVariantMap> - -#include <vector> - namespace qbs { class CodeLocation; namespace Internal { enum class FallbackMode; +class Item; class LoaderState; -class StoredModuleProviderInfo; - -class ModuleLoader -{ -public: - ModuleLoader(LoaderState &loaderState); - ~ModuleLoader(); - - struct ProductContext { - Item * const productItem; - const Item * const projectItem; - const QString &name; - const QString &uniqueName; - const QString &profile; - const QString &multiplexId; - const QVariantMap &moduleProperties; - const QVariantMap &profileModuleProperties; - }; - struct Result { - Item *moduleItem = nullptr; - std::vector<ProbeConstPtr> providerProbes; - }; - Result searchAndLoadModuleFile(const ProductContext &productContext, - const CodeLocation &dependsItemLocation, - const QualifiedId &moduleName, - FallbackMode fallbackMode, bool isRequired); - - void setStoredModuleProviderInfo(const StoredModuleProviderInfo &moduleProviderInfo); - StoredModuleProviderInfo storedModuleProviderInfo() const; - const Set<QString> &tempQbsFiles() const; - - void checkDependencyParameterDeclarations(const Item *productItem, - const QString &productName) const; - void forwardParameterDeclarations(const Item *dependsItem, const Item::Modules &modules); - void printProfilingInfo(int indent); +class ProductContext; +class QualifiedId; -private: - class Private; - Pimpl<Private> d; -}; +Item *searchAndLoadModuleFile(LoaderState &loaderState, ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, FallbackMode fallbackMode); } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/loader/modulepropertymerger.cpp b/src/lib/corelib/loader/modulepropertymerger.cpp index 17ad63f28..abadd5b34 100644 --- a/src/lib/corelib/loader/modulepropertymerger.cpp +++ b/src/lib/corelib/loader/modulepropertymerger.cpp @@ -52,45 +52,63 @@ #include <unordered_set> namespace qbs::Internal { -class ModulePropertyMerger::Private +class ModulePropertyMerger { public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} + ModulePropertyMerger(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} - int compareValuePriorities(const Item *productItem, const ValueConstPtr &v1, - const ValueConstPtr &v2); - ValuePtr mergeListValues(const Item *productItem, const ValuePtr ¤tHead, const ValuePtr &newElem); - void mergePropertyFromLocalInstance(const Item *productItem, Item *loadingItem, - const QString &loadingName, Item *globalInstance, - const QString &name, const ValuePtr &value); - bool doFinalMerge(const Item *productItem, Item *moduleItem); - bool doFinalMerge(const Item *productItem, const PropertyDeclaration &propertyDecl, - ValuePtr &propertyValue); + void mergeFromLocalInstance(Item *loadingItem, const QString &loadingName, + const Item *localInstance, Item *globalInstance); + void doFinalMerge(); - LoaderState &loaderState; - qint64 elapsedTime = 0; +private: + int compareValuePriorities(const ValueConstPtr &v1, const ValueConstPtr &v2); + ValuePtr mergeListValues(const ValuePtr ¤tHead, const ValuePtr &newElem); + void mergePropertyFromLocalInstance(Item *loadingItem, const QString &loadingName, + Item *globalInstance, const QString &name, + const ValuePtr &value); + bool doFinalMerge(Item *moduleItem); + bool doFinalMerge(const PropertyDeclaration &propertyDecl, ValuePtr &propertyValue); + + ProductContext & m_product; + LoaderState &m_loaderState; }; -void ModulePropertyMerger::mergeFromLocalInstance( - const Item *productItem, Item *loadingItem, const QString &loadingName, - const Item *localInstance, Item *globalInstance) +void mergeFromLocalInstance(ProductContext &product, Item *loadingItem, const QString &loadingName, + const Item *localInstance, Item *globalInstance, + LoaderState &loaderState) +{ + ModulePropertyMerger(product, loaderState).mergeFromLocalInstance( + loadingItem, loadingName, localInstance, globalInstance); +} + +void doFinalMerge(ProductContext &product, LoaderState &loaderState) +{ + ModulePropertyMerger(product, loaderState).doFinalMerge(); +} + +void ModulePropertyMerger::mergeFromLocalInstance(Item *loadingItem, const QString &loadingName, + const Item *localInstance, Item *globalInstance) { - AccumulatingTimer t(d->loaderState.parameters().logElapsedTime() ? &d->elapsedTime : nullptr); + AccumulatingTimer t(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyMerging : nullptr); for (auto it = localInstance->properties().constBegin(); it != localInstance->properties().constEnd(); ++it) { - d->mergePropertyFromLocalInstance(productItem, loadingItem, loadingName, - globalInstance, it.key(), it.value()); + mergePropertyFromLocalInstance(loadingItem, loadingName, globalInstance, it.key(), + it.value()); } } -void ModulePropertyMerger::doFinalMerge(const Item *productItem) +void ModulePropertyMerger::doFinalMerge() { - AccumulatingTimer t(d->loaderState.parameters().logElapsedTime() ? &d->elapsedTime : nullptr); + AccumulatingTimer t(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyMerging : nullptr); std::unordered_set<const Item *> itemsToInvalidate; - for (const Item::Module &module : productItem->modules()) { - if (d->doFinalMerge(productItem, module.item)) + for (const Item::Module &module : m_product.item->modules()) { + if (doFinalMerge(module.item)) itemsToInvalidate.insert(module.item); } @@ -112,35 +130,20 @@ void ModulePropertyMerger::doFinalMerge(const Item *productItem) itemsToInvalidate.insert(item); return addItem || alreadyInSet; }; - collectDependentItems(productItem, collectDependentItems); + collectDependentItems(m_product.item, collectDependentItems); for (const Item * const item : itemsToInvalidate) - d->loaderState.evaluator().clearCache(item); -} - -void ModulePropertyMerger::printProfilingInfo(int indent) -{ - if (!d->loaderState.parameters().logElapsedTime()) - return; - d->loaderState.logger().qbsLog(LoggerInfo, true) - << QByteArray(indent, ' ') - << Tr::tr("Merging module property values took %1.") - .arg(elapsedTimeString(d->elapsedTime)); + m_loaderState.evaluator().clearCache(item); } -ModulePropertyMerger::ModulePropertyMerger(LoaderState &loaderState) - : d(makePimpl<Private>(loaderState)) { } -ModulePropertyMerger::~ModulePropertyMerger() = default; - -int ModulePropertyMerger::Private::compareValuePriorities( - const Item *productItem, const ValueConstPtr &v1, const ValueConstPtr &v2) +int ModulePropertyMerger::compareValuePriorities(const ValueConstPtr &v1, const ValueConstPtr &v2) { QBS_CHECK(v1); QBS_CHECK(v2); QBS_CHECK(v1->scope() != v2->scope()); QBS_CHECK(v1->type() == Value::JSSourceValueType || v2->type() == Value::JSSourceValueType); - const int prio1 = v1->priority(productItem); - const int prio2 = v2->priority(productItem); + const int prio1 = v1->priority(m_product.item); + const int prio2 = v2->priority(m_product.item); if (prio1 != prio2) return prio1 - prio2; const int prioDiff = v1->scopeName().compare(v2->scopeName()); // Sic! See 8ff1dd0044 @@ -148,30 +151,29 @@ int ModulePropertyMerger::Private::compareValuePriorities( return prioDiff; } -ValuePtr ModulePropertyMerger::Private::mergeListValues( - const Item *productItem, const ValuePtr ¤tHead, const ValuePtr &newElem) +ValuePtr ModulePropertyMerger::mergeListValues(const ValuePtr ¤tHead, const ValuePtr &newElem) { QBS_CHECK(newElem); QBS_CHECK(!newElem->next()); if (!currentHead) - return !newElem->expired(productItem) ? newElem : newElem->next(); + return !newElem->expired(m_product.item) ? newElem : newElem->next(); - QBS_CHECK(!currentHead->expired(productItem)); + QBS_CHECK(!currentHead->expired(m_product.item)); - if (newElem->expired(productItem)) + if (newElem->expired(m_product.item)) return currentHead; - if (compareValuePriorities(productItem, currentHead, newElem) < 0) { + if (compareValuePriorities(currentHead, newElem) < 0) { newElem->setNext(currentHead); return newElem; } - currentHead->setNext(mergeListValues(productItem, currentHead->next(), newElem)); + currentHead->setNext(mergeListValues(currentHead->next(), newElem)); return currentHead; } -void ModulePropertyMerger::Private::mergePropertyFromLocalInstance( - const Item *productItem, Item *loadingItem, const QString &loadingName, Item *globalInstance, +void ModulePropertyMerger::mergePropertyFromLocalInstance( + Item *loadingItem, const QString &loadingName, Item *globalInstance, const QString &name, const ValuePtr &value) { const PropertyDeclaration decl = globalInstance->propertyDeclaration(name); @@ -182,9 +184,9 @@ void ModulePropertyMerger::Private::mergePropertyFromLocalInstance( .arg(name), value->location()); } if (const ErrorInfo error = decl.checkForDeprecation( - loaderState.parameters().deprecationWarningMode(), value->location(), - loaderState.logger()); error.hasError()) { - handlePropertyError(error, loaderState.parameters(), loaderState.logger()); + m_loaderState.parameters().deprecationWarningMode(), value->location(), + m_loaderState.logger()); error.hasError()) { + handlePropertyError(error, m_loaderState.parameters(), m_loaderState.logger()); return; } if (value->setInternally()) { // E.g. qbs.architecture after multiplexing. @@ -205,9 +207,9 @@ void ModulePropertyMerger::Private::mergePropertyFromLocalInstance( QBS_CHECK(value->type() == Value::JSSourceValueType); if (decl.isScalar()) { - QBS_CHECK(!globalVal->expired(productItem)); - QBS_CHECK(!value->expired(productItem)); - if (compareValuePriorities(productItem, globalVal, value) < 0) { + QBS_CHECK(!globalVal->expired(m_product.item)); + QBS_CHECK(!value->expired(m_product.item)); + if (compareValuePriorities(globalVal, value) < 0) { value->setCandidates(globalVal->candidates()); globalVal->setCandidates({}); value->addCandidate(globalVal); @@ -216,27 +218,27 @@ void ModulePropertyMerger::Private::mergePropertyFromLocalInstance( globalVal->addCandidate(value); } } else { - if (const ValuePtr &newChainStart = mergeListValues(productItem, globalVal, value); + if (const ValuePtr &newChainStart = mergeListValues(globalVal, value); newChainStart != globalVal) { globalInstance->setProperty(decl.name(), newChainStart); } } } -bool ModulePropertyMerger::Private::doFinalMerge(const Item *productItem, Item *moduleItem) +bool ModulePropertyMerger::doFinalMerge(Item *moduleItem) { if (!moduleItem->isPresentModule()) return false; bool mustInvalidateCache = false; for (auto it = moduleItem->properties().begin(); it != moduleItem->properties().end(); ++it) { - if (doFinalMerge(productItem, moduleItem->propertyDeclaration(it.key()), it.value())) + if (doFinalMerge(moduleItem->propertyDeclaration(it.key()), it.value())) mustInvalidateCache = true; } return mustInvalidateCache; } -bool ModulePropertyMerger::Private::doFinalMerge( - const Item *productItem, const PropertyDeclaration &propertyDecl, ValuePtr &propertyValue) +bool ModulePropertyMerger::doFinalMerge(const PropertyDeclaration &propertyDecl, + ValuePtr &propertyValue) { if (propertyValue->type() == Value::VariantValueType) { QBS_CHECK(!propertyValue->next()); @@ -249,10 +251,10 @@ bool ModulePropertyMerger::Private::doFinalMerge( if (propertyValue->candidates().empty()) return false; std::pair<int, std::vector<ValuePtr>> candidatesWithHighestPrio; - candidatesWithHighestPrio.first = propertyValue->priority(productItem); + candidatesWithHighestPrio.first = propertyValue->priority(m_product.item); candidatesWithHighestPrio.second.push_back(propertyValue); for (const ValuePtr &v : propertyValue->candidates()) { - const int prio = v->priority(productItem); + const int prio = v->priority(m_product.item); if (prio < candidatesWithHighestPrio.first) continue; if (prio > candidatesWithHighestPrio.first) { @@ -280,7 +282,7 @@ bool ModulePropertyMerger::Private::doFinalMerge( error.append({}, v->location()); } if (error.items().size() > 2) - loaderState.logger().printWarning(error); + m_loaderState.logger().printWarning(error); } if (propertyValue == chosenValue) @@ -300,7 +302,7 @@ bool ModulePropertyMerger::Private::doFinalMerge( } ValuePtr newValue; for (const ValuePtr &v : singleValuesBefore) - newValue = mergeListValues(productItem, newValue, v); + newValue = mergeListValues(newValue, v); std::vector<ValuePtr> singleValuesAfter; for (ValuePtr current = propertyValue; current; current = current->next()) singleValuesAfter.push_back(current); diff --git a/src/lib/corelib/loader/modulepropertymerger.h b/src/lib/corelib/loader/modulepropertymerger.h index 4ea72c2a4..a6643216f 100644 --- a/src/lib/corelib/loader/modulepropertymerger.h +++ b/src/lib/corelib/loader/modulepropertymerger.h @@ -39,15 +39,18 @@ #pragma once -#include <tools/pimpl.h> - #include <QtGlobal> +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + namespace qbs::Internal { class Item; class LoaderState; +class ProductContext; -// This class comprises functions for collecting values attached to module properties +// This module comprises functions for collecting values attached to module properties // in different contexts. // For example, in the Qt.core module you will find a property binding such as this: // cpp.defines: "QT_CORE_LIB" @@ -62,35 +65,23 @@ class LoaderState; // with the same priority trigger a warning message. // Since the right-hand side of a binding can refer to properties of the surrounding context, // each such value gets its own scope. -class ModulePropertyMerger -{ -public: - ModulePropertyMerger(LoaderState &loaderState); - ~ModulePropertyMerger(); - - // This function is called when a module is loaded via a Depends item. - // loadingItem is the product or module containing the Depends item. - // loadingName is the name of that module. It is used as a tie-breaker for list property values - // with equal priority. - // localInstance is the module instance placeholder in the ItemValue of a property binding, - // i.e. the "cpp" in "cpp.defines". - // globalInstance is the actual module into which the properties from localInstance get merged. - void mergeFromLocalInstance(const Item *productItem, Item *loadingItem, - const QString &loadingName, const Item *localInstance, - Item *globalInstance); - - // This function is called after all dependencies have been resolved. It uses its global - // knowledge of module priorities to potentially adjust the order of list values or - // favor different scalar values. It can also remove previously merged-in values again; - // this can happen if a module fails to load after it already merged some values, or - // if it fails validation in the end. - void doFinalMerge(const Item *productItem); - void printProfilingInfo(int indent); +// This function is called when a module is loaded via a Depends item. +// loadingItem is the product or module containing the Depends item. +// loadingName is the name of that module. It is used as a tie-breaker for list property values +// with equal priority. +// localInstance is the module instance placeholder in the ItemValue of a property binding, +// i.e. the "cpp" in "cpp.defines". +// globalInstance is the actual module into which the properties from localInstance get merged. +void mergeFromLocalInstance(ProductContext &product, Item *loadingItem, + const QString &loadingName, const Item *localInstance, + Item *globalInstance, LoaderState &loaderState); -private: - class Private; - Pimpl<Private> d; -}; +// This function is called after all dependencies have been resolved. It uses its global +// knowledge of module priorities to potentially adjust the order of list values or +// favor different scalar values. It can also remove previously merged-in values again; +// this can happen if a module fails to load after it already merged some values, or +// if it fails validation in the end. +void doFinalMerge(ProductContext &product, LoaderState &loaderState); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/moduleproviderloader.cpp b/src/lib/corelib/loader/moduleproviderloader.cpp index d03a165f4..1d4af1896 100644 --- a/src/lib/corelib/loader/moduleproviderloader.cpp +++ b/src/lib/corelib/loader/moduleproviderloader.cpp @@ -57,80 +57,84 @@ #include <tools/stlutils.h> #include <tools/stringconstants.h> -#include <QtCore/qtemporaryfile.h> +#include <QtCore/qcryptographichash.h> +#include <QtCore/qjsondocument.h> +#include <QtCore/qjsonobject.h> namespace qbs { namespace Internal { +static QString getConfigHash(const QVariantMap& config) +{ + QJsonDocument doc; + doc.setObject(QJsonObject::fromVariantMap(config)); + return QString::fromLatin1( + QCryptographicHash::hash(doc.toJson(), QCryptographicHash::Sha1).toHex().left(16)); +} + ModuleProviderLoader::ModuleProviderLoader(LoaderState &loaderState) : m_loaderState(loaderState) {} ModuleProviderLoader::ModuleProviderResult ModuleProviderLoader::executeModuleProviders( - const ProductContext &productContext, + ProductContext &productContext, const CodeLocation &dependsItemLocation, const QualifiedId &moduleName, FallbackMode fallbackMode) { ModuleProviderLoader::ModuleProviderResult result; - std::vector<Provider> providersToRun; - qCDebug(lcModuleLoader) << "Module" << moduleName.toString() - << "not found, checking for module providers"; - const auto providerNames = getModuleProviders(productContext.productItem); - if (providerNames) { - providersToRun = transformed<std::vector<Provider>>(*providerNames, [](const auto &name) { - return Provider{name, ModuleProviderLookup::Named}; }); - } else { - for (QualifiedId providerName = moduleName; !providerName.empty(); - providerName.pop_back()) { - providersToRun.push_back({providerName, ModuleProviderLookup::Scoped}); + try { + std::vector<Provider> providersToRun; + qCDebug(lcModuleLoader) << "Module" << moduleName.toString() + << "not found, checking for module providers"; + const auto providerNames = getModuleProviders(productContext.item); + if (providerNames) { + providersToRun = transformed<std::vector<Provider>>(*providerNames, [](const auto &name) { + return Provider{name, ModuleProviderLookup::Named}; }); + } else { + for (QualifiedId providerName = moduleName; !providerName.empty(); + providerName.pop_back()) { + providersToRun.push_back({providerName, ModuleProviderLookup::Scoped}); + } } - } - result = executeModuleProvidersHelper(productContext, dependsItemLocation, providersToRun); - - if (fallbackMode == FallbackMode::Enabled - && !result.providerFound - && !providerNames) { - qCDebug(lcModuleLoader) << "Specific module provider not found for" - << moduleName.toString() << ", setting up fallback."; result = executeModuleProvidersHelper( - productContext, - dependsItemLocation, - {{moduleName, ModuleProviderLookup::Fallback}}); - } + productContext, dependsItemLocation, moduleName, providersToRun); + if (fallbackMode == FallbackMode::Enabled + && !result.providerFound + && !providerNames) { + qCDebug(lcModuleLoader) << "Specific module provider not found for" + << moduleName.toString() << ", setting up fallback."; + result = executeModuleProvidersHelper( + productContext, + dependsItemLocation, + moduleName, + {{moduleName, ModuleProviderLookup::Fallback}}); + } + } catch (const ErrorInfo &error) { + auto ei = error; + ei.prepend( + Tr::tr("Error executing provider for module '%1':").arg(moduleName.toString()), + dependsItemLocation); + productContext.handleError(ei); + } return result; } ModuleProviderLoader::ModuleProviderResult ModuleProviderLoader::executeModuleProvidersHelper( - const ProductContext &product, + ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const std::vector<Provider> &providers) { if (providers.empty()) return {}; QStringList allSearchPaths; ModuleProviderResult result; - result.providerConfig = product.providerConfig ? *product.providerConfig - : getModuleProviderConfig(product); + setupModuleProviderConfig(product); const auto qbsModule = evaluateQbsModule(product); for (const auto &[name, lookupType] : providers) { - const QVariantMap config = result.providerConfig.value(name.toString()).toMap(); - ModuleProviderInfo &info = m_storedModuleProviderInfo.providers[ - {name.toString(), config, qbsModule, int(lookupType)}]; - const bool fromCache = !info.name.isEmpty(); - if (!fromCache) { - info.name = name; - info.config = config; - info.providerFile = findModuleProviderFile(name, lookupType); - if (!info.providerFile.isEmpty()) { - qCDebug(lcModuleLoader) << "Running provider" << name << "at" << info.providerFile; - const auto evalResult = evaluateModuleProvider( - product, dependsItemLocation, name, info.providerFile, config, qbsModule); - info.searchPaths = evalResult.first; - result.probes << evalResult.second; - info.transientOutput = m_loaderState.parameters().dryRun(); - } - } + const auto &[info, fromCache] = findOrCreateProviderInfo( + product, dependsItemLocation, moduleName, name, lookupType, qbsModule); if (info.providerFile.isEmpty()) { if (lookupType == ModuleProviderLookup::Named) throw ErrorInfo(Tr::tr("Unknown provider '%1'").arg(name.toString())); @@ -158,11 +162,51 @@ ModuleProviderLoader::ModuleProviderResult ModuleProviderLoader::executeModulePr return result; } -QVariantMap ModuleProviderLoader::getModuleProviderConfig(const ProductContext &product) +std::pair<const ModuleProviderInfo &, bool /*fromCache*/> +ModuleProviderLoader::findOrCreateProviderInfo( + ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const QualifiedId &name, ModuleProviderLookup lookupType, + const QVariantMap &qbsModule) +{ + const QVariantMap config = product.providerConfig->value(name.toString()).toMap(); + std::lock_guard lock(m_loaderState.topLevelProject().moduleProvidersCacheLock()); + ModuleProvidersCacheKey cacheKey{name.toString(), {}, config, qbsModule, int(lookupType)}; + // TODO: get rid of non-eager providers and eliminate following if-logic + // first, try to find eager provider (stored with an empty module name) + if (ModuleProviderInfo *provider = m_loaderState.topLevelProject().moduleProvider(cacheKey)) + return {*provider, true}; + // second, try to find non-eager provider for a specific module name + std::get<1>(cacheKey) = moduleName.toString(); // override moduleName + if (ModuleProviderInfo *provider = m_loaderState.topLevelProject().moduleProvider(cacheKey)) + return {*provider, true}; + bool isEager = false; + ModuleProviderInfo info; + info.name = name; + info.config = config; + info.providerFile = findModuleProviderFile(name, lookupType); + if (!info.providerFile.isEmpty()) { + qCDebug(lcModuleLoader) << "Running provider" << name << "at" << info.providerFile; + std::tie(info.searchPaths, isEager) = evaluateModuleProvider( + product, + dependsItemLocation, + moduleName, + name, + info.providerFile, + config, + qbsModule); + info.transientOutput = m_loaderState.parameters().dryRun(); + } + std::get<1>(cacheKey) = isEager ? QString() : moduleName.toString(); + return {m_loaderState.topLevelProject().addModuleProvider(cacheKey, info), false}; +} + +void ModuleProviderLoader::setupModuleProviderConfig(ProductContext &product) { + if (product.providerConfig) + return; QVariantMap providerConfig; const ItemValueConstPtr configItemValue = - product.productItem->itemProperty(StringConstants::moduleProviders()); + product.item->itemProperty(StringConstants::moduleProviders(), m_loaderState.itemPool()); if (configItemValue) { const std::function<void(const Item *, QualifiedId)> collectMap = [this, &providerConfig, &collectMap](const Item *item, const QualifiedId &name) { @@ -192,7 +236,7 @@ QVariantMap ModuleProviderLoader::getModuleProviderConfig(const ProductContext & providerConfig.insert(name.toString(), m); } }; - configItemValue->item()->setScope(product.productItem); + configItemValue->item()->setScope(product.item); collectMap(configItemValue->item(), QualifiedId()); } for (auto it = product.moduleProperties.begin(); it != product.moduleProperties.end(); ++it) { @@ -209,7 +253,7 @@ QVariantMap ModuleProviderLoader::getModuleProviderConfig(const ProductContext & } providerConfig.insert(provider, currentMapForProvider); } - return providerConfig; + product.providerConfig = providerConfig; } std::optional<std::vector<QualifiedId>> ModuleProviderLoader::getModuleProviders(Item *item) @@ -266,7 +310,7 @@ QVariantMap ModuleProviderLoader::evaluateQbsModule(const ProductContext &produc QStringLiteral("toolchain"), }; const auto qbsItemValue = std::static_pointer_cast<ItemValue>( - product.productItem->property(StringConstants::qbsModule())); + product.item->property(StringConstants::qbsModule())); QVariantMap result; for (const auto &property : properties) { const ScopedJsValue val(m_loaderState.evaluator().engine()->context(), @@ -285,39 +329,35 @@ QVariantMap ModuleProviderLoader::evaluateQbsModule(const ProductContext &produc return result; } -Item *ModuleProviderLoader::createProviderScope(const ProductContext &product, const QVariantMap &qbsModule) +Item *ModuleProviderLoader::createProviderScope( + const ProductContext &product, const QVariantMap &qbsModule) { const auto qbsItemValue = std::static_pointer_cast<ItemValue>( - product.productItem->property(StringConstants::qbsModule())); + product.item->property(StringConstants::qbsModule())); - Item *fakeQbsModule = Item::create(product.productItem->pool(), ItemType::Scope); + Item *fakeQbsModule = Item::create(&m_loaderState.itemPool(), ItemType::Scope); for (auto it = qbsModule.begin(), end = qbsModule.end(); it != end; ++it) { fakeQbsModule->setProperty(it.key(), VariantValue::create(it.value())); } - Item *scope = Item::create(product.productItem->pool(), ItemType::Scope); + Item *scope = Item::create(&m_loaderState.itemPool(), ItemType::Scope); scope->setFile(qbsItemValue->item()->file()); scope->setProperty(StringConstants::qbsModule(), ItemValue::create(fakeQbsModule)); return scope; } -std::pair<QStringList, std::vector<ProbeConstPtr> > ModuleProviderLoader::evaluateModuleProvider(const ProductContext &product, +ModuleProviderLoader::EvaluationResult ModuleProviderLoader::evaluateModuleProvider( + ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const QualifiedId &name, const QString &providerFile, const QVariantMap &moduleConfig, const QVariantMap &qbsModule) { - QTemporaryFile dummyItemFile; - if (!dummyItemFile.open()) { - throw ErrorInfo(Tr::tr("Failed to create temporary file for running module provider " - "for dependency '%1': %2").arg(name.toString(), - dummyItemFile.errorString())); - } - m_tempQbsFiles << dummyItemFile.fileName(); qCDebug(lcModuleLoader) << "Instantiating module provider at" << providerFile; - const QString projectBuildDir = product.projectItem->variantProperty( + const QString projectBuildDir = product.project->item->variantProperty( StringConstants::buildDirectoryProperty())->value().toString(); const QString searchPathBaseDir = ModuleProviderInfo::outputDirPath(projectBuildDir, name); @@ -325,25 +365,9 @@ std::pair<QStringList, std::vector<ProbeConstPtr> > ModuleProviderLoader::evalua auto jsConfig = moduleConfig; jsConfig[StringConstants::qbsModule()] = qbsModule; - QTextStream stream(&dummyItemFile); - using Qt::endl; - setupDefaultCodec(stream); - stream << "import qbs.FileInfo" << endl; - stream << "import qbs.Utilities" << endl; - stream << "import '" << providerFile << "' as Provider" << endl; - stream << "Provider {" << endl; - stream << " name: " << toJSLiteral(name.toString()) << endl; - stream << " property var config: (" << toJSLiteral(jsConfig) << ')' << endl; - stream << " outputBaseDir: FileInfo.joinPaths(baseDirPrefix, " - " Utilities.getHash(JSON.stringify(config)))" << endl; - stream << " property string baseDirPrefix: " << toJSLiteral(searchPathBaseDir) << endl; - stream << " property stringList searchPaths: (relativeSearchPaths || [])" - " .map(function(p) { return FileInfo.joinPaths(outputBaseDir, p); })" - << endl; - stream << "}" << endl; - stream.flush(); + QString outputBaseDir = searchPathBaseDir + QLatin1Char('/') + getConfigHash(jsConfig); Item * const providerItem = m_loaderState.itemReader().setupItemFromFile( - dummyItemFile.fileName(), dependsItemLocation); + providerFile, dependsItemLocation); if (providerItem->type() != ItemType::ModuleProvider) { throw ErrorInfo(Tr::tr("File '%1' declares an item of type '%2', " "but '%3' was expected.") @@ -352,16 +376,34 @@ std::pair<QStringList, std::vector<ProbeConstPtr> > ModuleProviderLoader::evalua } providerItem->setScope(createProviderScope(product, qbsModule)); - providerItem->overrideProperties(moduleConfig, name, m_loaderState.parameters(), - m_loaderState.logger()); - std::vector<ProbeConstPtr> probes = m_loaderState.probesResolver().resolveProbes( - {product.name, product.uniqueName}, providerItem); + providerItem->setProperty( + StringConstants::nameProperty(), + VariantValue::create(name.toString())); + providerItem->setProperty( + QStringLiteral("outputBaseDir"), + VariantValue::create(outputBaseDir)); + providerItem->overrideProperties(moduleConfig, name, + m_loaderState.parameters(), m_loaderState.logger()); + + const bool isEager = m_loaderState.evaluator().boolValue( + providerItem, StringConstants::isEagerProperty()); + if (!isEager) { + providerItem->setProperty( + StringConstants::moduleNameProperty(), + VariantValue::create(moduleName.toString())); + } + + ProbesResolver(m_loaderState).resolveProbes(product, providerItem); EvalContextSwitcher contextSwitcher(m_loaderState.evaluator().engine(), EvalContext::ModuleProvider); - return std::make_pair(m_loaderState.evaluator().stringListValue( - providerItem, QStringLiteral("searchPaths")), - std::move(probes)); + auto searchPaths = m_loaderState.evaluator().stringListValue( + providerItem, QStringLiteral("relativeSearchPaths")); + auto prependBaseDir = [&outputBaseDir](const auto &path) { + return outputBaseDir + QLatin1Char('/') + path; + }; + std::transform(searchPaths.begin(), searchPaths.end(), searchPaths.begin(), prependBaseDir); + return {searchPaths, isEager}; } } // namespace Internal diff --git a/src/lib/corelib/loader/moduleproviderloader.h b/src/lib/corelib/loader/moduleproviderloader.h index 221830f22..5331bd0d0 100644 --- a/src/lib/corelib/loader/moduleproviderloader.h +++ b/src/lib/corelib/loader/moduleproviderloader.h @@ -46,87 +46,65 @@ #include <language/forward_decls.h> #include <language/moduleproviderinfo.h> -#include <QtCore/qmap.h> #include <QtCore/qvariant.h> #include <optional> +#include <utility> #include <vector> namespace qbs::Internal { class Item; class LoaderState; +class ProductContext; class ModuleProviderLoader { public: explicit ModuleProviderLoader(LoaderState &loaderState); - enum class ModuleProviderLookup { Scoped, Named, Fallback }; - - struct Provider - { - QualifiedId name; - ModuleProviderLookup lookup; - }; - - struct ModuleProviderResult - { - std::vector<ProbeConstPtr> probes; - QVariantMap providerConfig; + struct ModuleProviderResult { bool providerFound = false; std::optional<QStringList> searchPaths; }; - - const StoredModuleProviderInfo &storedModuleProviderInfo() const - { - return m_storedModuleProviderInfo; - } - - void setStoredModuleProviderInfo(StoredModuleProviderInfo moduleProviderInfo) - { - m_storedModuleProviderInfo = std::move(moduleProviderInfo); - } - - const Set<QString> &tempQbsFiles() const { return m_tempQbsFiles; } - - struct ProductContext { - Item * const productItem; - const Item * const projectItem; - const QString &name; - const QString &uniqueName; - const QVariantMap &moduleProperties; - const std::optional<QVariantMap> providerConfig; - }; ModuleProviderResult executeModuleProviders( - const ProductContext &productContext, + ProductContext &productContext, const CodeLocation &dependsItemLocation, const QualifiedId &moduleName, FallbackMode fallbackMode); private: + enum class ModuleProviderLookup { Scoped, Named, Fallback }; + struct Provider { + QualifiedId name; + ModuleProviderLookup lookup; + }; ModuleProviderResult executeModuleProvidersHelper( - const ProductContext &product, + ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const std::vector<Provider> &providers); - QVariantMap getModuleProviderConfig(const ProductContext &product); + std::pair<const ModuleProviderInfo &, bool> + findOrCreateProviderInfo(ProductContext &product, const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const QualifiedId &name, + ModuleProviderLookup lookupType, const QVariantMap &qbsModule); + void setupModuleProviderConfig(ProductContext &product); std::optional<std::vector<QualifiedId>> getModuleProviders(Item *item); QString findModuleProviderFile(const QualifiedId &name, ModuleProviderLookup lookupType); QVariantMap evaluateQbsModule(const ProductContext &product) const; Item *createProviderScope(const ProductContext &product, const QVariantMap &qbsModule); - std::pair<QStringList, std::vector<ProbeConstPtr>> evaluateModuleProvider( - const ProductContext &product, - const CodeLocation &location, + using EvaluationResult = std::pair<QStringList, bool /*isEager*/>; + EvaluationResult evaluateModuleProvider( + ProductContext &product, + const CodeLocation &dependsItemLocation, + const QualifiedId &moduleName, const QualifiedId &name, const QString &providerFile, const QVariantMap &moduleConfig, const QVariantMap &qbsModule); -private: LoaderState &m_loaderState; - StoredModuleProviderInfo m_storedModuleProviderInfo; - Set<QString> m_tempQbsFiles; }; } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/probesresolver.cpp b/src/lib/corelib/loader/probesresolver.cpp index 763f3ed29..b38366900 100644 --- a/src/lib/corelib/loader/probesresolver.cpp +++ b/src/lib/corelib/loader/probesresolver.cpp @@ -83,40 +83,29 @@ static QString probeGlobalId(Item *probe) ProbesResolver::ProbesResolver(LoaderState &loaderState) : m_loaderState(loaderState) {} -void ProbesResolver::setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes) -{ - m_oldProjectProbes.clear(); - for (const ProbeConstPtr& probe : oldProbes) - m_oldProjectProbes[probe->globalId()] << probe; -} - -void ProbesResolver::setOldProductProbes( - const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes) -{ - m_oldProductProbes = oldProbes; -} - -std::vector<ProbeConstPtr> ProbesResolver::resolveProbes(const ProductContext &productContext, Item *item) +void ProbesResolver::resolveProbes(ProductContext &productContext, Item *item) { AccumulatingTimer probesTimer(m_loaderState.parameters().logElapsedTime() - ? &m_elapsedTimeProbes : nullptr); + ? &productContext.timingData.probes : nullptr); + EvalContextSwitcher evalContextSwitcher(m_loaderState.evaluator().engine(), EvalContext::ProbeExecution); - std::vector<ProbeConstPtr> probes; - for (Item * const child : item->children()) + for (Item * const child : item->children()) { if (child->type() == ItemType::Probe) - probes.push_back(resolveProbe(productContext, item, child)); - return probes; + resolveProbe(productContext, item, child); + } } -ProbeConstPtr ProbesResolver::resolveProbe(const ProductContext &productContext, Item *parent, +void ProbesResolver::resolveProbe(ProductContext &productContext, Item *parent, Item *probe) { qCDebug(lcModuleLoader) << "Resolving Probe at " << probe->location().toString(); - ++m_probesEncountered; const QString &probeId = probeGlobalId(probe); if (Q_UNLIKELY(probeId.isEmpty())) throw ErrorInfo(Tr::tr("Probe.id must be set."), probe->location()); + const bool isProjectLevelProbe + = parent->type() == ItemType::Project + || productContext.name.startsWith(StringConstants::shadowProductPrefix()); const JSSourceValueConstPtr configureScript = probe->sourceProperty(StringConstants::configureProperty()); QBS_CHECK(configureScript); @@ -143,29 +132,30 @@ ProbeConstPtr ProbesResolver::resolveProbe(const ProductContext &productContext, const bool condition = evaluator.boolValue(probe, StringConstants::conditionProperty()); const QString &sourceCode = configureScript->sourceCode().toString(); ProbeConstPtr resolvedProbe; - if (parent->type() == ItemType::Project - || productContext.name.startsWith(StringConstants::shadowProductPrefix())) { + std::lock_guard lock(m_loaderState.topLevelProject().probesCacheLock()); + m_loaderState.topLevelProject().incrementProbesCount(); + if (isProjectLevelProbe) { resolvedProbe = findOldProjectProbe(probeId, condition, initialProperties, sourceCode); } else { - resolvedProbe = findOldProductProbe(productContext.uniqueName, condition, + resolvedProbe = findOldProductProbe(productContext.uniqueName(), condition, initialProperties, sourceCode); } if (!resolvedProbe) { resolvedProbe = findCurrentProbe(probe->location(), condition, initialProperties); if (resolvedProbe) { qCDebug(lcModuleLoader) << "probe results cached from current run"; - ++m_probesCachedCurrent; + m_loaderState.topLevelProject().incrementReusedCurrentProbesCount(); } } else { qCDebug(lcModuleLoader) << "probe results cached from earlier run"; - ++m_probesCachedOld; + m_loaderState.topLevelProject().incrementReusedOldProbesCount(); } ScopedJsValue configureScope(ctx, JS_UNDEFINED); std::vector<QString> importedFilesUsedInConfigure; if (!condition) { qCDebug(lcModuleLoader) << "Probe disabled; skipping"; } else if (!resolvedProbe) { - ++m_probesRun; + m_loaderState.topLevelProject().incrementRunProbesCount(); qCDebug(lcModuleLoader) << "configure script needs to run"; const Evaluator::FileContextScopes fileCtxScopes = evaluator.fileContextScopes(configureScript->file()); @@ -183,6 +173,8 @@ ProbeConstPtr ProbesResolver::resolveProbe(const ProductContext &productContext, importedFilesUsedInConfigure = resolvedProbe->importedFilesUsed(); } QVariantMap properties; + VariantValuePtr storedValue; + QMap<QString, VariantValuePtr> storedValues; for (const ProbeProperty &b : probeBindings) { QVariant newValue; if (resolvedProbe) { @@ -206,22 +198,43 @@ ProbeConstPtr ProbesResolver::resolveProbe(const ProductContext &productContext, if (JsException ex = engine->checkAndClearException({})) throw ex.toErrorInfo(); newValue = getJsVariant(ctx, v); + // special case, string lists are represented as js arrays and and we don't type + // info when converting + if (decl.type() == PropertyDeclaration::StringList + && newValue.userType() == QMetaType::QVariantList) { +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) + newValue.convert(QMetaType(QMetaType::QStringList)); +#else + newValue.convert(QMetaType::QStringList); +#endif + } } else { newValue = initialProperties.value(b.first); } } - if (newValue != getJsVariant(ctx, b.second)) - probe->setProperty(b.first, VariantValue::create(newValue)); - if (!resolvedProbe) + if (newValue != getJsVariant(ctx, b.second)) { + if (!resolvedProbe) + storedValue = VariantValue::createStored(newValue); + else + storedValue = resolvedProbe->values().value(b.first); + + probe->setProperty(b.first, storedValue); + } + if (!resolvedProbe) { properties.insert(b.first, newValue); + storedValues[b.first] = storedValue; + } } if (!resolvedProbe) { resolvedProbe = Probe::create(probeId, probe->location(), condition, - sourceCode, properties, initialProperties, + sourceCode, properties, initialProperties, storedValues, importedFilesUsedInConfigure); - m_currentProbes[probe->location()] << resolvedProbe; + m_loaderState.topLevelProject().addNewlyResolvedProbe(resolvedProbe); } - return resolvedProbe; + if (isProjectLevelProbe) + m_loaderState.topLevelProject().addProjectLevelProbe(resolvedProbe); + else + productContext.probes << resolvedProbe; } ProbeConstPtr ProbesResolver::findOldProjectProbe( @@ -232,13 +245,10 @@ ProbeConstPtr ProbesResolver::findOldProjectProbe( { if (m_loaderState.parameters().forceProbeExecution()) return {}; - - for (const ProbeConstPtr &oldProbe : m_oldProjectProbes.value(globalId)) { - if (probeMatches(oldProbe, condition, initialProperties, sourceCode, CompareScript::Yes)) - return oldProbe; - } - - return {}; + return m_loaderState.topLevelProject().findOldProjectProbe(globalId, + [&](const ProbeConstPtr &oldProbe) { + return probeMatches(oldProbe, condition, initialProperties, sourceCode, CompareScript::Yes); + }); } ProbeConstPtr ProbesResolver::findOldProductProbe( @@ -249,13 +259,10 @@ ProbeConstPtr ProbesResolver::findOldProductProbe( { if (m_loaderState.parameters().forceProbeExecution()) return {}; - - for (const ProbeConstPtr &oldProbe : m_oldProductProbes.value(productName)) { - if (probeMatches(oldProbe, condition, initialProperties, sourceCode, CompareScript::Yes)) - return oldProbe; - } - - return {}; + return m_loaderState.topLevelProject().findOldProductProbe(productName, + [&](const ProbeConstPtr &oldProbe) { + return probeMatches(oldProbe, condition, initialProperties, sourceCode, CompareScript::Yes); + }); } ProbeConstPtr ProbesResolver::findCurrentProbe( @@ -263,12 +270,10 @@ ProbeConstPtr ProbesResolver::findCurrentProbe( bool condition, const QVariantMap &initialProperties) const { - const std::vector<ProbeConstPtr> &cachedProbes = m_currentProbes.value(location); - for (const ProbeConstPtr &probe : cachedProbes) { - if (probeMatches(probe, condition, initialProperties, QString(), CompareScript::No)) - return probe; - } - return {}; + return m_loaderState.topLevelProject().findCurrentProbe(location, + [&](const ProbeConstPtr &probe) { + return probeMatches(probe, condition, initialProperties, QString(), CompareScript::No); + }); } bool ProbesResolver::probeMatches(const ProbeConstPtr &probe, bool condition, @@ -279,23 +284,7 @@ bool ProbesResolver::probeMatches(const ProbeConstPtr &probe, bool condition, && probe->initialProperties() == initialProperties && (compareScript == CompareScript::No || (probe->configureScript() == configureScript - && !probe->needsReconfigure(m_lastResolveTime))); -} - -void ProbesResolver::printProfilingInfo(int indent) -{ - if (!m_loaderState.parameters().logElapsedTime()) - return; - const QByteArray prefix(indent, ' '); - m_loaderState.logger().qbsLog(LoggerInfo, true) - << prefix - << Tr::tr("Running Probes took %1.").arg(elapsedTimeString(m_elapsedTimeProbes)); - m_loaderState.logger().qbsLog(LoggerInfo, true) - << prefix - << Tr::tr("%1 probes encountered, %2 configure scripts executed, " - "%3 re-used from current run, %4 re-used from earlier run.") - .arg(m_probesEncountered).arg(m_probesRun).arg(m_probesCachedCurrent) - .arg(m_probesCachedOld); + && !probe->needsReconfigure(m_loaderState.topLevelProject().lastResolveTime()))); } } // namespace Internal diff --git a/src/lib/corelib/loader/probesresolver.h b/src/lib/corelib/loader/probesresolver.h index 3e304eec6..4c49861d3 100644 --- a/src/lib/corelib/loader/probesresolver.h +++ b/src/lib/corelib/loader/probesresolver.h @@ -42,30 +42,20 @@ #define PROBESRESOLVER_H #include <language/forward_decls.h> - -#include <tools/filetime.h> +#include <tools/codelocation.h> #include <QString> -#include <vector> - namespace qbs::Internal { class Item; class LoaderState; +class ProductContext; class ProbesResolver { public: explicit ProbesResolver(LoaderState &loaderState); - void setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes); - void setOldProductProbes(const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes); - void printProfilingInfo(int indent); - - struct ProductContext { - const QString &name; - const QString &uniqueName; - }; - std::vector<ProbeConstPtr> resolveProbes(const ProductContext &productContext, Item *item); + void resolveProbes(ProductContext &productContext, Item *item); private: ProbeConstPtr findOldProjectProbe(const QString &globalId, bool condition, @@ -80,19 +70,9 @@ private: bool probeMatches(const ProbeConstPtr &probe, bool condition, const QVariantMap &initialProperties, const QString &configureScript, CompareScript compareScript) const; - ProbeConstPtr resolveProbe(const ProductContext &productContext, Item *parent, Item *probe); - - qint64 m_elapsedTimeProbes = 0; - quint64 m_probesEncountered = 0; - quint64 m_probesRun = 0; - quint64 m_probesCachedCurrent = 0; - quint64 m_probesCachedOld = 0; + void resolveProbe(ProductContext &productContext, Item *parent, Item *probe); LoaderState &m_loaderState; - QHash<QString, std::vector<ProbeConstPtr>> m_oldProjectProbes; - QHash<QString, std::vector<ProbeConstPtr>> m_oldProductProbes; - FileTime m_lastResolveTime; - QHash<CodeLocation, std::vector<ProbeConstPtr>> m_currentProbes; }; } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productitemmultiplexer.cpp b/src/lib/corelib/loader/productitemmultiplexer.cpp index f18c37caf..0adb3d7fd 100644 --- a/src/lib/corelib/loader/productitemmultiplexer.cpp +++ b/src/lib/corelib/loader/productitemmultiplexer.cpp @@ -40,6 +40,7 @@ #include "productitemmultiplexer.h" #include "loaderutils.h" +#include "moduleinstantiator.h" #include <language/evaluator.h> #include <language/item.h> @@ -51,14 +52,14 @@ #include <tools/stringconstants.h> #include <QJsonDocument> -#include <QThreadStorage> #include <vector> - +// This module deals with product multiplexing over the various defined axes. +// For instance, a product with qbs.architectures: ["x86", "arm"] will get multiplexed into +// two products with qbs.architecture: "x86" and qbs.architecture: "arm", respectively. namespace qbs::Internal { namespace { -using MultiplexConfigurationByIdTable = QThreadStorage<QHash<QString, QVariantMap>>; using MultiplexRow = std::vector<VariantValuePtr>; using MultiplexTable = std::vector<MultiplexRow>; class MultiplexInfo @@ -69,54 +70,50 @@ public: bool aggregate = false; VariantValuePtr multiplexedType; - QString toIdString(size_t row) const; + QString toIdString(size_t row, LoaderState &loaderState) const; }; } // namespace -Q_GLOBAL_STATIC(MultiplexConfigurationByIdTable, multiplexConfigurationsById); - -class ProductItemMultiplexer::Private +class ProductItemMultiplexer { public: - Private(LoaderState &loaderState, QbsItemRetriever qbsItemRetriever) - : loaderState(loaderState), qbsItemRetriever(std::move(qbsItemRetriever)) {} - - MultiplexInfo extractMultiplexInfo(Item *productItem, Item *qbsModuleItem); + ProductItemMultiplexer(const QString &productName, Item *productItem, Item *tempQbsModuleItem, + const std::function<void()> &dropTempQbsModule, LoaderState &loaderState) + : m_loaderState(loaderState), m_productName(productName), m_productItem(productItem), + m_tempQbsModuleItem(tempQbsModuleItem), m_dropTempQbsModule(dropTempQbsModule) {} + + QList<Item *> multiplex(); +private: + MultiplexInfo extractMultiplexInfo(); MultiplexTable combine(const MultiplexTable &table, const MultiplexRow &values); - LoaderState &loaderState; - const QbsItemRetriever qbsItemRetriever; + LoaderState &m_loaderState; + const QString &m_productName; + Item * const m_productItem; + Item * const m_tempQbsModuleItem; + const std::function<void()> &m_dropTempQbsModule; }; -ProductItemMultiplexer::ProductItemMultiplexer(LoaderState &loaderState, - const QbsItemRetriever &qbsItemRetriever) - : d(makePimpl<Private>(loaderState, qbsItemRetriever)) {} - -ProductItemMultiplexer::~ProductItemMultiplexer() = default; - -QList<Item *> ProductItemMultiplexer::multiplex( - const QString &productName, - Item *productItem, - Item *tempQbsModuleItem, - const std::function<void ()> &dropTempQbsModule) +QList<Item *> ProductItemMultiplexer::multiplex() { - const auto multiplexInfo = d->extractMultiplexInfo(productItem, tempQbsModuleItem); - dropTempQbsModule(); + const auto multiplexInfo = extractMultiplexInfo(); + m_dropTempQbsModule(); if (multiplexInfo.table.size() > 1) - productItem->setProperty(StringConstants::multiplexedProperty(), VariantValue::trueValue()); - VariantValuePtr productNameValue = VariantValue::create(productName); - Item *aggregator = multiplexInfo.aggregate ? productItem->clone() : nullptr; + m_productItem->setProperty(StringConstants::multiplexedProperty(), VariantValue::trueValue()); + VariantValuePtr productNameValue = VariantValue::create(m_productName); + Item *aggregator = multiplexInfo.aggregate ? m_productItem->clone(m_loaderState.itemPool()) + : nullptr; QList<Item *> additionalProductItems; std::vector<VariantValuePtr> multiplexConfigurationIdValues; for (size_t row = 0; row < multiplexInfo.table.size(); ++row) { - Item *item = productItem; + Item *item = m_productItem; const auto &mprow = multiplexInfo.table.at(row); QBS_CHECK(mprow.size() == multiplexInfo.properties.size()); if (row > 0) { - item = productItem->clone(); + item = m_productItem->clone(m_loaderState.itemPool()); additionalProductItems.push_back(item); } - const QString multiplexConfigurationId = multiplexInfo.toIdString(row); + const QString multiplexConfigurationId = multiplexInfo.toIdString(row, m_loaderState); const VariantValuePtr multiplexConfigurationIdValue = VariantValue::create(multiplexConfigurationId); if (multiplexInfo.table.size() > 1 || aggregator) { @@ -127,7 +124,7 @@ QList<Item *> ProductItemMultiplexer::multiplex( if (multiplexInfo.multiplexedType) item->setProperty(StringConstants::typeProperty(), multiplexInfo.multiplexedType); for (size_t column = 0; column < mprow.size(); ++column) { - Item * const qbsItem = d->qbsItemRetriever(item); + Item * const qbsItem = retrieveQbsItem(item, m_loaderState); const QString &propertyName = multiplexInfo.properties.at(column); const VariantValuePtr &mpvalue = mprow.at(column); qbsItem->setProperty(propertyName, mpvalue); @@ -139,14 +136,14 @@ QList<Item *> ProductItemMultiplexer::multiplex( // Add dependencies to all multiplexed instances. for (const auto &v : multiplexConfigurationIdValues) { - Item *dependsItem = Item::create(aggregator->pool(), ItemType::Depends); + Item *dependsItem = Item::create(&m_loaderState.itemPool(), ItemType::Depends); dependsItem->setProperty(StringConstants::nameProperty(), productNameValue); dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), v); dependsItem->setProperty(StringConstants::profilesProperty(), VariantValue::create(QStringList())); dependsItem->setFile(aggregator->file()); - dependsItem->setupForBuiltinType(d->loaderState.parameters().deprecationWarningMode(), - d->loaderState.logger()); + dependsItem->setupForBuiltinType(m_loaderState.parameters().deprecationWarningMode(), + m_loaderState.logger()); Item::addChild(aggregator, dependsItem); } } @@ -154,23 +151,22 @@ QList<Item *> ProductItemMultiplexer::multiplex( return additionalProductItems; } -MultiplexInfo ProductItemMultiplexer::Private::extractMultiplexInfo(Item *productItem, - Item *qbsModuleItem) +MultiplexInfo ProductItemMultiplexer::extractMultiplexInfo() { static const QString mpmKey = QStringLiteral("multiplexMap"); - Evaluator &evaluator = loaderState.evaluator(); + Evaluator &evaluator = m_loaderState.evaluator(); JSContext * const ctx = evaluator.engine()->context(); - const ScopedJsValue multiplexMap(ctx, evaluator.value(qbsModuleItem, mpmKey)); + const ScopedJsValue multiplexMap(ctx, evaluator.value(m_tempQbsModuleItem, mpmKey)); const QStringList multiplexByQbsProperties = evaluator.stringListValue( - productItem, StringConstants::multiplexByQbsPropertiesProperty()); + m_productItem, StringConstants::multiplexByQbsPropertiesProperty()); MultiplexInfo multiplexInfo; multiplexInfo.aggregate = evaluator.boolValue( - productItem, StringConstants::aggregateProperty()); + m_productItem, StringConstants::aggregateProperty()); const QString multiplexedType = evaluator.stringValue( - productItem, StringConstants::multiplexedTypeProperty()); + m_productItem, StringConstants::multiplexedTypeProperty()); if (!multiplexedType.isEmpty()) multiplexInfo.multiplexedType = VariantValue::create(multiplexedType); @@ -183,7 +179,7 @@ MultiplexInfo ProductItemMultiplexer::Private::extractMultiplexInfo(Item *produc if (!uniqueMultiplexByQbsProperties.insert(mappedKey).second) continue; - const ScopedJsValue arr(ctx, evaluator.value(qbsModuleItem, key)); + const ScopedJsValue arr(ctx, evaluator.value(m_tempQbsModuleItem, key)); if (JS_IsUndefined(arr)) continue; if (!JS_IsArray(ctx, arr)) @@ -210,8 +206,8 @@ MultiplexInfo ProductItemMultiplexer::Private::extractMultiplexInfo(Item *produc return multiplexInfo; } -MultiplexTable ProductItemMultiplexer::Private::combine(const MultiplexTable &table, - const MultiplexRow &values) +MultiplexTable ProductItemMultiplexer::combine(const MultiplexTable &table, + const MultiplexRow &values) { MultiplexTable result; if (table.empty()) { @@ -234,31 +230,7 @@ MultiplexTable ProductItemMultiplexer::Private::combine(const MultiplexTable &ta return result; } -QVariantMap ProductItemMultiplexer::multiplexIdToVariantMap(const QString &multiplexId) -{ - if (multiplexId.isEmpty()) - return QVariantMap(); - - // We assume that MultiplexInfo::toIdString() has been called for this - // particular multiplex configuration. - QVariantMap result = multiplexConfigurationsById->localData().value(multiplexId); - QBS_CHECK(!result.isEmpty()); - return result; -} - -QString ProductItemMultiplexer::fullProductDisplayName(const QString &name, - const QString &multiplexId) -{ - static const auto multiplexIdToString =[](const QString &id) { - return QString::fromUtf8(QByteArray::fromBase64(id.toUtf8())); - }; - QString result = name; - if (!multiplexId.isEmpty()) - result.append(QLatin1Char(' ')).append(multiplexIdToString(multiplexId)); - return result; -} - -QString MultiplexInfo::toIdString(size_t row) const +QString MultiplexInfo::toIdString(size_t row, LoaderState &loaderState) const { const auto &mprow = table.at(row); QVariantMap multiplexConfiguration; @@ -272,9 +244,16 @@ QString MultiplexInfo::toIdString(size_t row) const .toBase64()); // Cache for later use in multiplexIdToVariantMap() - multiplexConfigurationsById->localData().insert(id, multiplexConfiguration); + loaderState.topLevelProject().addMultiplexConfiguration(id, multiplexConfiguration); return id; } +QList<Item *> multiplex(const QString &productName, Item *productItem, Item *tempQbsModuleItem, + const std::function<void ()> &dropTempQbsModule, LoaderState &loaderState) +{ + return ProductItemMultiplexer(productName, productItem, tempQbsModuleItem, dropTempQbsModule, + loaderState).multiplex(); +} + } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productitemmultiplexer.h b/src/lib/corelib/loader/productitemmultiplexer.h index 53eb1a722..e02e21793 100644 --- a/src/lib/corelib/loader/productitemmultiplexer.h +++ b/src/lib/corelib/loader/productitemmultiplexer.h @@ -39,42 +39,26 @@ #pragma once -#include <tools/pimpl.h> - #include <QList> -#include <QVariantMap> #include <functional> +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + namespace qbs::Internal { class Item; class LoaderState; -// This class deals with product multiplexing over the various defined axes. -// For instance, a product with qbs.architectures: ["x86", "arm"] will get multiplexed into -// two products with qbs.architecture: "x86" and qbs.architecture: "arm", respectively. -class ProductItemMultiplexer -{ -public: - using QbsItemRetriever = std::function<Item *(Item *)>; - ProductItemMultiplexer(LoaderState &loaderState, const QbsItemRetriever &qbsItemRetriever); - ~ProductItemMultiplexer(); - - // Checks whether the product item is to be multiplexed and returns the list of additional - // product items. In the normal, non-multiplex case, this list is empty. - QList<Item *> multiplex( - const QString &productName, - Item *productItem, - Item *tempQbsModuleItem, - const std::function<void()> &dropTempQbsModule - ); - - static QVariantMap multiplexIdToVariantMap(const QString &multiplexId); - static QString fullProductDisplayName(const QString &name, const QString &multiplexId); - -private: - class Private; - Pimpl<Private> d; -}; +// Checks whether the product item is to be multiplexed and returns the list of additional +// product items. In the normal, non-multiplex case, this list is empty. +QList<Item *> multiplex( + const QString &productName, + Item *productItem, + Item *tempQbsModuleItem, + const std::function<void()> &dropTempQbsModule, + LoaderState &loaderState + ); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productresolver.cpp b/src/lib/corelib/loader/productresolver.cpp new file mode 100644 index 000000000..694b12c0a --- /dev/null +++ b/src/lib/corelib/loader/productresolver.cpp @@ -0,0 +1,1545 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "productresolver.h" + +#include "dependenciesresolver.h" +#include "groupshandler.h" +#include "loaderutils.h" +#include "modulepropertymerger.h" +#include "probesresolver.h" + +#include <jsextensions/jsextensions.h> +#include <jsextensions/moduleproperties.h> +#include <language/artifactproperties.h> +#include <language/builtindeclarations.h> +#include <language/evaluator.h> +#include <language/filecontext.h> +#include <language/item.h> +#include <language/language.h> +#include <language/propertymapinternal.h> +#include <language/scriptengine.h> +#include <language/value.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/fileinfo.h> +#include <tools/jsliterals.h> +#include <tools/profiling.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +namespace qbs::Internal { + +class PropertiesEvaluator +{ +public: + PropertiesEvaluator(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + + QVariantMap evaluateProperties(Item *item, bool lookupPrototype, bool checkErrors); + QVariantMap evaluateProperties(const Item *item, const Item *propertiesContainer, + const QVariantMap &tmplt, bool lookupPrototype, + bool checkErrors); + void evaluateProperty(const Item *item, const QString &propName, const ValuePtr &propValue, + QVariantMap &result, bool checkErrors); + +private: + ProductContext &m_product; + LoaderState &m_loaderState; +}; + +// Dependency resolving, Probe execution. +// Run for real products and shadow products. +class ProductResolverStage1 +{ +public: + ProductResolverStage1(ProductContext &product, Deferral deferral, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState), m_deferral(deferral) {} + void start(); + +private: + void resolveProbes(); + void resolveProbes(Item *item); + void runModuleProbes(const Item::Module &module); + void updateModulePresentState(const Item::Module &module); + bool validateModule(const Item::Module &module); + void handleModuleSetupError(const Item::Module &module, const ErrorInfo &error); + void checkPropertyDeclarations(); + void checkDependencyParameterDeclarations(const Item *productItem, + const QString &productName) const; + + + ProductContext &m_product; + LoaderState &m_loaderState; + const Deferral m_deferral; +}; + +// Setting up ResolvedProduct, incuding property evaluation and handling Product child items. +// Run only for real products. +class ProductResolverStage2 +{ +public: + ProductResolverStage2(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + void start(); + +private: + void resolveProductFully(); + void createProductConfig(); + void resolveGroup(Item *item); + void resolveGroupFully(Item *item, bool isEnabled); + QVariantMap resolveAdditionalModuleProperties(const Item *group, + const QVariantMap ¤tValues); + SourceArtifactPtr createSourceArtifact(const QString &fileName, const GroupPtr &group, + bool wildcard, const CodeLocation &filesLocation, + ErrorInfo *errorInfo); + void resolveExport(Item *exportItem); + std::unique_ptr<ExportedItem> resolveExportChild(const Item *item, + const ExportedModule &module); + void setupExportedProperties(const Item *item, const QString &namePrefix, + std::vector<ExportedProperty> &properties); + QVariantMap evaluateModuleValues(Item *item, bool lookupPrototype = true); + + void resolveScanner(Item *item, ModuleContext &moduleContext); + void resolveModules(); + void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, + const QVariantMap ¶meters, JobLimits &jobLimits); + void applyFileTaggers(); + void finalizeArtifactProperties(); + void collectProductDependencies(); + + ProductContext &m_product; + LoaderState &m_loaderState; + GroupConstPtr m_currentGroup; + FileLocations m_sourceArtifactLocations; + PropertiesEvaluator m_propertiesEvaluator{m_product, m_loaderState}; + + using ArtifactPropertiesInfo = std::pair<ArtifactPropertiesPtr, std::vector<CodeLocation>>; + QHash<QStringList, ArtifactPropertiesInfo> m_artifactPropertiesPerFilter; +}; + +class ExportsResolver +{ +public: + ExportsResolver(ProductContext &product, LoaderState &loaderState) + : m_product(product), m_loaderState(loaderState) {} + void start(); + +private: + void resolveShadowProduct(); + void collectPropertiesForExportItem(Item *productModuleInstance); + void collectPropertiesForExportItem(const QualifiedId &moduleName, const ValuePtr &value, + Item *moduleInstance, QVariantMap &moduleProps); + void collectPropertiesForModuleInExportItem(const Item::Module &module); + void adaptExportedPropertyValues(); + void collectExportedProductDependencies(); + + ProductContext &m_product; + LoaderState &m_loaderState; + PropertiesEvaluator m_propertiesEvaluator{m_product, m_loaderState}; +}; + +void resolveProduct(ProductContext &product, Deferral deferral, LoaderState &loaderState) +{ + try { + ProductResolverStage1(product, deferral, loaderState).start(); + } catch (const ErrorInfo &err) { + if (err.isCancelException()) { + loaderState.topLevelProject().setCanceled(); + return; + } + product.handleError(err); + } + + if (product.dependenciesResolvingPending()) + return; + + if (product.name.startsWith(StringConstants::shadowProductPrefix())) + return; + + // TODO: The weird double-forwarded error handling can hopefully be simplified now. + try { + ProductResolverStage2(product, loaderState).start(); + } catch (const ErrorInfo &err) { + if (err.isCancelException()) { + loaderState.topLevelProject().setCanceled(); + return; + } + loaderState.topLevelProject().addQueuedError(err); + } +} + +void setupExports(ProductContext &product, LoaderState &loaderState) +{ + ExportsResolver(product, loaderState).start(); +} + +void ProductResolverStage1::start() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + topLevelProject.checkCancelation(); + + if (m_product.delayedError.hasError()) + return; + + resolveDependencies(m_product, m_deferral, m_loaderState); + QBS_CHECK(m_product.dependenciesContext); + if (!m_product.dependenciesContext->dependenciesResolved) + return; + + // Run probes for modules and product. + resolveProbes(); + + // After the probes have run, we can switch on the evaluator cache. + Evaluator &evaluator = m_loaderState.evaluator(); + FileTags fileTags = evaluator.fileTagsValue(m_product.item, StringConstants::typeProperty()); + EvalCacheEnabler cacheEnabler(&evaluator, evaluator.stringValue( + m_product.item, + StringConstants::sourceDirectoryProperty())); + + // Run module validation scripts. + for (const Item::Module &module : m_product.item->modules()) { + if (!validateModule(module)) + return; + fileTags += evaluator.fileTagsValue( + module.item, StringConstants::additionalProductTypesProperty()); + } + + // Disable modules that have been pulled in only by now-disabled modules. + // Note that this has to happen in the reverse order compared to the other loops, + // with the leaves checked last. + for (auto it = m_product.item->modules().rbegin(); it != m_product.item->modules().rend(); ++it) + updateModulePresentState(*it); + + // Now do the canonical module property values merge. Note that this will remove + // previously attached values from modules that failed validation. + // Evaluator cache entries that could potentially change due to this will be purged. + doFinalMerge(m_product, m_loaderState); + + const bool enabled = topLevelProject.checkItemCondition(m_product.item, evaluator); + checkDependencyParameterDeclarations(m_product.item, m_product.name); + + setupGroups(m_product, m_loaderState); + + // Collect the full list of fileTags, including the values contributed by modules. + if (!m_product.delayedError.hasError() && enabled + && !m_product.name.startsWith(StringConstants::shadowProductPrefix())) { + topLevelProject.addProductByType(m_product, fileTags); + m_product.item->setProperty(StringConstants::typeProperty(), + VariantValue::create(sorted(fileTags.toStringList()))); + } + + checkPropertyDeclarations(); +} + +void ProductResolverStage1::resolveProbes() +{ + for (const Item::Module &module : m_product.item->modules()) { + runModuleProbes(module); + if (m_product.delayedError.hasError()) + return; + } + resolveProbes(m_product.item); +} + +void ProductResolverStage1::resolveProbes(Item *item) +{ + ProbesResolver(m_loaderState).resolveProbes(m_product, item); +} + +void ProductResolverStage1::runModuleProbes(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + if (module.product && m_loaderState.topLevelProject().isDisabledItem(module.product->item)) { + createNonPresentModule(m_loaderState.itemPool(), module.name.toString(), + QLatin1String("module's exporting product is disabled"), + module.item); + return; + } + try { + resolveProbes(module.item); + if (module.versionRange.minimum.isValid() + || module.versionRange.maximum.isValid()) { + if (module.versionRange.maximum.isValid() + && module.versionRange.minimum >= module.versionRange.maximum) { + throw ErrorInfo(Tr::tr("Impossible version constraint [%1,%2) set for module " + "'%3'").arg(module.versionRange.minimum.toString(), + module.versionRange.maximum.toString(), + module.name.toString())); + } + const Version moduleVersion = Version::fromString( + m_loaderState.evaluator().stringValue(module.item, + StringConstants::versionProperty())); + if (moduleVersion < module.versionRange.minimum) { + throw ErrorInfo(Tr::tr("Module '%1' has version %2, but it needs to be " + "at least %3.").arg(module.name.toString(), + moduleVersion.toString(), + module.versionRange.minimum.toString())); + } + if (module.versionRange.maximum.isValid() + && moduleVersion >= module.versionRange.maximum) { + throw ErrorInfo(Tr::tr("Module '%1' has version %2, but it needs to be " + "lower than %3.").arg(module.name.toString(), + moduleVersion.toString(), + module.versionRange.maximum.toString())); + } + } + } catch (const ErrorInfo &error) { + handleModuleSetupError(module, error); + } +} + +void ProductResolverStage1::updateModulePresentState(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + bool hasPresentLoadingItem = false; + for (const Item * const loadingItem : module.loadingItems) { + if (loadingItem == m_product.item) { + hasPresentLoadingItem = true; + break; + } + if (!loadingItem->isPresentModule()) + continue; + if (loadingItem->prototype() && loadingItem->prototype()->type() == ItemType::Export) { + QBS_CHECK(loadingItem->prototype()->parent()->type() == ItemType::Product); + if (m_loaderState.topLevelProject().isDisabledItem(loadingItem->prototype()->parent())) + continue; + } + hasPresentLoadingItem = true; + break; + } + if (!hasPresentLoadingItem) { + createNonPresentModule(m_loaderState.itemPool(), module.name.toString(), + QLatin1String("imported only by disabled module(s)"), + module.item); + } +} + +bool ProductResolverStage1::validateModule(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return true; + try { + m_loaderState.evaluator().boolValue(module.item, StringConstants::validateProperty()); + for (const auto &dep : module.item->modules()) { + if (dep.required && !dep.item->isPresentModule()) { + throw ErrorInfo(Tr::tr("Module '%1' depends on module '%2', which was not " + "loaded successfully") + .arg(module.name.toString(), dep.name.toString())); + } + } + } catch (const ErrorInfo &error) { + handleModuleSetupError(module, error); + if (m_product.delayedError.hasError()) + return false; + } + return true; +} + +void ProductResolverStage1::handleModuleSetupError(const Item::Module &module, + const ErrorInfo &error) +{ + if (module.required) { + m_product.handleError(error); + } else { + qCDebug(lcModuleLoader()) << "non-required module" << module.name.toString() + << "found, but not usable in product" << m_product.name + << error.toString(); + createNonPresentModule(m_loaderState.itemPool(), module.name.toString(), + QStringLiteral("failed validation"), module.item); + } +} + +void ProductResolverStage1::checkPropertyDeclarations() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyChecking : nullptr); + qbs::Internal::checkPropertyDeclarations(m_product.item, m_loaderState); +} + +class DependencyParameterDeclarationCheck +{ +public: + DependencyParameterDeclarationCheck(const QString &productName, const Item *productItem, + const TopLevelProjectContext &topLevelProject) + : m_productName(productName), m_productItem(productItem), m_topLevelProject(topLevelProject) + {} + + void operator()(const QVariantMap ¶meters) const { check(parameters, QualifiedId()); } + +private: + void check(const QVariantMap ¶meters, const QualifiedId &moduleName) const + { + for (auto it = parameters.begin(); it != parameters.end(); ++it) { + if (it.value().userType() == QMetaType::QVariantMap) { + check(it.value().toMap(), QualifiedId(moduleName) << it.key()); + } else { + const auto &deps = m_productItem->modules(); + auto m = std::find_if(deps.begin(), deps.end(), + [&moduleName] (const Item::Module &module) { + return module.name == moduleName; + }); + + if (m == deps.end()) { + const QualifiedId fullName = QualifiedId(moduleName) << it.key(); + throw ErrorInfo(Tr::tr("Cannot set parameter '%1', " + "because '%2' does not have a dependency on '%3'.") + .arg(fullName.toString(), m_productName, moduleName.toString()), + m_productItem->location()); + } + + const auto decls = m_topLevelProject.parameterDeclarations( + m->item->rootPrototype()); + if (!decls.contains(it.key())) { + const QualifiedId fullName = QualifiedId(moduleName) << it.key(); + throw ErrorInfo(Tr::tr("Parameter '%1' is not declared.") + .arg(fullName.toString()), m_productItem->location()); + } + } + } + } + + bool moduleExists(const QualifiedId &name) const + { + const auto &deps = m_productItem->modules(); + return any_of(deps, [&name](const Item::Module &module) { + return module.name == name; + }); + } + + const QString &m_productName; + const Item * const m_productItem; + const TopLevelProjectContext &m_topLevelProject; +}; + +void ProductResolverStage1::checkDependencyParameterDeclarations(const Item *productItem, + const QString &productName) const +{ + DependencyParameterDeclarationCheck dpdc(productName, productItem, + m_loaderState.topLevelProject()); + for (const Item::Module &dep : productItem->modules()) { + if (!dep.parameters.empty()) + dpdc(dep.parameters); + } +} + + +void ProductResolverStage2::start() +{ + m_loaderState.evaluator().clearPropertyDependencies(); + + ResolvedProductPtr product = ResolvedProduct::create(); + product->enabled = m_product.project->project->enabled; + product->moduleProperties = PropertyMapInternal::create(); + product->project = m_product.project->project; + m_product.product = product; + product->location = m_product.item->location(); + const auto errorFromDelayedError = [&] { + if (m_product.delayedError.hasError()) { + ErrorInfo errorInfo; + + // First item is "main error", gets prepended again in the catch clause. + const QList<ErrorItem> &items = m_product.delayedError.items(); + for (int i = 1; i < items.size(); ++i) + errorInfo.append(items.at(i)); + + m_product.delayedError.clear(); + return errorInfo; + } + return ErrorInfo(); + }; + + // Even if we previously encountered an error, try to continue for as long as possible + // to provide IDEs with useful data (e.g. the list of files). + // If we encounter a follow-up error, suppress it and report the original one instead. + try { + resolveProductFully(); + if (const ErrorInfo error = errorFromDelayedError(); error.hasError()) + throw error; + } catch (ErrorInfo e) { + if (const ErrorInfo error = errorFromDelayedError(); error.hasError()) + e = error; + QString mainErrorString = !product->name.isEmpty() + ? Tr::tr("Error while handling product '%1':").arg(product->name) + : Tr::tr("Error while handling product:"); + ErrorInfo fullError(mainErrorString, product->location); + appendError(fullError, e); + if (!product->enabled) { + qCDebug(lcProjectResolver) << fullError.toString(); + return; + } + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw fullError; + m_loaderState.logger().printWarning(fullError); + m_loaderState.logger().printWarning( + ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") + .arg(product->name), product->location)); + product->enabled = false; + } +} + +void ProductResolverStage2::resolveProductFully() +{ + Item * const item = m_product.item; + const ResolvedProductPtr product = m_product.product; + Evaluator &evaluator = m_loaderState.evaluator(); + product->name = evaluator.stringValue(item, StringConstants::nameProperty()); + + // product->buildDirectory() isn't valid yet, because the productProperties map is not ready. + m_product.buildDirectory = evaluator.stringValue( + item, StringConstants::buildDirectoryProperty()); + product->multiplexConfigurationId = evaluator.stringValue( + item, StringConstants::multiplexConfigurationIdProperty()); + qCDebug(lcProjectResolver) << "resolveProduct" << product->uniqueName(); + product->enabled = product->enabled + && evaluator.boolValue(item, StringConstants::conditionProperty()); + const VariantValuePtr type = item->variantProperty(StringConstants::typeProperty()); + if (type) + product->fileTags = FileTags::fromStringList(type->value().toStringList()); + product->targetName = evaluator.stringValue(item, StringConstants::targetNameProperty()); + product->sourceDirectory = evaluator.stringValue( + item, StringConstants::sourceDirectoryProperty()); + product->destinationDirectory = evaluator.stringValue( + item, StringConstants::destinationDirProperty()); + + if (product->destinationDirectory.isEmpty()) { + product->destinationDirectory = m_product.buildDirectory; + } else { + product->destinationDirectory = FileInfo::resolvePath( + product->topLevelProject()->buildDirectory, + product->destinationDirectory); + } + product->probes = m_product.probes; + createProductConfig(); + product->productProperties.insert(StringConstants::destinationDirProperty(), + product->destinationDirectory); + ModuleProperties::init(evaluator.engine(), evaluator.scriptValue(item), product.get()); + + QList<Item *> subItems = item->children(); + const ValuePtr filesProperty = item->property(StringConstants::filesProperty()); + if (filesProperty) { + Item *fakeGroup = Item::create(&m_loaderState.itemPool(), ItemType::Group); + fakeGroup->setFile(item->file()); + fakeGroup->setLocation(item->location()); + fakeGroup->setScope(item); + fakeGroup->setProperty(StringConstants::nameProperty(), VariantValue::create(product->name)); + fakeGroup->setProperty(StringConstants::filesProperty(), filesProperty); + fakeGroup->setProperty(StringConstants::excludeFilesProperty(), + item->property(StringConstants::excludeFilesProperty())); + fakeGroup->setProperty(StringConstants::overrideTagsProperty(), + VariantValue::falseValue()); + fakeGroup->setupForBuiltinType(m_loaderState.parameters().deprecationWarningMode(), + m_loaderState.logger()); + subItems.prepend(fakeGroup); + } + + for (Item * const child : std::as_const(subItems)) { + switch (child->type()) { + case ItemType::Rule: + resolveRule(m_loaderState, child, m_product.project, &m_product, nullptr); + break; + case ItemType::FileTagger: + resolveFileTagger(m_loaderState, child, nullptr, &m_product); + break; + case ItemType::JobLimit: + resolveJobLimit(m_loaderState, child, nullptr, &m_product, nullptr); + break; + case ItemType::Group: + resolveGroup(child); + break; + case ItemType::Export: + resolveExport(child); + break; + default: + break; + } + } + + for (const ProjectContext *p = m_product.project; p; p = p->parent) { + JobLimits tempLimits = p->jobLimits; + product->jobLimits = tempLimits.update(product->jobLimits); + } + + resolveModules(); + applyFileTaggers(); + finalizeArtifactProperties(); + + for (const RulePtr &rule : m_product.project->rules) { + RulePtr clonedRule = rule->clone(); + clonedRule->product = product.get(); + product->rules.push_back(clonedRule); + } + + collectProductDependencies(); +} + +void ProductResolverStage2::createProductConfig() +{ + EvalCacheEnabler cachingEnabler(&m_loaderState.evaluator(), + m_product.product->sourceDirectory); + m_product.product->moduleProperties->setValue(evaluateModuleValues(m_product.item)); + m_product.product->productProperties = m_propertiesEvaluator.evaluateProperties( + m_product.item, m_product.item, QVariantMap(), true, true); +} + +void ProductResolverStage2::resolveGroup(Item *item) +{ + const bool parentEnabled = m_currentGroup ? m_currentGroup->enabled + : m_product.product->enabled; + const bool isEnabled = parentEnabled + && m_loaderState.evaluator().boolValue(item, StringConstants::conditionProperty()); + try { + resolveGroupFully(item, isEnabled); + } catch (const ErrorInfo &error) { + if (!isEnabled) { + qCDebug(lcProjectResolver) << "error resolving group at" << item->location() + << error.toString(); + return; + } + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw; + m_loaderState.logger().printWarning(error); + } +} + +void ProductResolverStage2::resolveGroupFully(Item *item, bool isEnabled) +{ + AccumulatingTimer groupTimer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.groupsResolving + : nullptr); + + const auto getGroupPropertyMap = [&](const ArtifactProperties *existingProps) { + PropertyMapPtr moduleProperties; + bool newPropertyMapRequired = false; + if (existingProps) + moduleProperties = existingProps->propertyMap(); + if (!moduleProperties) { + newPropertyMapRequired = true; + moduleProperties = m_currentGroup + ? m_currentGroup->properties + : m_product.product->moduleProperties; + } + const QVariantMap newModuleProperties = resolveAdditionalModuleProperties( + item, moduleProperties->value()); + if (!newModuleProperties.empty()) { + if (newPropertyMapRequired) + moduleProperties = PropertyMapInternal::create(); + moduleProperties->setValue(newModuleProperties); + } + return moduleProperties; + }; + + Evaluator &evaluator = m_loaderState.evaluator(); + QStringList files = evaluator.stringListValue(item, StringConstants::filesProperty()); + bool fileTagsSet; + const FileTags fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty(), + &fileTagsSet); + const QStringList fileTagsFilter + = evaluator.stringListValue(item, StringConstants::fileTagsFilterProperty()); + if (!fileTagsFilter.empty()) { + if (Q_UNLIKELY(!files.empty())) + throw ErrorInfo(Tr::tr("Group.files and Group.fileTagsFilters are exclusive."), + item->location()); + + if (!isEnabled) + return; + + ArtifactPropertiesInfo &apinfo = m_artifactPropertiesPerFilter[fileTagsFilter]; + if (apinfo.first) { + const auto it = std::find_if(apinfo.second.cbegin(), apinfo.second.cend(), + [item](const CodeLocation &loc) { + return item->location().filePath() == loc.filePath(); + }); + if (it != apinfo.second.cend()) { + ErrorInfo error(Tr::tr("Conflicting fileTagsFilter in Group items.")); + error.append(Tr::tr("First item"), *it); + error.append(Tr::tr("Second item"), item->location()); + throw error; + } + } else { + apinfo.first = ArtifactProperties::create(); + apinfo.first->setFileTagsFilter(FileTags::fromStringList(fileTagsFilter)); + m_product.product->artifactProperties.push_back(apinfo.first); + } + apinfo.second.push_back(item->location()); + apinfo.first->setPropertyMapInternal(getGroupPropertyMap(apinfo.first.get())); + apinfo.first->addExtraFileTags(fileTags); + return; + } + QStringList patterns; + for (int i = files.size(); --i >= 0;) { + if (FileInfo::isPattern(files[i])) + patterns.push_back(files.takeAt(i)); + } + GroupPtr group = ResolvedGroup::create(); + bool prefixWasSet = false; + group->prefix = evaluator.stringValue(item, StringConstants::prefixProperty(), QString(), + &prefixWasSet); + if (!prefixWasSet && m_currentGroup) + group->prefix = m_currentGroup->prefix; + if (!group->prefix.isEmpty()) { + for (auto it = files.rbegin(), end = files.rend(); it != end; ++it) + it->prepend(group->prefix); + } + group->location = item->location(); + group->enabled = isEnabled; + group->properties = getGroupPropertyMap(nullptr); + group->fileTags = fileTags; + group->overrideTags = evaluator.boolValue(item, StringConstants::overrideTagsProperty()); + if (group->overrideTags && fileTagsSet) { + if (group->fileTags.empty() ) + group->fileTags.insert(unknownFileTag()); + } else if (m_currentGroup) { + group->fileTags.unite(m_currentGroup->fileTags); + } + + const CodeLocation filesLocation = item->property(StringConstants::filesProperty())->location(); + const VariantValueConstPtr moduleProp = item->variantProperty( + StringConstants::modulePropertyInternal()); + if (moduleProp) + group->targetOfModule = moduleProp->value().toString(); + ErrorInfo fileError; + if (!patterns.empty()) { + group->wildcards = std::make_unique<SourceWildCards>(); + SourceWildCards *wildcards = group->wildcards.get(); + wildcards->group = group.get(); + wildcards->excludePatterns = evaluator.stringListValue( + item, StringConstants::excludeFilesProperty()); + wildcards->patterns = patterns; + const Set<QString> files = wildcards->expandPatterns(group, + FileInfo::path(item->file()->filePath()), + m_product.project->project->topLevelProject()->buildDirectory); + for (const QString &fileName : files) + createSourceArtifact(fileName, group, true, filesLocation, &fileError); + } + + for (const QString &fileName : std::as_const(files)) + createSourceArtifact(fileName, group, false, filesLocation, &fileError); + if (fileError.hasError()) { + if (group->enabled) { + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw ErrorInfo(fileError); + m_loaderState.logger().printWarning(fileError); + } else { + qCDebug(lcProjectResolver) << "error for disabled group:" << fileError.toString(); + } + } + group->name = evaluator.stringValue(item, StringConstants::nameProperty()); + if (group->name.isEmpty()) + group->name = Tr::tr("Group %1").arg(m_product.product->groups.size()); + m_product.product->groups.push_back(group); + + class GroupContextSwitcher { + public: + GroupContextSwitcher(ProductResolverStage2 &resolver, const GroupConstPtr &newGroup) + : m_resolver(resolver), m_oldGroup(resolver.m_currentGroup) { + resolver.m_currentGroup = newGroup; + } + ~GroupContextSwitcher() { m_resolver.m_currentGroup = m_oldGroup; } + private: + ProductResolverStage2 &m_resolver; + const GroupConstPtr m_oldGroup; + }; + GroupContextSwitcher groupSwitcher(*this, group); + for (Item * const childItem : item->children()) + resolveGroup(childItem); +} + +SourceArtifactPtr ProductResolverStage2::createSourceArtifact( + const QString &fileName, const GroupPtr &group, bool wildcard, + const CodeLocation &filesLocation, ErrorInfo *errorInfo) +{ + const QString &baseDir = FileInfo::path(group->location.filePath()); + const QString absFilePath = QDir::cleanPath(FileInfo::resolvePath(baseDir, fileName)); + if (!wildcard && !FileInfo(absFilePath).exists()) { + if (errorInfo) + errorInfo->append(Tr::tr("File '%1' does not exist.").arg(absFilePath), filesLocation); + m_product.product->missingSourceFiles << absFilePath; + return {}; + } + if (group->enabled) { + CodeLocation &loc = m_sourceArtifactLocations[ + std::make_pair(group->targetOfModule, absFilePath)]; + if (loc.isValid()) { + if (errorInfo) { + errorInfo->append(Tr::tr("Duplicate source file '%1'.").arg(absFilePath)); + errorInfo->append(Tr::tr("First occurrence is here."), loc); + errorInfo->append(Tr::tr("Next occurrence is here."), filesLocation); + } + return {}; + } + loc = filesLocation; + } + SourceArtifactPtr artifact = SourceArtifactInternal::create(); + artifact->absoluteFilePath = absFilePath; + artifact->fileTags = group->fileTags; + artifact->overrideFileTags = group->overrideTags; + artifact->properties = group->properties; + artifact->targetOfModule = group->targetOfModule; + (wildcard ? group->wildcards->files : group->files).push_back(artifact); + return artifact; +} + +static QualifiedIdSet propertiesToEvaluate(std::deque<QualifiedId> initialProps, + const PropertyDependencies &deps) +{ + std::deque<QualifiedId> remainingProps = std::move(initialProps); + QualifiedIdSet allProperties; + while (!remainingProps.empty()) { + const QualifiedId prop = remainingProps.front(); + remainingProps.pop_front(); + const auto insertResult = allProperties.insert(prop); + if (!insertResult.second) + continue; + transform(deps.value(prop), remainingProps, [](const QualifiedId &id) { return id; }); + } + return allProperties; +} + +QVariantMap ProductResolverStage2::resolveAdditionalModuleProperties( + const Item *group, const QVariantMap ¤tValues) +{ + // Step 1: Retrieve the properties directly set in the group + const ModulePropertiesPerGroup &mp = m_product.modulePropertiesSetInGroups; + const auto it = mp.find(group); + if (it == mp.end()) + return {}; + const QualifiedIdSet &propsSetInGroup = it->second; + + // Step 2: Gather all properties that depend on these properties. + const QualifiedIdSet &propsToEval = propertiesToEvaluate( + rangeTo<std::deque<QualifiedId>>(propsSetInGroup), + m_loaderState.evaluator().propertyDependencies()); + + // Step 3: Evaluate all these properties and replace their values in the map + QVariantMap modulesMap = currentValues; + QHash<QString, QStringList> propsPerModule; + for (auto fullPropName : propsToEval) { + const QString moduleName + = QualifiedId(fullPropName.mid(0, fullPropName.size() - 1)).toString(); + propsPerModule[moduleName] << fullPropName.last(); + } + EvalCacheEnabler cachingEnabler(&m_loaderState.evaluator(), + m_product.product->sourceDirectory); + for (const Item::Module &module : group->modules()) { + const QString &fullModName = module.name.toString(); + const QStringList propsForModule = propsPerModule.take(fullModName); + if (propsForModule.empty()) + continue; + QVariantMap reusableValues = modulesMap.value(fullModName).toMap(); + for (const QString &prop : std::as_const(propsForModule)) + reusableValues.remove(prop); + modulesMap.insert(fullModName, m_propertiesEvaluator.evaluateProperties( + module.item, module.item, reusableValues, true, true)); + } + return modulesMap; +} + +static QString getLineAtLocation(const CodeLocation &loc, const QString &content) +{ + int pos = 0; + int currentLine = 1; + while (currentLine < loc.line()) { + while (content.at(pos++) != QLatin1Char('\n')) + ; + ++currentLine; + } + const int eolPos = content.indexOf(QLatin1Char('\n'), pos); + return content.mid(pos, eolPos - pos); +} + +static bool usesImport(const ExportedProperty &prop, const QRegularExpression ®ex) +{ + return prop.sourceCode.indexOf(regex) != -1; +} + +static bool usesImport(const ExportedItem &item, const QRegularExpression ®ex) +{ + return any_of(item.properties, + [regex](const ExportedProperty &p) { return usesImport(p, regex); }) + || any_of(item.children, + [regex](const ExportedItemPtr &child) { return usesImport(*child, regex); }); +} + +static bool usesImport(const ExportedModule &module, const QString &name) +{ + // Imports are used in three ways: + // (1) var f = new TextFile(...); + // (2) var path = FileInfo.joinPaths(...) + // (3) var obj = DataCollection; + const QString pattern = QStringLiteral("\\b%1\\b"); + + const QRegularExpression regex(pattern.arg(name)); // std::regex is much slower + return any_of(module.m_properties, + [regex](const ExportedProperty &p) { return usesImport(p, regex); }) + || any_of(module.children, + [regex](const ExportedItemPtr &child) { return usesImport(*child, regex); }); +} + +void ProductResolverStage2::resolveExport(Item *exportItem) +{ + ExportedModule &exportedModule = m_product.product->exportedModule; + setupExportedProperties(exportItem, QString(), exportedModule.m_properties); + static const auto cmpFunc = [](const ExportedProperty &p1, const ExportedProperty &p2) { + return p1.fullName < p2.fullName; + }; + std::sort(exportedModule.m_properties.begin(), exportedModule.m_properties.end(), cmpFunc); + + transform(exportItem->children(), exportedModule.children, + [&exportedModule, this](const auto &child) { + return resolveExportChild(child, exportedModule); }); + + for (const JsImport &jsImport : exportItem->file()->jsImports()) { + if (usesImport(exportedModule, jsImport.scopeName)) { + exportedModule.importStatements << getLineAtLocation(jsImport.location, + exportItem->file()->content()); + } + } + const auto builtInImports = JsExtensions::extensionNames(); + for (const QString &builtinImport: builtInImports) { + if (usesImport(exportedModule, builtinImport)) + exportedModule.importStatements << QStringLiteral("import qbs.") + builtinImport; + } + exportedModule.importStatements.sort(); +} + +// TODO: This probably wouldn't be necessary if we had item serialization. +std::unique_ptr<ExportedItem> ProductResolverStage2::resolveExportChild( + const Item *item, const ExportedModule &module) +{ + std::unique_ptr<ExportedItem> exportedItem(new ExportedItem); + + // This is the type of the built-in base item. It may turn out that we need to support + // derived items under Export. In that case, we probably need a new Item member holding + // the original type name. + exportedItem->name = item->typeName(); + + transform(item->children(), exportedItem->children, [&module, this](const auto &child) { + return resolveExportChild(child, module); }); + + setupExportedProperties(item, QString(), exportedItem->properties); + return exportedItem; +} + +void ProductResolverStage2::setupExportedProperties(const Item *item, const QString &namePrefix, + std::vector<ExportedProperty> &properties) +{ + const auto &props = item->properties(); + for (auto it = props.cbegin(); it != props.cend(); ++it) { + const QString qualifiedName = namePrefix.isEmpty() + ? it.key() : namePrefix + QLatin1Char('.') + it.key(); + if ((item->type() == ItemType::Export || item->type() == ItemType::Properties) + && qualifiedName == StringConstants::prefixMappingProperty()) { + continue; + } + const ValuePtr &v = it.value(); + if (v->type() == Value::ItemValueType) { + setupExportedProperties(std::static_pointer_cast<ItemValue>(v)->item(), + qualifiedName, properties); + continue; + } + ExportedProperty exportedProperty; + exportedProperty.fullName = qualifiedName; + exportedProperty.type = item->propertyDeclaration(it.key()).type(); + if (v->type() == Value::VariantValueType) { + exportedProperty.sourceCode = toJSLiteral( + std::static_pointer_cast<VariantValue>(v)->value()); + } else { + QBS_CHECK(v->type() == Value::JSSourceValueType); + const JSSourceValue * const sv = static_cast<JSSourceValue *>(v.get()); + exportedProperty.sourceCode = sv->sourceCode().toString(); + } + const ItemDeclaration itemDecl + = BuiltinDeclarations::instance().declarationsForType(item->type()); + PropertyDeclaration propertyDecl; + const auto itemProperties = itemDecl.properties(); + for (const PropertyDeclaration &decl : itemProperties) { + if (decl.name() == it.key()) { + propertyDecl = decl; + exportedProperty.isBuiltin = true; + break; + } + } + + // Do not add built-in properties that were left at their default value. + if (!exportedProperty.isBuiltin + || m_loaderState.evaluator().isNonDefaultValue(item, it.key())) { + properties.push_back(exportedProperty); + } + } + + // Order the list of properties, so the output won't look so random. + static const auto less = [](const ExportedProperty &p1, const ExportedProperty &p2) -> bool { + const int p1ComponentCount = p1.fullName.count(QLatin1Char('.')); + const int p2ComponentCount = p2.fullName.count(QLatin1Char('.')); + if (p1.isBuiltin && !p2.isBuiltin) + return true; + if (!p1.isBuiltin && p2.isBuiltin) + return false; + if (p1ComponentCount < p2ComponentCount) + return true; + if (p1ComponentCount > p2ComponentCount) + return false; + return p1.fullName < p2.fullName; + }; + std::sort(properties.begin(), properties.end(), less); +} + +QVariantMap ProductResolverStage2::evaluateModuleValues(Item *item, bool lookupPrototype) +{ + QVariantMap moduleValues; + for (const Item::Module &module : item->modules()) { + if (!module.item->isPresentModule()) + continue; + const QString fullName = module.name.toString(); + moduleValues[fullName] = m_propertiesEvaluator.evaluateProperties( + module.item, lookupPrototype, true); + } + return moduleValues; +} + +void ProductResolverStage2::resolveScanner(Item *item, ModuleContext &moduleContext) +{ + Evaluator &evaluator = m_loaderState.evaluator(); + if (!evaluator.boolValue(item, StringConstants::conditionProperty())) { + qCDebug(lcProjectResolver) << "scanner condition is false"; + return; + } + + ResolvedScannerPtr scanner = ResolvedScanner::create(); + scanner->module = moduleContext.module; + scanner->inputs = evaluator.fileTagsValue(item, StringConstants::inputsProperty()); + scanner->recursive = evaluator.boolValue(item, StringConstants::recursiveProperty()); + scanner->searchPathsScript.initialize(m_loaderState.topLevelProject().scriptFunctionValue( + item, StringConstants::searchPathsProperty())); + scanner->scanScript.initialize(m_loaderState.topLevelProject().scriptFunctionValue( + item, StringConstants::scanProperty())); + m_product.product->scanners.push_back(scanner); +} + +void ProductResolverStage2::resolveModules() +{ + JobLimits jobLimits; + for (const Item::Module &m : m_product.item->modules()) + resolveModule(m.name, m.item, m.product, m.parameters, jobLimits); + for (int i = 0; i < jobLimits.count(); ++i) { + const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i); + if (m_product.product->jobLimits.getLimit(moduleJobLimit.pool()) == -1) + m_product.product->jobLimits.setJobLimit(moduleJobLimit); + } +} + +void ProductResolverStage2::resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, + const QVariantMap ¶meters, JobLimits &jobLimits) +{ + if (!item->isPresentModule()) + return; + + ModuleContext moduleContext; + moduleContext.module = ResolvedModule::create(); + + const ResolvedModulePtr &module = moduleContext.module; + module->name = moduleName.toString(); + module->isProduct = isProduct; + module->product = m_product.product.get(); + module->setupBuildEnvironmentScript.initialize(m_loaderState.topLevelProject() + .scriptFunctionValue(item, StringConstants::setupBuildEnvironmentProperty())); + module->setupRunEnvironmentScript.initialize(m_loaderState.topLevelProject() + .scriptFunctionValue(item, StringConstants::setupRunEnvironmentProperty())); + + for (const Item::Module &m : item->modules()) { + if (m.item->isPresentModule()) + module->moduleDependencies += m.name.toString(); + } + + m_product.product->modules.push_back(module); + if (!parameters.empty()) + m_product.product->moduleParameters[module] = parameters; + + for (Item *child : item->children()) { + switch (child->type()) { + case ItemType::Rule: + resolveRule(m_loaderState, child, nullptr, &m_product, &moduleContext); + break; + case ItemType::FileTagger: + resolveFileTagger(m_loaderState, child, nullptr, &m_product); + break; + case ItemType::JobLimit: + resolveJobLimit(m_loaderState, child, nullptr, nullptr, &moduleContext); + break; + case ItemType::Scanner: + resolveScanner(child, moduleContext); + break; + default: + break; + } + } + for (int i = 0; i < moduleContext.jobLimits.count(); ++i) { + const JobLimit &newJobLimit = moduleContext.jobLimits.jobLimitAt(i); + const int oldLimit = jobLimits.getLimit(newJobLimit.pool()); + if (oldLimit == -1 || oldLimit > newJobLimit.limit()) + jobLimits.setJobLimit(newJobLimit); + } +} + +void ProductResolverStage2::applyFileTaggers() +{ + m_product.product->fileTaggers << m_product.project->fileTaggers; + m_product.product->fileTaggers = sorted(m_product.product->fileTaggers, + [] (const FileTaggerConstPtr &a, const FileTaggerConstPtr &b) { + return a->priority() > b->priority(); + }); + for (const SourceArtifactPtr &artifact : m_product.product->allEnabledFiles()) { + if (!artifact->overrideFileTags || artifact->fileTags.empty()) { + const QString fileName = FileInfo::fileName(artifact->absoluteFilePath); + const FileTags fileTags = m_product.product->fileTagsForFileName(fileName); + artifact->fileTags.unite(fileTags); + if (artifact->fileTags.empty()) + artifact->fileTags.insert(unknownFileTag()); + qCDebug(lcProjectResolver) << "adding file tags" << artifact->fileTags + << "to" << fileName; + } + } +} + +void ProductResolverStage2::finalizeArtifactProperties() +{ + for (const SourceArtifactPtr &artifact : m_product.product->allEnabledFiles()) { + for (const auto &artifactProperties : m_product.product->artifactProperties) { + if (!artifact->isTargetOfModule() + && artifact->fileTags.intersects(artifactProperties->fileTagsFilter())) { + // FIXME: Should be merged, not overwritten. + artifact->properties = artifactProperties->propertyMap(); + } + } + + // Let a positive value of qbs.install imply the file tag "installable". + if (artifact->properties->qbsPropertyValue(StringConstants::installProperty()).toBool()) + artifact->fileTags += "installable"; + } +} + +void ProductResolverStage2::collectProductDependencies() +{ + const ResolvedProductPtr &product = m_product.product; + if (!product) + return; + for (const Item::Module &module : m_product.item->modules()) { + if (!module.product) + continue; + const ResolvedProductPtr &dep = module.product->product; + QBS_CHECK(dep); + QBS_CHECK(dep != product); + product->dependencies << dep; + product->dependencyParameters.insert(dep, module.parameters); // TODO: Streamline this with normal module dependencies? + } + + // TODO: We might want to keep the topological sorting and get rid of "module module dependencies". + std::sort(product->dependencies.begin(),product->dependencies.end(), + [](const ResolvedProductPtr &p1, const ResolvedProductPtr &p2) { + return p1->fullDisplayName() < p2->fullDisplayName(); + }); +} + +void ExportsResolver::start() +{ + resolveShadowProduct(); + collectExportedProductDependencies(); +} + +void ExportsResolver::resolveShadowProduct() +{ + if (!m_product.product->enabled) + return; + if (!m_product.shadowProduct) + return; + for (const auto &m : m_product.shadowProduct->item->modules()) { + if (m.name.toString() != m_product.product->name) + continue; + collectPropertiesForExportItem(m.item); + for (const auto &dep : m.item->modules()) + collectPropertiesForModuleInExportItem(dep); + break; + } + try { + adaptExportedPropertyValues(); + } catch (const ErrorInfo &) {} +} + +class TempScopeSetter +{ +public: + TempScopeSetter(const ValuePtr &value, Item *newScope) : m_value(value), m_oldScope(value->scope()) + { + value->setScope(newScope, {}); + } + ~TempScopeSetter() { if (m_value) m_value->setScope(m_oldScope, {}); } + + TempScopeSetter(const TempScopeSetter &) = delete; + TempScopeSetter &operator=(const TempScopeSetter &) = delete; + TempScopeSetter &operator=(TempScopeSetter &&) = delete; + + TempScopeSetter(TempScopeSetter &&other) noexcept + : m_value(std::move(other.m_value)), m_oldScope(other.m_oldScope) + { + other.m_value.reset(); + other.m_oldScope = nullptr; + } + +private: + ValuePtr m_value; + Item *m_oldScope; +}; + +void ExportsResolver::collectPropertiesForExportItem( + const QualifiedId &moduleName, const ValuePtr &value, Item *moduleInstance, + QVariantMap &moduleProps) +{ + QBS_CHECK(value->type() == Value::ItemValueType); + Item * const itemValueItem = std::static_pointer_cast<ItemValue>(value)->item(); + if (itemValueItem->propertyDeclarations().isEmpty()) { + for (const Item::Module &module : moduleInstance->modules()) { + if (module.name == moduleName) { + itemValueItem->setPropertyDeclarations(module.item->propertyDeclarations()); + break; + } + } + } + if (itemValueItem->type() == ItemType::ModuleInstancePlaceholder) { + struct EvalPreparer { + EvalPreparer(Item *valueItem, const QualifiedId &moduleName) + : valueItem(valueItem), + hadName(!!valueItem->variantProperty(StringConstants::nameProperty())) + { + if (!hadName) { + // Evaluator expects a name here. + valueItem->setProperty(StringConstants::nameProperty(), + VariantValue::create(moduleName.toString())); + } + } + ~EvalPreparer() + { + if (!hadName) + valueItem->removeProperty(StringConstants::nameProperty()); + } + Item * const valueItem; + const bool hadName; + }; + EvalPreparer ep(itemValueItem, moduleName); + std::vector<TempScopeSetter> tss; + for (const ValuePtr &v : itemValueItem->properties()) + tss.emplace_back(v, moduleInstance); + moduleProps.insert(moduleName.toString(), m_propertiesEvaluator.evaluateProperties( + itemValueItem, false, false)); + return; + } + QBS_CHECK(itemValueItem->type() == ItemType::ModulePrefix); + const Item::PropertyMap &props = itemValueItem->properties(); + for (auto it = props.begin(); it != props.end(); ++it) { + QualifiedId fullModuleName = moduleName; + fullModuleName << it.key(); + collectPropertiesForExportItem(fullModuleName, it.value(), moduleInstance, moduleProps); + } +} + +void ExportsResolver::collectPropertiesForExportItem(Item *productModuleInstance) +{ + if (!productModuleInstance->isPresentModule()) + return; + Item * const exportItem = productModuleInstance->prototype(); + QBS_CHECK(exportItem); + QBS_CHECK(exportItem->type() == ItemType::Export); + const ItemDeclaration::Properties exportDecls = BuiltinDeclarations::instance() + .declarationsForType(ItemType::Export).properties(); + ExportedModule &exportedModule = m_product.product->exportedModule; + const auto &props = exportItem->properties(); + for (auto it = props.begin(); it != props.end(); ++it) { + const auto match + = [it](const PropertyDeclaration &decl) { return decl.name() == it.key(); }; + if (it.key() != StringConstants::prefixMappingProperty() && + std::find_if(exportDecls.begin(), exportDecls.end(), match) != exportDecls.end()) { + continue; + } + if (it.value()->type() == Value::ItemValueType) { + collectPropertiesForExportItem(it.key(), it.value(), productModuleInstance, + exportedModule.modulePropertyValues); + } else { + TempScopeSetter tss(it.value(), productModuleInstance); + m_propertiesEvaluator.evaluateProperty( + exportItem, it.key(), it.value(), exportedModule.propertyValues, false); + } + } +} + +// Collects module properties assigned to in other (higher-level) modules. +void ExportsResolver::collectPropertiesForModuleInExportItem(const Item::Module &module) +{ + if (!module.item->isPresentModule()) + return; + ExportedModule &exportedModule = m_product.product->exportedModule; + if (module.product || module.name.first() == StringConstants::qbsModule()) + return; + const auto checkName = [module](const ExportedModuleDependency &d) { + return module.name.toString() == d.name; + }; + if (any_of(exportedModule.moduleDependencies, checkName)) + return; + + Item *modulePrototype = module.item->prototype(); + while (modulePrototype && modulePrototype->type() != ItemType::Module) + modulePrototype = modulePrototype->prototype(); + if (!modulePrototype) // Can happen for broken products in relaxed mode. + return; + ModuleItemLocker locker(*modulePrototype); + const Item::PropertyMap &props = modulePrototype->properties(); + ExportedModuleDependency dep; + dep.name = module.name.toString(); + for (auto it = props.begin(); it != props.end(); ++it) { + if (it.value()->type() == Value::ItemValueType) + collectPropertiesForExportItem(it.key(), it.value(), module.item, dep.moduleProperties); + } + exportedModule.moduleDependencies.push_back(dep); + + for (const auto &dep : module.item->modules()) + collectPropertiesForModuleInExportItem(dep); +} + +void ExportsResolver::adaptExportedPropertyValues() +{ + QBS_CHECK(m_product.shadowProduct); + ExportedModule &m = m_product.product->exportedModule; + const QVariantList prefixList = m.propertyValues.take( + StringConstants::prefixMappingProperty()).toList(); + const QString shadowProductName = m_loaderState.evaluator().stringValue( + m_product.shadowProduct->item, StringConstants::nameProperty()); + const QString shadowProductBuildDir = m_loaderState.evaluator().stringValue( + m_product.shadowProduct->item, StringConstants::buildDirectoryProperty()); + QVariantMap prefixMap; + for (const QVariant &v : prefixList) { + const QVariantMap o = v.toMap(); + prefixMap.insert(o.value(QStringLiteral("prefix")).toString(), + o.value(QStringLiteral("replacement")).toString()); + } + const auto valueRefersToImportingProduct + = [shadowProductName, shadowProductBuildDir](const QString &value) { + return value.toLower().contains(shadowProductName.toLower()) + || value.contains(shadowProductBuildDir); + }; + static const auto stringMapper = [](const QVariantMap &mappings, const QString &value) + -> QString { + for (auto it = mappings.cbegin(); it != mappings.cend(); ++it) { + if (value.startsWith(it.key())) + return it.value().toString() + value.mid(it.key().size()); + } + return value; + }; + const auto stringListMapper = [&valueRefersToImportingProduct]( + const QVariantMap &mappings, const QStringList &value) -> QStringList { + QStringList result; + result.reserve(value.size()); + for (const QString &s : value) { + if (!valueRefersToImportingProduct(s)) + result.push_back(stringMapper(mappings, s)); + } + return result; + }; + const std::function<QVariant(const QVariantMap &, const QVariant &)> mapper + = [&stringListMapper, &mapper]( + const QVariantMap &mappings, const QVariant &value) -> QVariant { + switch (static_cast<QMetaType::Type>(value.userType())) { + case QMetaType::QString: + return stringMapper(mappings, value.toString()); + case QMetaType::QStringList: + return stringListMapper(mappings, value.toStringList()); + case QMetaType::QVariantMap: { + QVariantMap m = value.toMap(); + for (auto it = m.begin(); it != m.end(); ++it) + it.value() = mapper(mappings, it.value()); + return m; + } + default: + return value; + } + }; + for (auto it = m.propertyValues.begin(); it != m.propertyValues.end(); ++it) + it.value() = mapper(prefixMap, it.value()); + for (auto it = m.modulePropertyValues.begin(); it != m.modulePropertyValues.end(); ++it) + it.value() = mapper(prefixMap, it.value()); + for (ExportedModuleDependency &dep : m.moduleDependencies) { + for (auto it = dep.moduleProperties.begin(); it != dep.moduleProperties.end(); ++it) + it.value() = mapper(prefixMap, it.value()); + } +} + +void ExportsResolver::collectExportedProductDependencies() +{ + if (!m_product.shadowProduct) + return; + const ResolvedProductPtr exportingProduct = m_product.product; + if (!exportingProduct || !exportingProduct->enabled) + return; + Item * const importingProductItem = m_product.shadowProduct->item; + + std::vector<std::pair<ResolvedProductPtr, QVariantMap>> directDeps; + for (const Item::Module &m : importingProductItem->modules()) { + if (m.name.toString() != exportingProduct->name) + continue; + for (const Item::Module &dep : m.item->modules()) { + if (dep.product) + directDeps.emplace_back(dep.product->product, m.parameters); + } + } + for (const auto &dep : directDeps) { + if (!contains(exportingProduct->exportedModule.productDependencies, + dep.first->uniqueName())) { + exportingProduct->exportedModule.productDependencies.push_back( + dep.first->uniqueName()); + } + if (!dep.second.isEmpty()) { + exportingProduct->exportedModule.dependencyParameters.insert(dep.first, + dep.second); + } + } + auto &productDeps = exportingProduct->exportedModule.productDependencies; + std::sort(productDeps.begin(), productDeps.end()); +} + +QVariantMap PropertiesEvaluator::evaluateProperties( + const Item *item, const Item *propertiesContainer, const QVariantMap &tmplt, + bool lookupPrototype, bool checkErrors) +{ + AccumulatingTimer propEvalTimer(m_loaderState.parameters().logElapsedTime() + ? &m_product.timingData.propertyEvaluation + : nullptr); + QVariantMap result = tmplt; + for (auto it = propertiesContainer->properties().begin(); + it != propertiesContainer->properties().end(); ++it) { + evaluateProperty(item, it.key(), it.value(), result, checkErrors); + } + return lookupPrototype && propertiesContainer->prototype() + && propertiesContainer->prototype()->type() != ItemType::Module + ? evaluateProperties(item, propertiesContainer->prototype(), result, true, checkErrors) + : result; +} + +QVariantMap PropertiesEvaluator::evaluateProperties(Item *item, bool lookupPrototype, + bool checkErrors) +{ + const QVariantMap tmplt; + return evaluateProperties(item, item, tmplt, lookupPrototype, checkErrors); +} + +void PropertiesEvaluator::evaluateProperty( + const Item *item, const QString &propName, const ValuePtr &propValue, QVariantMap &result, + bool checkErrors) +{ + JSContext * const ctx = m_loaderState.evaluator().engine()->context(); + switch (propValue->type()) { + case Value::ItemValueType: + { + // Ignore items. Those point to module instances + // and are handled in evaluateModuleValues(). + break; + } + case Value::JSSourceValueType: + { + if (result.contains(propName)) + break; + const PropertyDeclaration pd = item->propertyDeclaration(propName); + if (pd.flags().testFlag(PropertyDeclaration::PropertyNotAvailableInConfig)) { + break; + } + const ScopedJsValue scriptValue(ctx, m_loaderState.evaluator().property(item, propName)); + if (JsException ex = m_loaderState.evaluator().engine()->checkAndClearException( + propValue->location())) { + if (checkErrors) + throw ex.toErrorInfo(); + } + + // NOTE: Loses type information if scriptValue.isUndefined == true, + // as such QScriptValues become invalid QVariants. + QVariant v; + if (JS_IsFunction(ctx, scriptValue)) { + v = getJsString(ctx, scriptValue); + } else { + v = getJsVariant(ctx, scriptValue); + QVariantMap m = v.toMap(); + if (m.contains(StringConstants::importScopeNamePropertyInternal())) { + QVariantMap tmp = m; + const ScopedJsValue proto(ctx, JS_GetPrototype(ctx, scriptValue)); + m = getJsVariant(ctx, proto).toMap(); + for (auto it = tmp.begin(); it != tmp.end(); ++it) + m.insert(it.key(), it.value()); + v = m; + } + } + + if (pd.type() == PropertyDeclaration::Path && v.isValid()) { + v = v.toString(); + } else if (pd.type() == PropertyDeclaration::PathList + || pd.type() == PropertyDeclaration::StringList) { + v = v.toStringList(); + } else if (pd.type() == PropertyDeclaration::VariantList) { + v = v.toList(); + } + pd.checkAllowedValues(v, propValue->location(), propName, m_loaderState); + result[propName] = v; + break; + } + case Value::VariantValueType: + { + if (result.contains(propName)) + break; + VariantValuePtr vvp = std::static_pointer_cast<VariantValue>(propValue); + QVariant v = vvp->value(); + + const PropertyDeclaration pd = item->propertyDeclaration(propName); + if (v.isNull() && !pd.isScalar()) // QTBUG-51237 + v = QStringList(); + + pd.checkAllowedValues(v, propValue->location(), propName, m_loaderState); + result[propName] = v; + break; + } + } +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productshandler.h b/src/lib/corelib/loader/productresolver.h index f1fc74a20..34c49b2bb 100644 --- a/src/lib/corelib/loader/productshandler.h +++ b/src/lib/corelib/loader/productresolver.h @@ -39,28 +39,19 @@ #pragma once -#include <tools/pimpl.h> - namespace qbs::Internal { +enum class Deferral; class LoaderState; +class ProductContext; // Responsibilities: // - Resolving dependencies to modules and other products (via DependenciesResolver). // - Module validation. // - Running probes (via ProbesResolver) in Product and Module items. -// - Preparing Group items for property evaluation. -class ProductsHandler -{ -public: - ProductsHandler(LoaderState &loaderState); - ~ProductsHandler(); - - void run(); - void printProfilingInfo(int indent); +// - Evaluating product and module properties. +// - Handling all Product child items, such as FileTaggers, Rules and so on. +void resolveProduct(ProductContext &product, Deferral deferral, LoaderState &loaderState); -private: - class Private; - Pimpl<Private> d; -}; +void setupExports(ProductContext &product, LoaderState &loaderState); } // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productscollector.cpp b/src/lib/corelib/loader/productscollector.cpp index 1c93f175f..85d425b8f 100644 --- a/src/lib/corelib/loader/productscollector.cpp +++ b/src/lib/corelib/loader/productscollector.cpp @@ -73,9 +73,11 @@ class ProductsCollector::Private public: Private(LoaderState &loaderState) : loaderState(loaderState) {} - void handleProject(Item *projectItem, const Set<QString> &referencedFilePaths); + void handleProject(Item *projectItem, ProjectContext *parentProject, + const Set<QString> &referencedFilePaths); QList<Item *> multiplexProductItem(ProductContext &dummyContext, Item *productItem); - void prepareProduct(ProjectContext &projectContext, Item *productItem); + void prepareProduct(ProjectContext &projectContext, Item *productItem, + ProductContext *mainProduct = nullptr); void handleSubProject(ProjectContext &projectContext, Item *projectItem, const Set<QString> &referencedFilePaths); void copyProperties(const Item *sourceProject, Item *targetProject); @@ -86,17 +88,18 @@ public: bool checkExportItemCondition(Item *exportItem, const ProductContext &product); void initProductProperties(const ProductContext &product); void checkProjectNamesInOverrides(); - void collectProductsByName(); + void collectProductsByNameAndItem(); void checkProductNamesInOverrides(); + void mergeProperty(Item *dst, const QString &name, const ValuePtr &value); LoaderState &loaderState; Settings settings{loaderState.parameters().settingsDirectory()}; Set<QString> disabledProjects; Version qbsVersion; Item *tempScopeItem = nullptr; - qint64 elapsedTimePrepareProducts = 0; private: + // TODO: Put this in loaderutils class TempBaseModuleAttacher { public: TempBaseModuleAttacher(Private *d, ProductContext &product); @@ -118,24 +121,15 @@ ProductsCollector::~ProductsCollector() = default; void ProductsCollector::run(Item *rootProject) { - d->handleProject(rootProject, {QDir::cleanPath(d->loaderState.parameters().projectFilePath())}); + d->handleProject(rootProject, nullptr, {rootProject->file()->filePath()}); d->checkProjectNamesInOverrides(); - d->collectProductsByName(); + d->collectProductsByNameAndItem(); d->checkProductNamesInOverrides(); + d->loaderState.topLevelProject().checkForLocalProfileAsTopLevelProfile( + d->loaderState.parameters().topLevelProfile()); } -void ProductsCollector::printProfilingInfo(int indent) -{ - if (!d->loaderState.parameters().logElapsedTime()) - return; - const QByteArray prefix(indent, ' '); - d->loaderState.logger().qbsLog(LoggerInfo, true) - << prefix - << Tr::tr("Preparing products took %1.") - .arg(elapsedTimeString(d->elapsedTimePrepareProducts)); -} - -void ProductsCollector::Private::handleProject(Item *projectItem, +void ProductsCollector::Private::handleProject(Item *projectItem, ProjectContext *parentProject, const Set<QString> &referencedFilePaths) { const SetupProjectParameters ¶meters = loaderState.parameters(); @@ -146,9 +140,11 @@ void ProductsCollector::Private::handleProject(Item *projectItem, auto p = std::make_unique<ProjectContext>(); auto &projectContext = *p; + projectContext.item = projectItem; + projectContext.parent = parentProject; projectContext.topLevelProject = &topLevelProject; ItemValuePtr itemValue = ItemValue::create(projectItem); - projectContext.scope = Item::create(projectItem->pool(), ItemType::Scope); + projectContext.scope = Item::create(&loaderState.itemPool(), ItemType::Scope); projectContext.scope->setFile(projectItem->file()); projectContext.scope->setProperty(StringConstants::projectVar(), itemValue); ProductContext dummyProduct; @@ -156,7 +152,7 @@ void ProductsCollector::Private::handleProject(Item *projectItem, dummyProduct.moduleProperties = parameters.finalBuildConfigurationTree(); dummyProduct.profileModuleProperties = dummyProduct.moduleProperties; dummyProduct.profileName = parameters.topLevelProfile(); - loaderState.dependenciesResolver().loadBaseModule(dummyProduct, projectItem); + loadBaseModule(dummyProduct, projectItem, loaderState); projectItem->overrideProperties(parameters.overriddenValuesTree(), StringConstants::projectPrefix(), parameters, logger); @@ -166,6 +162,9 @@ void ProductsCollector::Private::handleProject(Item *projectItem, projectItem->setProperty(StringConstants::nameProperty(), VariantValue::create(projectContext.name)); } + if (parentProject) + parentProject->children.push_back(p.get()); + topLevelProject.addProject(p.release()); projectItem->overrideProperties(parameters.overriddenValuesTree(), StringConstants::projectsOverridePrefix() + projectContext.name, parameters, logger); @@ -173,11 +172,9 @@ void ProductsCollector::Private::handleProject(Item *projectItem, disabledProjects.insert(projectContext.name); return; } - topLevelProject.projects.push_back(p.release()); SearchPathsManager searchPathsManager(itemReader, itemReader.readExtraSearchPaths(projectItem) << projectItem->file()->dirPath()); projectContext.searchPathsStack = itemReader.extraSearchPathsStack(); - projectContext.item = projectItem; const QString minVersionStr = evaluator.stringValue(projectItem, StringConstants::minimumQbsVersionProperty(), @@ -199,10 +196,9 @@ void ProductsCollector::Private::handleProject(Item *projectItem, for (Item * const child : projectItem->children()) child->setScope(projectContext.scope); - projectContext.topLevelProject->probes << loaderState.probesResolver().resolveProbes( - {dummyProduct.name, dummyProduct.uniqueName()}, projectItem); + ProbesResolver(loaderState).resolveProbes(dummyProduct, projectItem); - loaderState.localProfiles().collectProfilesFromItems(projectItem, projectContext.scope); + collectProfilesFromItems(projectItem, projectContext.scope, loaderState); QList<Item *> multiplexedProducts; for (Item * const child : projectItem->children()) { @@ -223,7 +219,7 @@ void ProductsCollector::Private::handleProject(Item *projectItem, break; case ItemType::Project: copyProperties(projectItem, child); - handleProject(child, referencedFilePaths); + handleProject(child, &projectContext, referencedFilePaths); break; default: break; @@ -253,7 +249,7 @@ void ProductsCollector::Private::handleProject(Item *projectItem, break; case ItemType::Project: copyProperties(projectItem, subItem); - handleProject(subItem, + handleProject(subItem, &projectContext, Set<QString>(referencedFilePaths) << subItem->file()->filePath()); break; default: @@ -278,28 +274,36 @@ QList<Item *> ProductsCollector::Private::multiplexProductItem(ProductContext &d loaderState.parameters(), loaderState.logger()); dummyContext.item = productItem; TempBaseModuleAttacher tbma(this, dummyContext); - return loaderState.multiplexer().multiplex(productName, productItem, tbma.tempBaseModuleItem(), - [&] { tbma.drop(); }); + return multiplex(productName, productItem, tbma.tempBaseModuleItem(), + [&] { tbma.drop(); }, loaderState); } -void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, Item *productItem) +void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, Item *productItem, + ProductContext *mainProduct) { const SetupProjectParameters ¶meters = loaderState.parameters(); Evaluator &evaluator = loaderState.evaluator(); TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); - AccumulatingTimer timer(parameters.logElapsedTime() ? &elapsedTimePrepareProducts : nullptr); - topLevelProject.checkCancelation(parameters); + AccumulatingTimer timer(parameters.logElapsedTime() + ? &topLevelProject.timingData().preparingProducts : nullptr); + topLevelProject.checkCancelation(); qCDebug(lcModuleLoader) << "prepareProduct" << productItem->file()->filePath(); - ProductContext productContext; + if (mainProduct) + mainProduct->shadowProduct = std::make_unique<ProductContext>(); + else + projectContext.products.emplace_back(); + ProductContext &productContext = mainProduct + ? *mainProduct->shadowProduct : projectContext.products.back(); productContext.item = productItem; productContext.project = &projectContext; // Retrieve name, profile and multiplex id. productContext.name = evaluator.stringValue(productItem, StringConstants::nameProperty()); QBS_CHECK(!productContext.name.isEmpty()); - const ItemValueConstPtr qbsItemValue = productItem->itemProperty(StringConstants::qbsModule()); + const ItemValueConstPtr qbsItemValue = productItem->itemProperty(StringConstants::qbsModule(), + loaderState.itemPool()); if (qbsItemValue && qbsItemValue->item()->hasProperty(StringConstants::profileProperty())) { TempBaseModuleAttacher tbma(this, productContext); productContext.profileName = evaluator.stringValue( @@ -312,11 +316,11 @@ void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, QBS_CHECK(!productContext.profileName.isEmpty()); // Set up full module property map based on the profile. - const auto it = topLevelProject.profileConfigs.constFind(productContext.profileName); - QVariantMap flatConfig; - if (it == topLevelProject.profileConfigs.constEnd()) { + std::optional<QVariantMap> flatConfig + = topLevelProject.profileConfig(productContext.profileName); + if (!flatConfig) { const Profile profile(productContext.profileName, &settings, - loaderState.localProfiles().profiles()); + loaderState.topLevelProject().localProfiles()); if (!profile.exists()) { ErrorInfo error(Tr::tr("Profile '%1' does not exist.").arg(profile.name()), productItem->location()); @@ -325,20 +329,18 @@ void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, } flatConfig = SetupProjectParameters::expandedBuildConfiguration( profile, parameters.configurationName()); - topLevelProject.profileConfigs.insert(productContext.profileName, flatConfig); - } else { - flatConfig = it.value().toMap(); + topLevelProject.addProfileConfig(productContext.profileName, *flatConfig); } productContext.profileModuleProperties = SetupProjectParameters::finalBuildConfigurationTree( - flatConfig, {}); + *flatConfig, {}); productContext.moduleProperties = SetupProjectParameters::finalBuildConfigurationTree( - flatConfig, parameters.overriddenValues()); + *flatConfig, parameters.overriddenValues()); initProductProperties(productContext); // Set up product scope. This is mainly for using the "product" and "project" // variables in some contexts. ItemValuePtr itemValue = ItemValue::create(productItem); - productContext.scope = Item::create(productItem->pool(), ItemType::Scope); + productContext.scope = Item::create(&loaderState.itemPool(), ItemType::Scope); productContext.scope->setProperty(StringConstants::productVar(), itemValue); productContext.scope->setFile(productItem->file()); productContext.scope->setScope(productContext.project->scope); @@ -352,7 +354,7 @@ void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, if (child->id().isEmpty()) continue; if (productItem->scope() == productContext.project->scope) { - productItem->setScope(Item::create(productItem->pool(), ItemType::Scope)); + productItem->setScope(Item::create(&loaderState.itemPool(), ItemType::Scope)); productItem->scope()->setScope(productContext.project->scope); } const ItemValuePtr childValue = ItemValue::create(child); @@ -364,15 +366,13 @@ void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, setScopeForDescendants(productItem, productContext.scope); - projectContext.products.push_back(productContext); - - if (!hasExportItems || getShadowProductInfo(productContext).first) + if (!hasExportItems) return; // This "shadow product" exists only to pull in a dependency on the actual product // and nothing else, thus providing us with the pure environment that we need to // evaluate the product's exported properties in isolation in the project resolver. - Item * const importer = Item::create(productItem->pool(), ItemType::Product); + Item * const importer = Item::create(&loaderState.itemPool(), ItemType::Product); importer->setProperty(QStringLiteral("name"), VariantValue::create(StringConstants::shadowProductPrefix() + productContext.name)); @@ -380,15 +380,16 @@ void ProductsCollector::Private::prepareProduct(ProjectContext &projectContext, importer->setLocation(productItem->location()); importer->setScope(projectContext.scope); importer->setupForBuiltinType(parameters.deprecationWarningMode(), loaderState.logger()); - Item * const dependsItem = Item::create(productItem->pool(), ItemType::Depends); + Item * const dependsItem = Item::create(&loaderState.itemPool(), ItemType::Depends); dependsItem->setProperty(QStringLiteral("name"), VariantValue::create(productContext.name)); dependsItem->setProperty(QStringLiteral("required"), VariantValue::create(false)); dependsItem->setFile(importer->file()); dependsItem->setLocation(importer->location()); dependsItem->setupForBuiltinType(parameters.deprecationWarningMode(), loaderState.logger()); + dependsItem->setProperty(StringConstants::multiplexConfigurationIdsProperty(), + VariantValue::create(productContext.multiplexConfigurationId)); Item::addChild(importer, dependsItem); - Item::addChild(productItem, importer); - prepareProduct(projectContext, importer); + prepareProduct(projectContext, importer, &productContext); } void ProductsCollector::Private::handleSubProject( @@ -443,7 +444,7 @@ void ProductsCollector::Private::handleSubProject( Item::addChild(projectItem, loadedItem); projectItem->setScope(projectContext.scope); - handleProject(loadedItem, Set<QString>(referencedFilePaths) << subProjectFilePath); + handleProject(loadedItem, &projectContext, Set<QString>(referencedFilePaths) << subProjectFilePath); } void ProductsCollector::Private::copyProperties(const Item *sourceProject, Item *targetProject) @@ -524,18 +525,19 @@ QList<Item *> ProductsCollector::Private::loadReferencedFile( QList<Item *> loadedItems; loadedItems << subItem; if (subItem->type() == ItemType::Product) { - loaderState.localProfiles().collectProfilesFromItems(subItem, dummyContext.project->scope); + collectProfilesFromItems(subItem, dummyContext.project->scope, loaderState); loadedItems << multiplexProductItem(dummyContext, subItem); } return loadedItems; } -static void mergeProperty(Item *dst, const QString &name, const ValuePtr &value) +void ProductsCollector::Private::mergeProperty(Item *dst, const QString &name, + const ValuePtr &value) { if (value->type() == Value::ItemValueType) { const ItemValueConstPtr itemValue = std::static_pointer_cast<ItemValue>(value); const Item * const valueItem = itemValue->item(); - Item * const subItem = dst->itemProperty(name, itemValue)->item(); + Item * const subItem = dst->itemProperty(name, itemValue, loaderState.itemPool())->item(); for (auto it = valueItem->properties().begin(); it != valueItem->properties().end(); ++it) mergeProperty(subItem, it.key(), it.value()); return; @@ -550,7 +552,7 @@ static void mergeProperty(Item *dst, const QString &name, const ValuePtr &value) if (baseValue) { QBS_CHECK(baseValue->type() == Value::JSSourceValueType); const JSSourceValuePtr jsBaseValue = std::static_pointer_cast<JSSourceValue>( - baseValue->clone()); + baseValue->clone(loaderState.itemPool())); jsValue->setBaseValue(jsBaseValue); std::vector<JSSourceValue::Alternative> alternatives = jsValue->alternatives(); jsValue->clearAlternatives(); @@ -583,14 +585,14 @@ bool ProductsCollector::Private::mergeExportItems(ProductContext &productContext if (!exportItems.empty()) productContext.item->setChildren(children); - Item *merged = Item::create(productContext.item->pool(), ItemType::Export); + Item *merged = Item::create(&loaderState.itemPool(), ItemType::Export); const QString &nameKey = StringConstants::nameProperty(); const ValuePtr nameValue = VariantValue::create(productContext.name); merged->setProperty(nameKey, nameValue); Set<FileContextConstPtr> filesWithExportItem; QVariantMap defaultParameters; for (Item * const exportItem : exportItems) { - topLevelProject.checkCancelation(parameters); + topLevelProject.checkCancelation(); if (Q_UNLIKELY(filesWithExportItem.contains(exportItem->file()))) throw ErrorInfo(Tr::tr("Multiple Export items in one product are prohibited."), exportItem->location()); @@ -643,11 +645,11 @@ bool ProductsCollector::Private::checkExportItemCondition(Item *exportItem, { class ScopeHandler { public: - ScopeHandler(Item *exportItem, const ProductContext &productContext, Item **cachedScopeItem) - : m_exportItem(exportItem) + ScopeHandler(Item *exportItem, const ProductContext &productContext, Item **cachedScopeItem, + ItemPool &itemPool) : m_exportItem(exportItem), m_itemPool(itemPool) { if (!*cachedScopeItem) - *cachedScopeItem = Item::create(exportItem->pool(), ItemType::Scope); + *cachedScopeItem = Item::create(&m_itemPool, ItemType::Scope); Item * const scope = *cachedScopeItem; QBS_CHECK(productContext.item->file()); scope->setFile(productContext.item->file()); @@ -661,7 +663,8 @@ bool ProductsCollector::Private::checkExportItemCondition(Item *exportItem, private: Item * const m_exportItem; - } scopeHandler(exportItem, product, &tempScopeItem); + ItemPool &m_itemPool; + } scopeHandler(exportItem, product, &tempScopeItem, loaderState.itemPool()); return loaderState.topLevelProject().checkItemCondition(exportItem, loaderState.evaluator()); } @@ -669,7 +672,7 @@ void ProductsCollector::Private::initProductProperties(const ProductContext &pro { QString buildDir = ResolvedProduct::deriveBuildDirectoryName(product.name, product.multiplexConfigurationId); - buildDir = FileInfo::resolvePath(product.project->topLevelProject->buildDirectory, buildDir); + buildDir = FileInfo::resolvePath(product.project->topLevelProject->buildDirectory(), buildDir); product.item->setProperty(StringConstants::buildDirectoryProperty(), VariantValue::create(buildDir)); const QString sourceDir = QFileInfo(product.item->file()->filePath()).absolutePath(); @@ -680,10 +683,10 @@ void ProductsCollector::Private::initProductProperties(const ProductContext &pro void ProductsCollector::Private::checkProjectNamesInOverrides() { for (const QString &projectNameInOverride - : loaderState.topLevelProject().projectNamesUsedInOverrides) { + : loaderState.topLevelProject().projectNamesUsedInOverrides()) { if (disabledProjects.contains(projectNameInOverride)) continue; - if (!any_of(loaderState.topLevelProject().projects, + if (!any_of(loaderState.topLevelProject().projects(), [&projectNameInOverride](const ProjectContext *p) { return p->name == projectNameInOverride; })) { handlePropertyError(Tr::tr("Unknown project '%1' in property override.") @@ -693,28 +696,26 @@ void ProductsCollector::Private::checkProjectNamesInOverrides() } } -void ProductsCollector::Private::collectProductsByName() +void ProductsCollector::Private::collectProductsByNameAndItem() { TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); - for (ProjectContext * const project : topLevelProject.projects) { + for (ProjectContext * const project : topLevelProject.projects()) { for (ProductContext &product : project->products) - topLevelProject.productsByName.insert({product.name, &product}); + topLevelProject.addProduct(product); } } void ProductsCollector::Private::checkProductNamesInOverrides() { TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); - for (const QString &productNameInOverride : topLevelProject.productNamesUsedInOverrides) { - if (topLevelProject.erroneousProducts.contains(productNameInOverride)) - continue; - if (!any_of(topLevelProject.productsByName, [&productNameInOverride]( - const std::pair<QString, ProductContext *> &elem) { + for (const QString &productNameInOverride : topLevelProject.productNamesUsedInOverrides()) { + if (!topLevelProject.productWithNameAndConstraint( + productNameInOverride, [&productNameInOverride](const ProductContext &product) { // In an override string such as "a.b.c:d, we cannot tell whether we have a product // "a" and a module "b.c" or a product "a.b" and a module "c", so we need to take // care not to emit false positives here. - return elem.first == productNameInOverride - || elem.first.startsWith(productNameInOverride + StringConstants::dot()); + return product.name == productNameInOverride + || product.name.startsWith(productNameInOverride + StringConstants::dot()); })) { handlePropertyError(Tr::tr("Unknown product '%1' in property override.") .arg(productNameInOverride), @@ -731,9 +732,9 @@ ProductsCollector::Private::TempBaseModuleAttacher::TempBaseModuleAttacher( // Cloning is necessary because the original value will get "instantiated" now. if (qbsValue) - m_origQbsValue = qbsValue->clone(); + m_origQbsValue = qbsValue->clone(d->loaderState.itemPool()); - m_tempBaseModule = d->loaderState.dependenciesResolver().loadBaseModule(product, m_productItem); + m_tempBaseModule = loadBaseModule(product, m_productItem, d->loaderState); } void ProductsCollector::Private::TempBaseModuleAttacher::drop() diff --git a/src/lib/corelib/loader/productscollector.h b/src/lib/corelib/loader/productscollector.h index fa2059872..246361b37 100644 --- a/src/lib/corelib/loader/productscollector.h +++ b/src/lib/corelib/loader/productscollector.h @@ -54,7 +54,6 @@ public: ~ProductsCollector(); void run(Item *rootProject); - void printProfilingInfo(int indent); private: class Private; diff --git a/src/lib/corelib/loader/productshandler.cpp b/src/lib/corelib/loader/productshandler.cpp deleted file mode 100644 index e0ca79acc..000000000 --- a/src/lib/corelib/loader/productshandler.cpp +++ /dev/null @@ -1,336 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2023 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of Qbs. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "productshandler.h" - -#include "dependenciesresolver.h" -#include "groupshandler.h" -#include "itemreader.h" -#include "loaderutils.h" -#include "modulepropertymerger.h" -#include "probesresolver.h" - -#include <language/evaluator.h> -#include <language/item.h> -#include <language/value.h> -#include <logging/categories.h> -#include <logging/translator.h> -#include <tools/profiling.h> -#include <tools/setupprojectparameters.h> -#include <tools/stringconstants.h> - -namespace qbs::Internal { - -class ProductsHandler::Private -{ -public: - Private(LoaderState &loaderState) : loaderState(loaderState) {} - - void handleNextProduct(); - void handleProduct(ProductContext &product, Deferral deferral); - void resolveProbes(ProductContext &product); - void resolveProbes(ProductContext &product, Item *item); - void handleModuleSetupError(ProductContext &product, const Item::Module &module, - const ErrorInfo &error); - void runModuleProbes(ProductContext &product, const Item::Module &module); - bool validateModule(ProductContext &product, const Item::Module &module); - void updateModulePresentState(ProductContext &product, const Item::Module &module); - void handleGroups(ProductContext &product); - - LoaderState &loaderState; - GroupsHandler groupsHandler{loaderState}; - qint64 elapsedTime = 0; -}; - -ProductsHandler::ProductsHandler(LoaderState &loaderState) : d(makePimpl<Private>(loaderState)) {} - -void ProductsHandler::run() -{ - AccumulatingTimer timer(d->loaderState.parameters().logElapsedTime() - ? &d->elapsedTime : nullptr); - - TopLevelProjectContext &topLevelProject = d->loaderState.topLevelProject(); - for (ProjectContext * const projectContext : topLevelProject.projects) { - for (ProductContext &productContext : projectContext->products) - topLevelProject.productsToHandle.emplace_back(&productContext, -1); - } - while (!topLevelProject.productsToHandle.empty()) - d->handleNextProduct(); -} - -void ProductsHandler::printProfilingInfo(int indent) -{ - if (!d->loaderState.parameters().logElapsedTime()) - return; - const QByteArray prefix(indent, ' '); - d->loaderState.logger().qbsLog(LoggerInfo, true) - << prefix - << Tr::tr("Handling products took %1.") - .arg(elapsedTimeString(d->elapsedTime)); - d->groupsHandler.printProfilingInfo(indent + 2); -} - -ProductsHandler::~ProductsHandler() = default; - -void ProductsHandler::Private::handleNextProduct() -{ - TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); - - auto [product, queueSizeOnInsert] = topLevelProject.productsToHandle.front(); - topLevelProject.productsToHandle.pop_front(); - - // If the queue of in-progress products has shrunk since the last time we tried handling - // this product, there has been forward progress and we can allow a deferral. - const Deferral deferral = queueSizeOnInsert == -1 - || queueSizeOnInsert > int(topLevelProject.productsToHandle.size()) - ? Deferral::Allowed : Deferral::NotAllowed; - - loaderState.itemReader().setExtraSearchPathsStack(product->project->searchPathsStack); - try { - handleProduct(*product, deferral); - if (product->name.startsWith(StringConstants::shadowProductPrefix())) - topLevelProject.probes << product->info.probes; - } catch (const ErrorInfo &err) { - product->handleError(err); - } - - // The search paths stack can change during dependency resolution (due to module providers); - // check that we've rolled back all the changes - QBS_CHECK(loaderState.itemReader().extraSearchPathsStack() == product->project->searchPathsStack); - - // If we encountered a dependency to an in-progress product or to a bulk dependency, - // we defer handling this product if it hasn't failed yet and there is still forward progress. - if (!product->info.delayedError.hasError() && !product->dependenciesResolved) { - topLevelProject.productsToHandle.emplace_back( - product, int(topLevelProject.productsToHandle.size())); - } -} - -void ProductsHandler::Private::handleProduct(ProductContext &product, Deferral deferral) -{ - TopLevelProjectContext &topLevelProject = loaderState.topLevelProject(); - topLevelProject.checkCancelation(loaderState.parameters()); - - if (product.info.delayedError.hasError()) - return; - - product.dependenciesResolved = loaderState.dependenciesResolver() - .resolveDependencies(product, deferral); - if (!product.dependenciesResolved) - return; - - // Run probes for modules and product. - resolveProbes(product); - - // After the probes have run, we can switch on the evaluator cache. - Evaluator &evaluator = loaderState.evaluator(); - FileTags fileTags = evaluator.fileTagsValue(product.item, StringConstants::typeProperty()); - EvalCacheEnabler cacheEnabler(&evaluator, evaluator.stringValue( - product.item, - StringConstants::sourceDirectoryProperty())); - - // Run module validation scripts. - for (const Item::Module &module : product.item->modules()) { - if (!validateModule(product, module)) - return; - fileTags += evaluator.fileTagsValue( - module.item, StringConstants::additionalProductTypesProperty()); - } - - // Disable modules that have been pulled in only by now-disabled modules. - // Note that this has to happen in the reverse order compared to the other loops, - // with the leaves checked last. - for (auto it = product.item->modules().rbegin(); it != product.item->modules().rend(); ++it) - updateModulePresentState(product, *it); - - // Now do the canonical module property values merge. Note that this will remove - // previously attached values from modules that failed validation. - // Evaluator cache entries that could potentially change due to this will be purged. - loaderState.propertyMerger().doFinalMerge(product.item); - - const bool enabled = topLevelProject.checkItemCondition(product.item, evaluator); - loaderState.dependenciesResolver().checkDependencyParameterDeclarations( - product.item, product.name); - - handleGroups(product); - - // Collect the full list of fileTags, including the values contributed by modules. - if (!product.info.delayedError.hasError() && enabled - && !product.name.startsWith(StringConstants::shadowProductPrefix())) { - for (const FileTag &tag : fileTags) - topLevelProject.productsByType.insert({tag, &product}); - product.item->setProperty(StringConstants::typeProperty(), - VariantValue::create(sorted(fileTags.toStringList()))); - } - topLevelProject.productInfos[product.item] = product.info; -} - -void ProductsHandler::Private::resolveProbes(ProductContext &product) -{ - for (const Item::Module &module : product.item->modules()) { - runModuleProbes(product, module); - if (product.info.delayedError.hasError()) - return; - } - resolveProbes(product, product.item); -} - -void ProductsHandler::Private::resolveProbes(ProductContext &product, Item *item) -{ - product.info.probes << loaderState.probesResolver().resolveProbes( - {product.name, product.uniqueName()}, item); -} - -void ProductsHandler::Private::handleModuleSetupError( - ProductContext &product, const Item::Module &module, const ErrorInfo &error) -{ - if (module.required) { - product.handleError(error); - } else { - qCDebug(lcModuleLoader()) << "non-required module" << module.name.toString() - << "found, but not usable in product" << product.name - << error.toString(); - createNonPresentModule(*module.item->pool(), module.name.toString(), - QStringLiteral("failed validation"), module.item); - } -} - -void ProductsHandler::Private::runModuleProbes(ProductContext &product, const Item::Module &module) -{ - if (!module.item->isPresentModule()) - return; - if (module.productInfo && loaderState.topLevelProject().disabledItems.contains( - module.productInfo->item)) { - createNonPresentModule(*module.item->pool(), module.name.toString(), - QLatin1String("module's exporting product is disabled"), - module.item); - return; - } - try { - resolveProbes(product, module.item); - if (module.versionRange.minimum.isValid() - || module.versionRange.maximum.isValid()) { - if (module.versionRange.maximum.isValid() - && module.versionRange.minimum >= module.versionRange.maximum) { - throw ErrorInfo(Tr::tr("Impossible version constraint [%1,%2) set for module " - "'%3'").arg(module.versionRange.minimum.toString(), - module.versionRange.maximum.toString(), - module.name.toString())); - } - const Version moduleVersion = Version::fromString( - loaderState.evaluator().stringValue(module.item, - StringConstants::versionProperty())); - if (moduleVersion < module.versionRange.minimum) { - throw ErrorInfo(Tr::tr("Module '%1' has version %2, but it needs to be " - "at least %3.").arg(module.name.toString(), - moduleVersion.toString(), - module.versionRange.minimum.toString())); - } - if (module.versionRange.maximum.isValid() - && moduleVersion >= module.versionRange.maximum) { - throw ErrorInfo(Tr::tr("Module '%1' has version %2, but it needs to be " - "lower than %3.").arg(module.name.toString(), - moduleVersion.toString(), - module.versionRange.maximum.toString())); - } - } - } catch (const ErrorInfo &error) { - handleModuleSetupError(product, module, error); - } -} - -bool ProductsHandler::Private::validateModule(ProductContext &product, const Item::Module &module) -{ - if (!module.item->isPresentModule()) - return true; - try { - loaderState.evaluator().boolValue(module.item, StringConstants::validateProperty()); - for (const auto &dep : module.item->modules()) { - if (dep.required && !dep.item->isPresentModule()) { - throw ErrorInfo(Tr::tr("Module '%1' depends on module '%2', which was not " - "loaded successfully") - .arg(module.name.toString(), dep.name.toString())); - } - } - } catch (const ErrorInfo &error) { - handleModuleSetupError(product, module, error); - if (product.info.delayedError.hasError()) - return false; - } - return true; -} - -void ProductsHandler::Private::updateModulePresentState(ProductContext &product, - const Item::Module &module) -{ - if (!module.item->isPresentModule()) - return; - bool hasPresentLoadingItem = false; - for (const Item * const loadingItem : module.loadingItems) { - if (loadingItem == product.item) { - hasPresentLoadingItem = true; - break; - } - if (!loadingItem->isPresentModule()) - continue; - if (loadingItem->prototype() && loadingItem->prototype()->type() == ItemType::Export) { - QBS_CHECK(loadingItem->prototype()->parent()->type() == ItemType::Product); - if (loaderState.topLevelProject().disabledItems.contains( - loadingItem->prototype()->parent())) { - continue; - } - } - hasPresentLoadingItem = true; - break; - } - if (!hasPresentLoadingItem) { - createNonPresentModule(*module.item->pool(), module.name.toString(), - QLatin1String("imported only by disabled module(s)"), - module.item); - } -} - -void ProductsHandler::Private::handleGroups(ProductContext &product) -{ - groupsHandler.setupGroups(product.item, product.scope); - product.info.modulePropertiesSetInGroups = groupsHandler.modulePropertiesSetInGroups(); - loaderState.topLevelProject().disabledItems.unite(groupsHandler.disabledGroups()); -} - -} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productsresolver.cpp b/src/lib/corelib/loader/productsresolver.cpp new file mode 100644 index 000000000..997a4dc57 --- /dev/null +++ b/src/lib/corelib/loader/productsresolver.cpp @@ -0,0 +1,599 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "productsresolver.h" + +#include "itemreader.h" +#include "loaderutils.h" +#include "productresolver.h" + +#include <language/language.h> +#include <language/scriptengine.h> +#include <logging/categories.h> +#include <logging/translator.h> +#include <tools/profiling.h> +#include <tools/progressobserver.h> +#include <tools/setupprojectparameters.h> +#include <tools/stringconstants.h> + +#include <algorithm> +#include <condition_variable> +#include <future> +#include <mutex> +#include <queue> +#include <system_error> +#include <thread> +#include <unordered_map> +#include <vector> + +namespace qbs::Internal { +namespace { +struct ThreadInfo { + ThreadInfo(std::future<void> &&future, LoaderState &loaderState) + : future(std::move(future)), loaderState(loaderState) + {} + std::future<void> future; + LoaderState &loaderState; + bool done = false; +}; + +struct ProductWithLoaderState { + ProductWithLoaderState(ProductContext &product, LoaderState *loaderState) + : product(&product), loaderState(loaderState) {} + ProductContext * const product; + LoaderState *loaderState; +}; + +class ThreadsLocker { +public: + ThreadsLocker(std::launch mode, std::mutex &mutex) { + if (mode == std::launch::async) + lock = std::make_unique<std::unique_lock<std::mutex>>(mutex); + } + std::unique_ptr<std::unique_lock<std::mutex>> lock; +}; +} // namespace + +class ProductsResolver +{ +public: + ProductsResolver(LoaderState &loaderState) : m_loaderState(loaderState) {} + void resolve(); + +private: + void initialize(); + void initializeProductQueue(); + void initializeLoaderStatePool(); + void runScheduler(); + void scheduleNext(); + bool tryToReserveLoaderState(ProductWithLoaderState &product, Deferral deferral); + std::optional<std::pair<ProductContext *, Deferral>> + unblockProductWaitingForLoaderState(LoaderState &loaderState); + void startJob(const ProductWithLoaderState &product, Deferral deferral); + void checkForCancelation(); + void handleFinishedThreads(); + void queueProductForScheduling(const ProductWithLoaderState &product, Deferral deferral); + void waitForSingleDependency(const ProductWithLoaderState &product, ProductContext &dependency); + void waitForBulkDependency(const ProductWithLoaderState &product); + void unblockProductsWaitingForDependency(ProductContext &finishedProduct); + void postProcess(); + void checkForMissedBulkDependencies(const ProductContext &product); + + static int dependsItemCount(const Item *item); + static int dependsItemCount(ProductContext &product); + + LoaderState &m_loaderState; + std::queue<std::pair<ProductWithLoaderState, int>> m_productsToSchedule; + std::vector<ProductContext *> m_finishedProducts; + std::unordered_map<ProductContext *, + std::vector<ProductWithLoaderState>> m_waitingForSingleDependency; + std::vector<ProductWithLoaderState> m_waitingForBulkDependency; + std::unordered_map<LoaderState *, std::queue<std::pair<ProductContext *, Deferral>>> m_waitingForLoaderState; + std::unordered_map<ProductContext *, ThreadInfo> m_runningThreads; + std::mutex m_threadsMutex; + std::condition_variable m_threadsNotifier; + std::vector<std::unique_ptr<ScriptEngine>> m_enginePool; + std::vector<std::unique_ptr<LoaderState>> m_loaderStatePool; + std::vector<LoaderState *> m_availableLoaderStates; + std::mutex m_cancelingMutex; + std::launch m_asyncMode = std::launch::async; + int m_maxJobCount = m_loaderState.parameters().maxJobCount(); + bool m_canceling = false; +}; + +void resolveProducts(LoaderState &loaderState) +{ + ProductsResolver(loaderState).resolve(); +} + +void ProductsResolver::resolve() +{ + initialize(); + try { + runScheduler(); + } catch (const ErrorInfo &e) { + for (auto &thread : m_runningThreads) + thread.second.future.wait(); + throw e; + } + postProcess(); +} + +void ProductsResolver::initialize() +{ + initializeProductQueue(); + initializeLoaderStatePool(); +} + +void ProductsResolver::initializeProductQueue() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + std::vector<ProductContext *> sortedProducts; + for (ProjectContext * const projectContext : topLevelProject.projects()) { + for (ProductContext &product : projectContext->products) { + topLevelProject.addProductToHandle(product); + const auto it = std::lower_bound(sortedProducts.begin(), sortedProducts.end(), product, + [&product](ProductContext *p1, const ProductContext &) { + return dependsItemCount(*p1) < dependsItemCount(product); + }); + sortedProducts.insert(it, &product); + } + } + + for (ProductContext * const product : sortedProducts) { + queueProductForScheduling(ProductWithLoaderState(*product, nullptr), Deferral::Allowed); + if (product->shadowProduct) { + topLevelProject.addProductToHandle(*product->shadowProduct); + queueProductForScheduling(ProductWithLoaderState(*product->shadowProduct, nullptr), + Deferral::Allowed); + } + } +} + +void ProductsResolver::initializeLoaderStatePool() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + + // Adapt max job count: It makes no sense to have it be higher than the number of products + // or what can actually be run concurrently. In both cases, we would simply waste resources. + const int maxConcurrency = std::thread::hardware_concurrency(); + if (maxConcurrency > 0 && maxConcurrency < m_maxJobCount) + m_maxJobCount = maxConcurrency; + if (m_maxJobCount > topLevelProject.productsToHandleCount()) + m_maxJobCount = topLevelProject.productsToHandleCount(); + + // The number of engines and loader states we need to allocate here is one less than the + // total number of concurrent jobs, as we already have one loader state that we can re-use. + if (m_maxJobCount > 1) + m_enginePool.reserve(m_maxJobCount - 1); + m_loaderStatePool.reserve(m_enginePool.size()); + m_availableLoaderStates.reserve(m_enginePool.size() + 1); + m_availableLoaderStates.push_back(&m_loaderState); + for (std::size_t i = 0; i < m_enginePool.capacity(); ++i) { + ScriptEngine &engine = *m_enginePool.emplace_back( + ScriptEngine::create(m_loaderState.logger(), EvalContext::PropertyEvaluation)); + ItemPool &itemPool = topLevelProject.createItemPool(); + engine.setEnvironment(m_loaderState.parameters().adjustedEnvironment()); + auto loaderState = std::make_unique<LoaderState>( + m_loaderState.parameters(), topLevelProject, itemPool, engine, + m_loaderState.logger()); + m_loaderStatePool.push_back(std::move(loaderState)); + m_availableLoaderStates.push_back(m_loaderStatePool.back().get()); + if (topLevelProject.progressObserver()) + topLevelProject.progressObserver()->addScriptEngine(m_enginePool.back().get()); + } + qCDebug(lcLoaderScheduling) << "using" << m_availableLoaderStates.size() << "loader states"; + if (int(m_availableLoaderStates.size()) == 1) + m_asyncMode = std::launch::deferred; +} + +void ProductsResolver::runScheduler() +{ + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &m_loaderState.topLevelProject().timingData().resolvingProducts + : nullptr); + + ThreadsLocker threadsLock(m_asyncMode, m_threadsMutex); + while (true) { + scheduleNext(); + if (m_runningThreads.empty()) + break; + if (m_asyncMode == std::launch::async) { + qCDebug(lcLoaderScheduling()) << "scheduling paused, waiting for threads to finish"; + m_threadsNotifier.wait(*threadsLock.lock); + } + checkForCancelation(); + handleFinishedThreads(); + } + + QBS_CHECK(m_productsToSchedule.empty()); + QBS_CHECK(m_loaderState.topLevelProject().productsToHandleCount() == 0); + QBS_CHECK(m_runningThreads.empty()); + QBS_CHECK(m_waitingForSingleDependency.empty()); + QBS_CHECK(m_waitingForBulkDependency.empty()); +} + +void ProductsResolver::scheduleNext() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &topLevelProject.timingData().schedulingProducts : nullptr); + while (m_maxJobCount > int(m_runningThreads.size()) && !m_productsToSchedule.empty()) { + auto [product, toHandleCountOnInsert] = m_productsToSchedule.front(); + m_productsToSchedule.pop(); + + qCDebug(lcLoaderScheduling) << "potentially scheduling product" + << product.product->displayName() + << "unhandled product count on queue insertion" + << toHandleCountOnInsert << "current unhandled product count" + << topLevelProject.productsToHandleCount(); + + // If the number of unfinished products has shrunk since the last time we tried handling + // this product, there has been forward progress and we can allow a deferral. + const Deferral deferral = toHandleCountOnInsert == -1 + || toHandleCountOnInsert > topLevelProject.productsToHandleCount() + ? Deferral::Allowed : Deferral::NotAllowed; + + if (!tryToReserveLoaderState(product, deferral)) + continue; + + startJob(product, deferral); + } + + // There are jobs running, so forward progress is still possible. + if (!m_runningThreads.empty()) + return; + + QBS_CHECK(m_productsToSchedule.empty()); + + // If we end up here, nothing was scheduled in the loop above, which means that either ... + // a) ... we are done or + // b) ... we finally need to schedule our bulk dependencies or + // c) ... we need to schedule products waiting for an unhandled dependency. + // In the latter case, the project has at least one dependency cycle, and the + // DependencyResolver will emit an error. + + // a) + if (m_waitingForBulkDependency.empty() && m_waitingForSingleDependency.empty()) + return; + + // b) + for (const ProductWithLoaderState &product : m_waitingForBulkDependency) + queueProductForScheduling(product, Deferral::NotAllowed); + if (!m_productsToSchedule.empty()) { + m_waitingForBulkDependency.clear(); + scheduleNext(); + return; + } + + // c) + for (const auto &e : m_waitingForSingleDependency) { + for (const ProductWithLoaderState &p : e.second) + queueProductForScheduling(p, Deferral::NotAllowed); + } + QBS_CHECK(!m_productsToSchedule.empty()); + scheduleNext(); +} + +bool ProductsResolver::tryToReserveLoaderState(ProductWithLoaderState &product, Deferral deferral) +{ + QBS_CHECK(!m_availableLoaderStates.empty()); + if (!product.loaderState) { + product.loaderState = m_availableLoaderStates.back(); + m_availableLoaderStates.pop_back(); + return true; + } + if (const auto it = std::find(m_availableLoaderStates.begin(), m_availableLoaderStates.end(), + product.loaderState); it != m_availableLoaderStates.end()) { + m_availableLoaderStates.erase(it); + return true; + } + qCDebug(lcLoaderScheduling) << "loader state" << product.loaderState << " for product" + << product.product->displayName() + << "not available, adding product to wait queue"; + m_waitingForLoaderState[product.loaderState].push({product.product, deferral}); + return false; +} + +std::optional<std::pair<ProductContext *, Deferral>> +ProductsResolver::unblockProductWaitingForLoaderState(LoaderState &loaderState) +{ + auto &waitingForLoaderState = m_waitingForLoaderState[&loaderState]; + if (waitingForLoaderState.empty()) + return {}; + const auto product = waitingForLoaderState.front(); + waitingForLoaderState.pop(); + qCDebug(lcLoaderScheduling) << "loader state" << &loaderState << "now available for product" + << product.first->displayName(); + return product; +} + +void ProductsResolver::startJob(const ProductWithLoaderState &product, Deferral deferral) +{ + QBS_CHECK(product.loaderState); + qCDebug(lcLoaderScheduling) << "scheduling product" << product.product->displayName() + << "with loader state" << product.loaderState + << "and deferral mode" << int(deferral); + try { + const auto it = m_runningThreads.emplace(product.product, ThreadInfo(std::async(m_asyncMode, + [this, product, deferral] { + product.loaderState->itemReader().setExtraSearchPathsStack( + product.product->project->searchPathsStack); + resolveProduct(*product.product, deferral, *product.loaderState); + + // The search paths stack can change during dependency resolution + // (due to module providers); check that we've rolled back all the changes + QBS_CHECK(product.loaderState->itemReader().extraSearchPathsStack() + == product.product->project->searchPathsStack); + + std::lock_guard cancelingLock(m_cancelingMutex); + if (m_canceling) + return; + ThreadsLocker threadsLock(m_asyncMode, m_threadsMutex); + if (const auto it = m_runningThreads.find(product.product); + it != m_runningThreads.end()) { + it->second.done = true; + qCDebug(lcLoaderScheduling) << "thread for product" + << product.product->displayName() + << "finished, waking up scheduler"; + m_threadsNotifier.notify_one(); + } + }), *product.loaderState)); + + // With just one worker thread, the notify/wait overhead would be excessive, so + // we run the task synchronously. + if (m_asyncMode == std::launch::deferred) { + qCDebug(lcLoaderScheduling) << "blocking on product thread immediately"; + it.first->second.future.wait(); + } + } catch (const std::system_error &e) { + if (e.code() != std::errc::resource_unavailable_try_again) + throw e; + qCWarning(lcLoaderScheduling) << "failed to create thread"; + if (m_maxJobCount >= 2) { + m_maxJobCount /= 2; + qCWarning(lcLoaderScheduling) << "throttling down to" << m_maxJobCount << "jobs"; + } + queueProductForScheduling(product, deferral); + m_availableLoaderStates.push_back(product.loaderState); + } +} + +void ProductsResolver::checkForCancelation() +{ + if (m_loaderState.topLevelProject().isCanceled()) { + m_cancelingMutex.lock(); + m_canceling = true; + m_cancelingMutex.unlock(); + for (auto &thread : m_runningThreads) + thread.second.future.wait(); + throw CancelException(); + } +} + +void ProductsResolver::handleFinishedThreads() +{ + TopLevelProjectContext &topLevelProject = m_loaderState.topLevelProject(); + AccumulatingTimer timer(m_loaderState.parameters().logElapsedTime() + ? &topLevelProject.timingData().schedulingProducts : nullptr); + + std::vector<std::pair<ProductWithLoaderState, Deferral>> productsToScheduleDirectly; + for (auto it = m_runningThreads.begin(); it != m_runningThreads.end();) { + ThreadInfo &ti = it->second; + if (!ti.done) { + ++it; + continue; + } + ti.future.wait(); + ProductContext &product = *it->first; + LoaderState &loaderState = ti.loaderState; + it = m_runningThreads.erase(it); + + qCDebug(lcLoaderScheduling) << "handling finished thread for product" + << product.displayName() + << "current unhandled product count is" + << topLevelProject.productsToHandleCount(); + + // If there are products waiting for the loader state used in the finished thread, + // we can start a job for one of them right away (but not in the loop, + // because startJob() modifies the thread list we are currently iterating over). + if (const auto productInfo = unblockProductWaitingForLoaderState(loaderState)) { + productsToScheduleDirectly.emplace_back( + ProductWithLoaderState(*productInfo->first, &loaderState), + productInfo->second); + } else { + qCDebug(lcLoaderScheduling) << "making loader state" << &loaderState + << "available again"; + m_availableLoaderStates.push_back(&loaderState); + } + + // If we encountered a dependency to an in-progress product or to a bulk dependency, + // we defer handling this product. + if (product.dependenciesResolvingPending()) { + qCDebug(lcLoaderScheduling) << "dependencies resolving not finished for product" + << product.displayName(); + const auto pending = product.pendingDependency(); + switch (pending.first) { + case ProductDependency::Single: + waitForSingleDependency(ProductWithLoaderState(product, &loaderState), + *pending.second); + break; + case ProductDependency::Bulk: + waitForBulkDependency(ProductWithLoaderState(product, &loaderState)); + break; + case ProductDependency::None: + // This can happen if the dependency has finished in between the check in + // DependencyResolver and the one here. + QBS_CHECK(pending.second); + queueProductForScheduling(ProductWithLoaderState(product, &loaderState), + Deferral::Allowed); + break; + } + topLevelProject.incProductDeferrals(); + } else { + qCDebug(lcLoaderScheduling) << "product" << product.displayName() << "finished"; + topLevelProject.removeProductToHandle(product); + if (!product.name.startsWith(StringConstants::shadowProductPrefix())) + m_finishedProducts.push_back(&product); + topLevelProject.timingData() += product.timingData; + checkForMissedBulkDependencies(product); + topLevelProject.registerBulkDependencies(product); + unblockProductsWaitingForDependency(product); + } + } + + for (const auto &productInfo : productsToScheduleDirectly) + startJob(productInfo.first, productInfo.second); +} + +void ProductsResolver::queueProductForScheduling(const ProductWithLoaderState &product, + Deferral deferral) +{ + qCDebug(lcLoaderScheduling) << "queueing product" << product.product->displayName() + << "with deferral mode" << int(deferral); + m_productsToSchedule.emplace(product, deferral == Deferral::Allowed + ? -1 : m_loaderState.topLevelProject().productsToHandleCount()); +} + +void ProductsResolver::waitForSingleDependency(const ProductWithLoaderState &product, + ProductContext &dependency) +{ + qCDebug(lcLoaderScheduling) << "product" << product.product->displayName() + << "now waiting for single dependency" + << dependency.displayName(); + m_waitingForSingleDependency[&dependency].push_back(product); +} + +void ProductsResolver::waitForBulkDependency(const ProductWithLoaderState &product) +{ + qCDebug(lcLoaderScheduling) << "product" << product.product->displayName() + << "now waiting for bulk dependency"; + m_waitingForBulkDependency.push_back(product); +} + +void ProductsResolver::unblockProductsWaitingForDependency(ProductContext &finishedProduct) +{ + const auto it = m_waitingForSingleDependency.find(&finishedProduct); + if (it == m_waitingForSingleDependency.end()) + return; + + qCDebug(lcLoaderScheduling) << "unblocking all products waiting for now-finished product" << + finishedProduct.displayName(); + for (const ProductWithLoaderState &p : it->second) { + qCDebug(lcLoaderScheduling) << " unblocking product" << p.product->displayName(); + queueProductForScheduling(p, Deferral::Allowed); + } + m_waitingForSingleDependency.erase(it); +} + +void ProductsResolver::postProcess() +{ + for (ProductContext * const product : m_finishedProducts) { + if (product->product) + product->product->project->products.push_back(product->product); + + // This has to be done in post-processing, because we need both product and shadow product + // to be ready, and contrary to what one might assume, there is no proper ordering + // between them regarding dependency resolving. + setupExports(*product, m_loaderState); + } + + for (const auto &engine : m_enginePool) + m_loaderState.topLevelProject().collectDataFromEngine(*engine); + + QBS_CHECK(!m_loaderState.topLevelProject().projects().empty()); + const auto project = std::dynamic_pointer_cast<TopLevelProject>( + m_loaderState.topLevelProject().projects().front()->project); + QBS_CHECK(project); + for (LoaderState * const loaderState : m_availableLoaderStates) + project->warningsEncountered << loaderState->logger().warnings(); +} + +void ProductsResolver::checkForMissedBulkDependencies(const ProductContext &product) +{ + if (!product.product || !product.product->enabled || !product.bulkDependencies.empty()) + return; + for (const FileTag &tag : product.product->fileTags) { + for (const auto &[p, location] + : m_loaderState.topLevelProject().finishedProductsWithBulkDependency(tag)) { + if (!p->product->enabled) + continue; + if (p->name == product.name) + continue; + ErrorInfo e; + e.append(Tr::tr("Cyclic dependencies detected:")); + e.append(Tr::tr("First product is '%1'.") + .arg(product.displayName()), product.item->location()); + e.append(Tr::tr("Second product is '%1'.") + .arg(p->displayName()), p->item->location()); + e.append(Tr::tr("Dependency from %1 to %2 was established via Depends.productTypes.") + .arg(p->displayName(), product.displayName()), location); + if (m_loaderState.parameters().productErrorMode() == ErrorHandlingMode::Strict) + throw e; + m_loaderState.logger().printWarning(e); + m_loaderState.logger().printWarning( + ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") + .arg(product.displayName()), product.item->location())); + m_loaderState.logger().printWarning( + ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") + .arg(p->displayName()), p->item->location())); + product.product->enabled = false; + p->product->enabled = false; + } + } +} + +int ProductsResolver::dependsItemCount(const Item *item) +{ + int count = 0; + for (const Item * const child : item->children()) { + if (child->type() == ItemType::Depends) + ++count; + } + return count; +} + +int ProductsResolver::dependsItemCount(ProductContext &product) +{ + if (product.dependsItemCount == -1) + product.dependsItemCount = dependsItemCount(product.item); + return product.dependsItemCount; +} + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/productsresolver.h b/src/lib/corelib/loader/productsresolver.h new file mode 100644 index 000000000..b8b4b7516 --- /dev/null +++ b/src/lib/corelib/loader/productsresolver.h @@ -0,0 +1,47 @@ +/**************************************************************************** +** +** Copyright (C) 2023 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of Qbs. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#pragma once + +namespace qbs::Internal { +class LoaderState; + +void resolveProducts(LoaderState &loaderState); + +} // namespace qbs::Internal diff --git a/src/lib/corelib/loader/projectresolver.cpp b/src/lib/corelib/loader/projectresolver.cpp index 17684fc94..d928048fb 100644 --- a/src/lib/corelib/loader/projectresolver.cpp +++ b/src/lib/corelib/loader/projectresolver.cpp @@ -39,15 +39,10 @@ #include "projectresolver.h" -#include "dependenciesresolver.h" #include "itemreader.h" #include "loaderutils.h" -#include "localprofiles.h" -#include "moduleinstantiator.h" -#include "modulepropertymerger.h" -#include "probesresolver.h" #include "productscollector.h" -#include "productshandler.h" +#include "productsresolver.h" #include <jsextensions/jsextensions.h> #include <jsextensions/moduleproperties.h> @@ -95,14 +90,6 @@ namespace qbs { namespace Internal { -extern bool debugProperties; - -static const FileTag unknownFileTag() -{ - static const FileTag tag("unknown-file-tag"); - return tag; -} - static SetupProjectParameters finalizedProjectParameters(const SetupProjectParameters &in, Logger &logger) { @@ -126,36 +113,6 @@ static SetupProjectParameters finalizedProjectParameters(const SetupProjectParam return params; } -struct ResolverProjectContext -{ - ResolverProjectContext *parentContext = nullptr; - ResolvedProjectPtr project; - std::vector<FileTaggerConstPtr> fileTaggers; - std::vector<RulePtr> rules; - JobLimits jobLimits; - ResolvedModulePtr dummyModule; -}; - -using FileLocations = QHash<std::pair<QString, QString>, CodeLocation>; -struct ResolverProductContext -{ - ResolvedProductPtr product; - QString buildDirectory; - Item *item = nullptr; - using ArtifactPropertiesInfo = std::pair<ArtifactPropertiesPtr, std::vector<CodeLocation>>; - QHash<QStringList, ArtifactPropertiesInfo> artifactPropertiesPerFilter; - FileLocations sourceArtifactLocations; - GroupConstPtr currentGroup; -}; - -struct ModuleContext -{ - ResolvedModulePtr module; - JobLimits jobLimits; -}; - -class CancelException { }; - class ProjectResolver::Private { public: @@ -163,124 +120,24 @@ public: : setupParams(finalizedProjectParameters(parameters, logger)), engine(engine), logger(std::move(logger)) {} - static void applyFileTaggers(const SourceArtifactPtr &artifact, - const ResolvedProductConstPtr &product); - static SourceArtifactPtr createSourceArtifact( - const ResolvedProductPtr &rproduct, const QString &fileName, const GroupPtr &group, - bool wildcard, const CodeLocation &filesLocation = CodeLocation(), - FileLocations *fileLocations = nullptr, ErrorInfo *errorInfo = nullptr); - void checkCancelation() const; - QString verbatimValue(const ValueConstPtr &value, bool *propertyWasSet = nullptr) const; - QString verbatimValue(Item *item, const QString &name, bool *propertyWasSet = nullptr) const; - ScriptFunctionPtr scriptFunctionValue(Item *item, const QString &name) const; - ResolvedFileContextPtr resolvedFileContext(const FileContextConstPtr &ctx) const; - void ignoreItem(Item *item, ResolverProjectContext *projectContext); TopLevelProjectPtr resolveTopLevelProject(); - void resolveProject(Item *item, ResolverProjectContext *projectContext); - void resolveProjectFully(Item *item, ResolverProjectContext *projectContext); - void resolveSubProject(Item *item, ResolverProjectContext *projectContext); - void resolveProduct(Item *item, ResolverProjectContext *projectContext); - void resolveProductFully(Item *item, ResolverProjectContext *projectContext); - void resolveModules(const Item *item, ResolverProjectContext *projectContext); - void resolveModule(const QualifiedId &moduleName, Item *item, bool isProduct, - const QVariantMap ¶meters, JobLimits &jobLimits, - ResolverProjectContext *projectContext); - void gatherProductTypes(ResolvedProduct *product, Item *item); - QVariantMap resolveAdditionalModuleProperties(const Item *group, - const QVariantMap ¤tValues); - void resolveGroup(Item *item, ResolverProjectContext *projectContext); - void resolveGroupFully(Item *item, ResolverProjectContext *projectContext, bool isEnabled); - void resolveShadowProduct(Item *item, ResolverProjectContext *); - void resolveExport(Item *exportItem, ResolverProjectContext *); - std::unique_ptr<ExportedItem> resolveExportChild(const Item *item, - const ExportedModule &module); - void resolveRule(Item *item, ResolverProjectContext *projectContext); - void resolveRuleArtifact(const RulePtr &rule, Item *item); - void resolveRuleArtifactBinding(const RuleArtifactPtr &ruleArtifact, Item *item, - const QStringList &namePrefix, - QualifiedIdSet *seenBindings); - void resolveFileTagger(Item *item, ResolverProjectContext *projectContext); - void resolveJobLimit(Item *item, ResolverProjectContext *projectContext); - void resolveScanner(Item *item, ResolverProjectContext *projectContext); - void resolveProductDependencies(); - void postProcess(const ResolvedProductPtr &product, ResolverProjectContext *projectContext) const; - void applyFileTaggers(const ResolvedProductPtr &product) const; - QVariantMap evaluateModuleValues(Item *item, bool lookupPrototype = true); - QVariantMap evaluateProperties(Item *item, bool lookupPrototype, bool checkErrors); - QVariantMap evaluateProperties(const Item *item, const Item *propertiesContainer, - const QVariantMap &tmplt, bool lookupPrototype, - bool checkErrors); - void evaluateProperty(const Item *item, const QString &propName, const ValuePtr &propValue, - QVariantMap &result, bool checkErrors); - void checkAllowedValues( - const QVariant &v, const CodeLocation &loc, const PropertyDeclaration &decl, - const QString &key) const; - void createProductConfig(ResolvedProduct *product); - ResolverProjectContext createProjectContext(ResolverProjectContext *parentProjectContext) const; - void adaptExportedPropertyValues(const Item *shadowProductItem); - void collectExportedProductDependencies(); + void resolveProject(ProjectContext *projectContext); + void resolveProjectFully(ProjectContext *projectContext); + void resolveSubProject(Item *item, ProjectContext *projectContext); void checkOverriddenValues(); void collectNameFromOverride(const QString &overrideString); void loadTopLevelProjectItem(); void buildProjectTree(); - struct ProductDependencyInfo - { - ProductDependencyInfo(ResolvedProductPtr product, - QVariantMap parameters = QVariantMap()) - : product(std::move(product)), parameters(std::move(parameters)) - { - } - - ResolvedProductPtr product; - QVariantMap parameters; - }; - - QString sourceCodeAsFunction(const JSSourceValueConstPtr &value, - const PropertyDeclaration &decl) const; - QString sourceCodeForEvaluation(const JSSourceValueConstPtr &value) const; - static void matchArtifactProperties(const ResolvedProductPtr &product, - const std::vector<SourceArtifactPtr> &artifacts); void printProfilingInfo(); - void collectPropertiesForExportItem(Item *productModuleInstance); - void collectPropertiesForModuleInExportItem(const Item::Module &module); - - void collectPropertiesForExportItem(const QualifiedId &moduleName, - const ValuePtr &value, Item *moduleInstance, QVariantMap &moduleProps); - void setupExportedProperties(const Item *item, const QString &namePrefix, - std::vector<ExportedProperty> &properties); - - using ItemFuncPtr = void (ProjectResolver::Private::*)(Item *item, - ResolverProjectContext *projectContext); - using ItemFuncMap = QMap<ItemType, ItemFuncPtr>; - void callItemFunction(const ItemFuncMap &mappings, Item *item, ResolverProjectContext *projectContext); - const SetupProjectParameters setupParams; ScriptEngine * const engine; mutable Logger logger; - Evaluator evaluator{engine}; ItemPool itemPool; - LoaderState state{setupParams, itemPool, evaluator, logger}; - ProductsCollector productsCollector{state}; - ProductsHandler productsHandler{state}; + TopLevelProjectContext topLevelProject; + LoaderState state{setupParams, topLevelProject, itemPool, *engine, logger}; Item *rootProjectItem = nullptr; - ProgressObserver *progressObserver = nullptr; - ResolverProductContext *productContext = nullptr; - ModuleContext *moduleContext = nullptr; - QHash<Item *, ResolvedProductPtr> productsByItem; - mutable QHash<FileContextConstPtr, ResolvedFileContextPtr> fileContextMap; - mutable QHash<CodeLocation, ScriptFunctionPtr> scriptFunctionMap; - mutable QHash<std::pair<QStringView, QStringList>, QString> scriptFunctions; - mutable QHash<QStringView, QString> sourceCode; - Set<CodeLocation> groupLocationWarnings; - std::vector<std::pair<ResolvedProductPtr, Item *>> productExportInfo; - std::vector<ErrorInfo> queuedErrors; - FileTime lastResolveTime; - qint64 elapsedTimeModPropEval = 0; - qint64 elapsedTimeAllPropEval = 0; - qint64 elapsedTimeGroups = 0; - qint64 elapsedTimePropertyChecking = 0; }; ProjectResolver::ProjectResolver(const SetupProjectParameters ¶meters, ScriptEngine *engine, @@ -294,34 +151,33 @@ ProjectResolver::~ProjectResolver() = default; void ProjectResolver::setProgressObserver(ProgressObserver *observer) { - d->progressObserver = observer; - d->state.topLevelProject().progressObserver = observer; + d->state.topLevelProject().setProgressObserver(observer); } void ProjectResolver::setOldProjectProbes(const std::vector<ProbeConstPtr> &oldProbes) { - d->state.probesResolver().setOldProjectProbes(oldProbes); + d->state.topLevelProject().setOldProjectProbes(oldProbes); } void ProjectResolver::setOldProductProbes( const QHash<QString, std::vector<ProbeConstPtr>> &oldProbes) { - d->state.probesResolver().setOldProductProbes(oldProbes); + d->state.topLevelProject().setOldProductProbes(oldProbes); } void ProjectResolver::setLastResolveTime(const FileTime &time) { - d->lastResolveTime = time; + d->state.topLevelProject().setLastResolveTime(time); } void ProjectResolver::setStoredProfiles(const QVariantMap &profiles) { - d->state.topLevelProject().profileConfigs = profiles; + d->state.topLevelProject().setProfileConfigs(profiles); } void ProjectResolver::setStoredModuleProviderInfo(const StoredModuleProviderInfo &providerInfo) { - d->state.dependenciesResolver().setStoredModuleProviderInfo(providerInfo); + d->state.topLevelProject().setModuleProvidersCache(providerInfo.providers); } static void checkForDuplicateProductNames(const TopLevelProjectConstPtr &project) @@ -359,10 +215,10 @@ TopLevelProjectPtr ProjectResolver::resolve() // the project yet. That's why we use a placeholder here, so the user at least // sees that an operation is starting. The real total effort will be set later when // we have enough information. - if (d->progressObserver) { - d->progressObserver->initialize(Tr::tr("Resolving project for configuration %1") + if (ProgressObserver * const observer = d->state.topLevelProject().progressObserver()) { + observer->initialize(Tr::tr("Resolving project for configuration %1") .arg(TopLevelProject::deriveId(d->setupParams.finalBuildConfigurationTree())), 1); - d->progressObserver->setScriptEngine(d->engine); + observer->addScriptEngine(d->engine); } const FileTime resolveTime = FileTime::currentTime(); @@ -381,48 +237,13 @@ TopLevelProjectPtr ProjectResolver::resolve() tlp->lastEndResolveTime = FileTime::currentTime(); // E.g. if the top-level project is disabled. - if (d->progressObserver) { - d->progressObserver->setFinished(); + if (ProgressObserver * const observer = d->state.topLevelProject().progressObserver()) { + observer->setFinished(); d->printProfilingInfo(); } return tlp; } -void ProjectResolver::Private::checkCancelation() const -{ - if (progressObserver && progressObserver->canceled()) - throw CancelException(); -} - -QString ProjectResolver::Private::verbatimValue( - const ValueConstPtr &value, bool *propertyWasSet) const -{ - QString result; - if (value && value->type() == Value::JSSourceValueType) { - const JSSourceValueConstPtr sourceValue = std::static_pointer_cast<const JSSourceValue>( - value); - result = sourceCodeForEvaluation(sourceValue); - if (propertyWasSet) - *propertyWasSet = !sourceValue->isBuiltinDefaultValue(); - } else { - if (propertyWasSet) - *propertyWasSet = false; - } - return result; -} - -QString ProjectResolver::Private::verbatimValue(Item *item, const QString &name, - bool *propertyWasSet) const -{ - return verbatimValue(item->property(name), propertyWasSet); -} - -void ProjectResolver::Private::ignoreItem(Item *item, ResolverProjectContext *projectContext) -{ - Q_UNUSED(item); - Q_UNUSED(projectContext); -} - static void makeSubProjectNamesUniqe(const ResolvedProjectPtr &parentProject) { Set<QString> subProjectNames; @@ -448,70 +269,56 @@ static void makeSubProjectNamesUniqe(const ResolvedProjectPtr &parentProject) TopLevelProjectPtr ProjectResolver::Private::resolveTopLevelProject() { - if (progressObserver) - progressObserver->setMaximum(int(state.topLevelProject().productInfos.size())); + if (ProgressObserver * const observer = state.topLevelProject().progressObserver()) + observer->setMaximum(state.topLevelProject().productCount()); TopLevelProjectPtr project = TopLevelProject::create(); project->buildDirectory = TopLevelProject::deriveBuildDirectory( setupParams.buildRoot(), TopLevelProject::deriveId(setupParams.finalBuildConfigurationTree())); - project->buildSystemFiles = state.itemReader().filesRead() - - state.dependenciesResolver().tempQbsFiles(); - project->profileConfigs = state.topLevelProject().profileConfigs; - const QVariantMap &profiles = state.localProfiles().profiles(); - for (auto it = profiles.begin(); it != profiles.end(); ++it) - project->profileConfigs.remove(it.key()); - - project->probes = state.topLevelProject().probes; - project->moduleProviderInfo = state.dependenciesResolver().storedModuleProviderInfo(); - ResolverProjectContext projectContext; - projectContext.project = project; - - resolveProject(rootProjectItem, &projectContext); + if (!state.topLevelProject().projects().empty()) { + ProjectContext * const projectContext = state.topLevelProject().projects().front(); + QBS_CHECK(projectContext->item == rootProjectItem); + projectContext->project = project; + resolveProject(projectContext); + } + resolveProducts(state); ErrorInfo accumulatedErrors; - for (const ErrorInfo &e : queuedErrors) + for (const ErrorInfo &e : state.topLevelProject().queuedErrors()) appendError(accumulatedErrors, e); if (accumulatedErrors.hasError()) throw accumulatedErrors; + project->buildSystemFiles.unite(state.topLevelProject().buildSystemFiles()); + project->profileConfigs = state.topLevelProject().profileConfigs(); + const QVariantMap &profiles = state.topLevelProject().localProfiles(); + for (auto it = profiles.begin(); it != profiles.end(); ++it) + project->profileConfigs.remove(it.key()); + project->probes = state.topLevelProject().projectLevelProbes(); + project->moduleProviderInfo.providers = state.topLevelProject().moduleProvidersCache(); project->setBuildConfiguration(setupParams.finalBuildConfigurationTree()); project->overriddenValues = setupParams.overriddenValues(); - project->canonicalFilePathResults = engine->canonicalFilePathResults(); - project->fileExistsResults = engine->fileExistsResults(); - project->directoryEntriesResults = engine->directoryEntriesResults(); - project->fileLastModifiedResults = engine->fileLastModifiedResults(); - project->environment = engine->environment(); - project->buildSystemFiles.unite(engine->imports()); + state.topLevelProject().collectDataFromEngine(*engine); makeSubProjectNamesUniqe(project); - resolveProductDependencies(); - collectExportedProductDependencies(); checkForDuplicateProductNames(project); + project->warningsEncountered << logger.warnings(); - for (const ResolvedProductPtr &product : project->allProducts()) { - if (!product->enabled) - continue; - - applyFileTaggers(product); - matchArtifactProperties(product, product->allEnabledFiles()); - - // Let a positive value of qbs.install imply the file tag "installable". - for (const SourceArtifactPtr &artifact : product->allFiles()) { - if (artifact->properties->qbsPropertyValue(StringConstants::installProperty()).toBool()) - artifact->fileTags += "installable"; - } - } - project->warningsEncountered = logger.warnings(); return project; } -void ProjectResolver::Private::resolveProject(Item *item, ResolverProjectContext *projectContext) +void ProjectResolver::Private::resolveProject(ProjectContext *projectContext) { - checkCancelation(); + state.topLevelProject().checkCancelation(); + + if (projectContext->parent) { + projectContext->project = ResolvedProject::create(); + projectContext->parent->project->subProjects.push_back(projectContext->project); + projectContext->project->parentProject = projectContext->parent->project; + projectContext->project->enabled = projectContext->parent->project->enabled; + } + projectContext->project->location = projectContext->item->location(); - if (projectContext->parentContext) - projectContext->project->enabled = projectContext->parentContext->project->enabled; - projectContext->project->location = item->location(); try { - resolveProjectFully(item, projectContext); + resolveProjectFully(projectContext); } catch (const ErrorInfo &error) { if (!projectContext->project->enabled) { qCDebug(lcProjectResolver) << "error resolving project" @@ -524,17 +331,20 @@ void ProjectResolver::Private::resolveProject(Item *item, ResolverProjectContext } } -void ProjectResolver::Private::resolveProjectFully(Item *item, ResolverProjectContext *projectContext) +void ProjectResolver::Private::resolveProjectFully(ProjectContext *projectContext) { + Item * const item = projectContext->item; projectContext->project->enabled = projectContext->project->enabled - && evaluator.boolValue(item, StringConstants::conditionProperty()); - projectContext->project->name = evaluator.stringValue(item, StringConstants::nameProperty()); + && state.evaluator().boolValue(item, StringConstants::conditionProperty()); + projectContext->project->name = state.evaluator().stringValue( + item, StringConstants::nameProperty()); if (projectContext->project->name.isEmpty()) projectContext->project->name = FileInfo::baseName(item->location().filePath()); // FIXME: Must also be changed in item? QVariantMap projectProperties; if (!projectContext->project->enabled) { projectProperties.insert(StringConstants::profileProperty(), - evaluator.stringValue(item, StringConstants::profileProperty())); + state.evaluator().stringValue( + item, StringConstants::profileProperty())); projectContext->project->setProjectProperties(projectProperties); return; } @@ -548,635 +358,54 @@ void ProjectResolver::Private::resolveProjectFully(Item *item, ResolverProjectCo continue; const ValueConstPtr v = item->property(it.key()); QBS_ASSERT(v && v->type() != Value::ItemValueType, continue); - const ScopedJsValue sv(engine->context(), evaluator.value(item, it.key())); + const ScopedJsValue sv(engine->context(), state.evaluator().value(item, it.key())); projectProperties.insert(it.key(), getJsVariant(engine->context(), sv)); } projectContext->project->setProjectProperties(projectProperties); - static const ItemFuncMap mapping = { - { ItemType::Project, &ProjectResolver::Private::resolveProject }, - { ItemType::SubProject, &ProjectResolver::Private::resolveSubProject }, - { ItemType::Product, &ProjectResolver::Private::resolveProduct }, - { ItemType::Probe, &ProjectResolver::Private::ignoreItem }, - { ItemType::FileTagger, &ProjectResolver::Private::resolveFileTagger }, - { ItemType::JobLimit, &ProjectResolver::Private::resolveJobLimit }, - { ItemType::Rule, &ProjectResolver::Private::resolveRule }, - { ItemType::PropertyOptions, &ProjectResolver::Private::ignoreItem } - }; - for (Item * const child : item->children()) { + state.topLevelProject().checkCancelation(); try { - callItemFunction(mapping, child, projectContext); - } catch (const ErrorInfo &e) { - queuedErrors.push_back(e); - } - } - - for (const ResolvedProductPtr &product : projectContext->project->products) - postProcess(product, projectContext); -} - -void ProjectResolver::Private::resolveSubProject(Item *item, ResolverProjectContext *projectContext) -{ - ResolverProjectContext subProjectContext = createProjectContext(projectContext); - - Item * const projectItem = item->child(ItemType::Project); - if (projectItem) { - resolveProject(projectItem, &subProjectContext); - return; - } - - // No project item was found, which means the project was disabled. - subProjectContext.project->enabled = false; - Item * const propertiesItem = item->child(ItemType::PropertiesInSubProject); - if (propertiesItem) { - subProjectContext.project->name = evaluator.stringValue( - propertiesItem, StringConstants::nameProperty()); - } -} - -void ProjectResolver::Private::resolveProduct(Item *item, ResolverProjectContext *projectContext) -{ - checkCancelation(); - evaluator.clearPropertyDependencies(); - ResolverProductContext productContext; - productContext.item = item; - ResolvedProductPtr product = ResolvedProduct::create(); - product->enabled = projectContext->project->enabled; - product->moduleProperties = PropertyMapInternal::create(); - product->project = projectContext->project; - productContext.product = product; - product->location = item->location(); - class ProductContextSwitcher { - public: - ProductContextSwitcher(ProjectResolver::Private &resolver, ResolverProductContext &newContext, - ProgressObserver *progressObserver) - : m_resolver(resolver), m_progressObserver(progressObserver) { - QBS_CHECK(!m_resolver.productContext); - m_resolver.productContext = &newContext; - } - ~ProductContextSwitcher() { - if (m_progressObserver) - m_progressObserver->incrementProgressValue(); - m_resolver.productContext = nullptr; - } - private: - ProjectResolver::Private &m_resolver; - ProgressObserver * const m_progressObserver; - } contextSwitcher(*this, productContext, progressObserver); - const auto errorFromDelayedError = [&] { - ProductInfo &pi = state.topLevelProject().productInfos[item]; - if (pi.delayedError.hasError()) { - ErrorInfo errorInfo; - - // First item is "main error", gets prepended again in the catch clause. - const QList<ErrorItem> &items = pi.delayedError.items(); - for (int i = 1; i < items.size(); ++i) - errorInfo.append(items.at(i)); - - pi.delayedError.clear(); - return errorInfo; - } - return ErrorInfo(); - }; - - // Even if we previously encountered an error, try to continue for as long as possible - // to provide IDEs with useful data (e.g. the list of files). - // If we encounter a follow-up error, suppress it and report the original one instead. - try { - resolveProductFully(item, projectContext); - if (const ErrorInfo error = errorFromDelayedError(); error.hasError()) - throw error; - } catch (ErrorInfo e) { - if (const ErrorInfo error = errorFromDelayedError(); error.hasError()) - e = error; - QString mainErrorString = !product->name.isEmpty() - ? Tr::tr("Error while handling product '%1':").arg(product->name) - : Tr::tr("Error while handling product:"); - ErrorInfo fullError(mainErrorString, item->location()); - appendError(fullError, e); - if (!product->enabled) { - qCDebug(lcProjectResolver) << fullError.toString(); - return; - } - if (setupParams.productErrorMode() == ErrorHandlingMode::Strict) - throw fullError; - logger.printWarning(fullError); - logger.printWarning(ErrorInfo(Tr::tr("Product '%1' had errors and was disabled.") - .arg(product->name), item->location())); - product->enabled = false; - } -} - -void ProjectResolver::Private::resolveProductFully(Item *item, ResolverProjectContext *projectContext) -{ - const ResolvedProductPtr product = productContext->product; - projectContext->project->products.push_back(product); - product->name = evaluator.stringValue(item, StringConstants::nameProperty()); - - // product->buildDirectory() isn't valid yet, because the productProperties map is not ready. - productContext->buildDirectory - = evaluator.stringValue(item, StringConstants::buildDirectoryProperty()); - product->multiplexConfigurationId - = evaluator.stringValue(item, StringConstants::multiplexConfigurationIdProperty()); - qCDebug(lcProjectResolver) << "resolveProduct" << product->uniqueName(); - productsByItem.insert(item, product); - product->enabled = product->enabled - && evaluator.boolValue(item, StringConstants::conditionProperty()); - ProductInfo &pi = state.topLevelProject().productInfos[item]; - gatherProductTypes(product.get(), item); - product->targetName = evaluator.stringValue(item, StringConstants::targetNameProperty()); - product->sourceDirectory = evaluator.stringValue( - item, StringConstants::sourceDirectoryProperty()); - product->destinationDirectory = evaluator.stringValue( - item, StringConstants::destinationDirProperty()); - - if (product->destinationDirectory.isEmpty()) { - product->destinationDirectory = productContext->buildDirectory; - } else { - product->destinationDirectory = FileInfo::resolvePath( - product->topLevelProject()->buildDirectory, - product->destinationDirectory); - } - product->probes = pi.probes; - createProductConfig(product.get()); - product->productProperties.insert(StringConstants::destinationDirProperty(), - product->destinationDirectory); - ModuleProperties::init(evaluator.engine(), evaluator.scriptValue(item), product.get()); - - QList<Item *> subItems = item->children(); - const ValuePtr filesProperty = item->property(StringConstants::filesProperty()); - if (filesProperty) { - Item *fakeGroup = Item::create(item->pool(), ItemType::Group); - fakeGroup->setFile(item->file()); - fakeGroup->setLocation(item->location()); - fakeGroup->setScope(item); - fakeGroup->setProperty(StringConstants::nameProperty(), VariantValue::create(product->name)); - fakeGroup->setProperty(StringConstants::filesProperty(), filesProperty); - fakeGroup->setProperty(StringConstants::excludeFilesProperty(), - item->property(StringConstants::excludeFilesProperty())); - fakeGroup->setProperty(StringConstants::overrideTagsProperty(), - VariantValue::falseValue()); - fakeGroup->setupForBuiltinType(setupParams.deprecationWarningMode(), logger); - subItems.prepend(fakeGroup); - } - - static const ItemFuncMap mapping = { - { ItemType::Depends, &ProjectResolver::Private::ignoreItem }, - { ItemType::Rule, &ProjectResolver::Private::resolveRule }, - { ItemType::FileTagger, &ProjectResolver::Private::resolveFileTagger }, - { ItemType::JobLimit, &ProjectResolver::Private::resolveJobLimit }, - { ItemType::Group, &ProjectResolver::Private::resolveGroup }, - { ItemType::Product, &ProjectResolver::Private::resolveShadowProduct }, - { ItemType::Export, &ProjectResolver::Private::resolveExport }, - { ItemType::Probe, &ProjectResolver::Private::ignoreItem }, - { ItemType::PropertyOptions, &ProjectResolver::Private::ignoreItem } - }; - - for (Item * const child : std::as_const(subItems)) - callItemFunction(mapping, child, projectContext); - - for (const ResolverProjectContext *p = projectContext; p; p = p->parentContext) { - JobLimits tempLimits = p->jobLimits; - product->jobLimits = tempLimits.update(product->jobLimits); - } - - resolveModules(item, projectContext); -} - -void ProjectResolver::Private::resolveModules(const Item *item, ResolverProjectContext *projectContext) -{ - JobLimits jobLimits; - for (const Item::Module &m : item->modules()) { - resolveModule(m.name, m.item, m.productInfo.has_value(), m.parameters, jobLimits, - projectContext); - } - for (int i = 0; i < jobLimits.count(); ++i) { - const JobLimit &moduleJobLimit = jobLimits.jobLimitAt(i); - if (productContext->product->jobLimits.getLimit(moduleJobLimit.pool()) == -1) - productContext->product->jobLimits.setJobLimit(moduleJobLimit); - } -} - -void ProjectResolver::Private::resolveModule( - const QualifiedId &moduleName, Item *item, bool isProduct, const QVariantMap ¶meters, - JobLimits &jobLimits, ResolverProjectContext *projectContext) -{ - checkCancelation(); - if (!item->isPresentModule()) - return; - - ModuleContext * const oldModuleContext = moduleContext; - ModuleContext newModuleContext; - newModuleContext.module = ResolvedModule::create(); - moduleContext = &newModuleContext; - - const ResolvedModulePtr &module = newModuleContext.module; - module->name = moduleName.toString(); - module->isProduct = isProduct; - module->product = productContext->product.get(); - module->setupBuildEnvironmentScript.initialize( - scriptFunctionValue(item, StringConstants::setupBuildEnvironmentProperty())); - module->setupRunEnvironmentScript.initialize( - scriptFunctionValue(item, StringConstants::setupRunEnvironmentProperty())); - - for (const Item::Module &m : item->modules()) { - if (m.item->isPresentModule()) - module->moduleDependencies += m.name.toString(); - } - - productContext->product->modules.push_back(module); - if (!parameters.empty()) - productContext->product->moduleParameters[module] = parameters; - - static const ItemFuncMap mapping { - { ItemType::Group, &ProjectResolver::Private::ignoreItem }, - { ItemType::Rule, &ProjectResolver::Private::resolveRule }, - { ItemType::FileTagger, &ProjectResolver::Private::resolveFileTagger }, - { ItemType::JobLimit, &ProjectResolver::Private::resolveJobLimit }, - { ItemType::Scanner, &ProjectResolver::Private::resolveScanner }, - { ItemType::PropertyOptions, &ProjectResolver::Private::ignoreItem }, - { ItemType::Depends, &ProjectResolver::Private::ignoreItem }, - { ItemType::Parameter, &ProjectResolver::Private::ignoreItem }, - { ItemType::Properties, &ProjectResolver::Private::ignoreItem }, - { ItemType::Probe, &ProjectResolver::Private::ignoreItem } - }; - for (Item *child : item->children()) - callItemFunction(mapping, child, projectContext); - for (int i = 0; i < newModuleContext.jobLimits.count(); ++i) { - const JobLimit &newJobLimit = newModuleContext.jobLimits.jobLimitAt(i); - const int oldLimit = jobLimits.getLimit(newJobLimit.pool()); - if (oldLimit == -1 || oldLimit > newJobLimit.limit()) - jobLimits.setJobLimit(newJobLimit); - } - - moduleContext = oldModuleContext; // FIXME: Exception safety -} - -void ProjectResolver::Private::gatherProductTypes(ResolvedProduct *product, Item *item) -{ - const VariantValuePtr type = item->variantProperty(StringConstants::typeProperty()); - if (type) - product->fileTags = FileTags::fromStringList(type->value().toStringList()); -} - -SourceArtifactPtr ProjectResolver::Private::createSourceArtifact( - const ResolvedProductPtr &rproduct, const QString &fileName, const GroupPtr &group, - bool wildcard, const CodeLocation &filesLocation, FileLocations *fileLocations, - ErrorInfo *errorInfo) -{ - const QString &baseDir = FileInfo::path(group->location.filePath()); - const QString absFilePath = QDir::cleanPath(FileInfo::resolvePath(baseDir, fileName)); - if (!wildcard && !FileInfo(absFilePath).exists()) { - if (errorInfo) - errorInfo->append(Tr::tr("File '%1' does not exist.").arg(absFilePath), filesLocation); - rproduct->missingSourceFiles << absFilePath; - return {}; - } - if (group->enabled && fileLocations) { - CodeLocation &loc = (*fileLocations)[std::make_pair(group->targetOfModule, absFilePath)]; - if (loc.isValid()) { - if (errorInfo) { - errorInfo->append(Tr::tr("Duplicate source file '%1'.").arg(absFilePath)); - errorInfo->append(Tr::tr("First occurrence is here."), loc); - errorInfo->append(Tr::tr("Next occurrence is here."), filesLocation); + switch (child->type()) { + case ItemType::SubProject: + resolveSubProject(child, projectContext); + break; + case ItemType::FileTagger: + resolveFileTagger(state, child, projectContext, nullptr); + break; + case ItemType::JobLimit: + resolveJobLimit(state, child, projectContext, nullptr, nullptr); + break; + case ItemType::Rule: + resolveRule(state, child, projectContext, nullptr, nullptr); + break; + default: + break; } - return {}; + } catch (const ErrorInfo &e) { + state.topLevelProject().addQueuedError(e); } - loc = filesLocation; - } - SourceArtifactPtr artifact = SourceArtifactInternal::create(); - artifact->absoluteFilePath = absFilePath; - artifact->fileTags = group->fileTags; - artifact->overrideFileTags = group->overrideTags; - artifact->properties = group->properties; - artifact->targetOfModule = group->targetOfModule; - (wildcard ? group->wildcards->files : group->files).push_back(artifact); - return artifact; -} - -static QualifiedIdSet propertiesToEvaluate(std::deque<QualifiedId> initialProps, - const PropertyDependencies &deps) -{ - std::deque<QualifiedId> remainingProps = std::move(initialProps); - QualifiedIdSet allProperties; - while (!remainingProps.empty()) { - const QualifiedId prop = remainingProps.front(); - remainingProps.pop_front(); - const auto insertResult = allProperties.insert(prop); - if (!insertResult.second) - continue; - transform(deps.value(prop), remainingProps, [](const QualifiedId &id) { return id; }); - } - return allProperties; -} - -QVariantMap ProjectResolver::Private::resolveAdditionalModuleProperties( - const Item *group, const QVariantMap ¤tValues) -{ - // Step 1: Retrieve the properties directly set in the group - const ModulePropertiesPerGroup &mp = mapValue( - state.topLevelProject().productInfos, productContext->item).modulePropertiesSetInGroups; - const auto it = mp.find(group); - if (it == mp.end()) - return {}; - const QualifiedIdSet &propsSetInGroup = it->second; - - // Step 2: Gather all properties that depend on these properties. - const QualifiedIdSet &propsToEval = propertiesToEvaluate( - rangeTo<std::deque<QualifiedId>>(propsSetInGroup), - evaluator.propertyDependencies()); - - // Step 3: Evaluate all these properties and replace their values in the map - QVariantMap modulesMap = currentValues; - QHash<QString, QStringList> propsPerModule; - for (auto fullPropName : propsToEval) { - const QString moduleName - = QualifiedId(fullPropName.mid(0, fullPropName.size() - 1)).toString(); - propsPerModule[moduleName] << fullPropName.last(); - } - EvalCacheEnabler cachingEnabler(&evaluator, productContext->product->sourceDirectory); - for (const Item::Module &module : group->modules()) { - const QString &fullModName = module.name.toString(); - const QStringList propsForModule = propsPerModule.take(fullModName); - if (propsForModule.empty()) - continue; - QVariantMap reusableValues = modulesMap.value(fullModName).toMap(); - for (const QString &prop : std::as_const(propsForModule)) - reusableValues.remove(prop); - modulesMap.insert(fullModName, - evaluateProperties(module.item, module.item, reusableValues, true, true)); } - return modulesMap; -} -void ProjectResolver::Private::resolveGroup(Item *item, ResolverProjectContext *projectContext) -{ - checkCancelation(); - const bool parentEnabled = productContext->currentGroup - ? productContext->currentGroup->enabled - : productContext->product->enabled; - const bool isEnabled = parentEnabled - && evaluator.boolValue(item, StringConstants::conditionProperty()); - try { - resolveGroupFully(item, projectContext, isEnabled); - } catch (const ErrorInfo &error) { - if (!isEnabled) { - qCDebug(lcProjectResolver) << "error resolving group at" << item->location() - << error.toString(); - return; - } - if (setupParams.productErrorMode() == ErrorHandlingMode::Strict) - throw; - logger.printWarning(error); - } + for (ProjectContext * const childContext : projectContext->children) + resolveProject(childContext); } -void ProjectResolver::Private::resolveGroupFully( - Item *item, ResolverProjectContext *projectContext, bool isEnabled) +void ProjectResolver::Private::resolveSubProject(Item *item, ProjectContext *projectContext) { - AccumulatingTimer groupTimer(setupParams.logElapsedTime() ? &elapsedTimeGroups : nullptr); - - const auto getGroupPropertyMap = [this, item](const ArtifactProperties *existingProps) { - PropertyMapPtr moduleProperties; - bool newPropertyMapRequired = false; - if (existingProps) - moduleProperties = existingProps->propertyMap(); - if (!moduleProperties) { - newPropertyMapRequired = true; - moduleProperties = productContext->currentGroup - ? productContext->currentGroup->properties - : productContext->product->moduleProperties; - } - const QVariantMap newModuleProperties - = resolveAdditionalModuleProperties(item, moduleProperties->value()); - if (!newModuleProperties.empty()) { - if (newPropertyMapRequired) - moduleProperties = PropertyMapInternal::create(); - moduleProperties->setValue(newModuleProperties); - } - return moduleProperties; - }; - - QStringList files = evaluator.stringListValue(item, StringConstants::filesProperty()); - bool fileTagsSet; - const FileTags fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty(), - &fileTagsSet); - const QStringList fileTagsFilter - = evaluator.stringListValue(item, StringConstants::fileTagsFilterProperty()); - if (!fileTagsFilter.empty()) { - if (Q_UNLIKELY(!files.empty())) - throw ErrorInfo(Tr::tr("Group.files and Group.fileTagsFilters are exclusive."), - item->location()); - - if (!isEnabled) - return; - - ResolverProductContext::ArtifactPropertiesInfo &apinfo - = productContext->artifactPropertiesPerFilter[fileTagsFilter]; - if (apinfo.first) { - const auto it = std::find_if(apinfo.second.cbegin(), apinfo.second.cend(), - [item](const CodeLocation &loc) { - return item->location().filePath() == loc.filePath(); - }); - if (it != apinfo.second.cend()) { - ErrorInfo error(Tr::tr("Conflicting fileTagsFilter in Group items.")); - error.append(Tr::tr("First item"), *it); - error.append(Tr::tr("Second item"), item->location()); - throw error; - } - } else { - apinfo.first = ArtifactProperties::create(); - apinfo.first->setFileTagsFilter(FileTags::fromStringList(fileTagsFilter)); - productContext->product->artifactProperties.push_back(apinfo.first); - } - apinfo.second.push_back(item->location()); - apinfo.first->setPropertyMapInternal(getGroupPropertyMap(apinfo.first.get())); - apinfo.first->addExtraFileTags(fileTags); + // If we added a Project child item in ProductsCollector, then the sub-project will be + // resolved normally via resolveProject(). Otherwise, it was not loaded, for instance + // because its Properties condition was false, and special handling applies. + if (item->child(ItemType::Project)) return; - } - QStringList patterns; - for (int i = files.size(); --i >= 0;) { - if (FileInfo::isPattern(files[i])) - patterns.push_back(files.takeAt(i)); - } - GroupPtr group = ResolvedGroup::create(); - bool prefixWasSet = false; - group->prefix = evaluator.stringValue(item, StringConstants::prefixProperty(), QString(), - &prefixWasSet); - if (!prefixWasSet && productContext->currentGroup) - group->prefix = productContext->currentGroup->prefix; - if (!group->prefix.isEmpty()) { - for (auto it = files.rbegin(), end = files.rend(); it != end; ++it) - it->prepend(group->prefix); - } - group->location = item->location(); - group->enabled = isEnabled; - group->properties = getGroupPropertyMap(nullptr); - group->fileTags = fileTags; - group->overrideTags = evaluator.boolValue(item, StringConstants::overrideTagsProperty()); - if (group->overrideTags && fileTagsSet) { - if (group->fileTags.empty() ) - group->fileTags.insert(unknownFileTag()); - } else if (productContext->currentGroup) { - group->fileTags.unite(productContext->currentGroup->fileTags); - } - const CodeLocation filesLocation = item->property(StringConstants::filesProperty())->location(); - const VariantValueConstPtr moduleProp = item->variantProperty( - StringConstants::modulePropertyInternal()); - if (moduleProp) - group->targetOfModule = moduleProp->value().toString(); - ErrorInfo fileError; - if (!patterns.empty()) { - group->wildcards = std::make_unique<SourceWildCards>(); - SourceWildCards *wildcards = group->wildcards.get(); - wildcards->group = group.get(); - wildcards->excludePatterns = evaluator.stringListValue( - item, StringConstants::excludeFilesProperty()); - wildcards->patterns = patterns; - const Set<QString> files = wildcards->expandPatterns(group, - FileInfo::path(item->file()->filePath()), - projectContext->project->topLevelProject()->buildDirectory); - for (const QString &fileName : files) - createSourceArtifact(productContext->product, fileName, group, true, filesLocation, - &productContext->sourceArtifactLocations, &fileError); - } - - for (const QString &fileName : std::as_const(files)) { - createSourceArtifact(productContext->product, fileName, group, false, filesLocation, - &productContext->sourceArtifactLocations, &fileError); - } - if (fileError.hasError()) { - if (group->enabled) { - if (setupParams.productErrorMode() == ErrorHandlingMode::Strict) - throw ErrorInfo(fileError); - logger.printWarning(fileError); - } else { - qCDebug(lcProjectResolver) << "error for disabled group:" << fileError.toString(); - } - } - group->name = evaluator.stringValue(item, StringConstants::nameProperty()); - if (group->name.isEmpty()) - group->name = Tr::tr("Group %1").arg(productContext->product->groups.size()); - productContext->product->groups.push_back(group); - - class GroupContextSwitcher { - public: - GroupContextSwitcher(ResolverProductContext &context, const GroupConstPtr &newGroup) - : m_context(context), m_oldGroup(context.currentGroup) { - m_context.currentGroup = newGroup; - } - ~GroupContextSwitcher() { m_context.currentGroup = m_oldGroup; } - private: - ResolverProductContext &m_context; - const GroupConstPtr m_oldGroup; - }; - GroupContextSwitcher groupSwitcher(*productContext, group); - for (Item * const childItem : item->children()) - resolveGroup(childItem, projectContext); -} - -void ProjectResolver::Private::adaptExportedPropertyValues(const Item *shadowProductItem) -{ - ExportedModule &m = productContext->product->exportedModule; - const QVariantList prefixList = m.propertyValues.take( - StringConstants::prefixMappingProperty()).toList(); - const QString shadowProductName = evaluator.stringValue( - shadowProductItem, StringConstants::nameProperty()); - const QString shadowProductBuildDir = evaluator.stringValue( - shadowProductItem, StringConstants::buildDirectoryProperty()); - QVariantMap prefixMap; - for (const QVariant &v : prefixList) { - const QVariantMap o = v.toMap(); - prefixMap.insert(o.value(QStringLiteral("prefix")).toString(), - o.value(QStringLiteral("replacement")).toString()); - } - const auto valueRefersToImportingProduct - = [shadowProductName, shadowProductBuildDir](const QString &value) { - return value.toLower().contains(shadowProductName.toLower()) - || value.contains(shadowProductBuildDir); - }; - static const auto stringMapper = [](const QVariantMap &mappings, const QString &value) - -> QString { - for (auto it = mappings.cbegin(); it != mappings.cend(); ++it) { - if (value.startsWith(it.key())) - return it.value().toString() + value.mid(it.key().size()); - } - return value; - }; - const auto stringListMapper = [&valueRefersToImportingProduct]( - const QVariantMap &mappings, const QStringList &value) -> QStringList { - QStringList result; - result.reserve(value.size()); - for (const QString &s : value) { - if (!valueRefersToImportingProduct(s)) - result.push_back(stringMapper(mappings, s)); - } - return result; - }; - const std::function<QVariant(const QVariantMap &, const QVariant &)> mapper - = [&stringListMapper, &mapper]( - const QVariantMap &mappings, const QVariant &value) -> QVariant { - switch (static_cast<QMetaType::Type>(value.userType())) { - case QMetaType::QString: - return stringMapper(mappings, value.toString()); - case QMetaType::QStringList: - return stringListMapper(mappings, value.toStringList()); - case QMetaType::QVariantMap: { - QVariantMap m = value.toMap(); - for (auto it = m.begin(); it != m.end(); ++it) - it.value() = mapper(mappings, it.value()); - return m; - } - default: - return value; - } - }; - for (auto it = m.propertyValues.begin(); it != m.propertyValues.end(); ++it) - it.value() = mapper(prefixMap, it.value()); - for (auto it = m.modulePropertyValues.begin(); it != m.modulePropertyValues.end(); ++it) - it.value() = mapper(prefixMap, it.value()); - for (ExportedModuleDependency &dep : m.moduleDependencies) { - for (auto it = dep.moduleProperties.begin(); it != dep.moduleProperties.end(); ++it) - it.value() = mapper(prefixMap, it.value()); - } -} - -void ProjectResolver::Private::collectExportedProductDependencies() -{ - ResolvedProductPtr dummyProduct = ResolvedProduct::create(); - dummyProduct->enabled = false; - for (const auto &exportingProductInfo : std::as_const(productExportInfo)) { - const ResolvedProductPtr exportingProduct = exportingProductInfo.first; - if (!exportingProduct->enabled) - continue; - Item * const importingProductItem = exportingProductInfo.second; - - std::vector<std::pair<ResolvedProductPtr, QVariantMap>> directDeps; - for (const Item::Module &m : importingProductItem->modules()) { - if (m.name.toString() != exportingProduct->name) - continue; - for (const Item::Module &dep : m.item->modules()) { - if (dep.productInfo) { - directDeps.emplace_back(productsByItem.value(dep.productInfo->item), - m.parameters); - } - } - } - for (const auto &dep : directDeps) { - if (!contains(exportingProduct->exportedModule.productDependencies, - dep.first->uniqueName())) { - exportingProduct->exportedModule.productDependencies.push_back( - dep.first->uniqueName()); - } - if (!dep.second.isEmpty()) { - exportingProduct->exportedModule.dependencyParameters.insert(dep.first, - dep.second); - } - } - auto &productDeps = exportingProduct->exportedModule.productDependencies; - std::sort(productDeps.begin(), productDeps.end()); + ResolvedProjectPtr project = ResolvedProject::create(); + project->enabled = false; + project->parentProject = projectContext->project; + projectContext->project->subProjects << project; + if (Item * const propertiesItem = item->child(ItemType::PropertiesInSubProject)) { + project->name = state.evaluator().stringValue(propertiesItem, + StringConstants::nameProperty()); } } @@ -1224,12 +453,12 @@ void ProjectResolver::Private::collectNameFromOverride(const QString &overrideSt }; const QString &projectName = extract(StringConstants::projectsOverridePrefix()); if (!projectName.isEmpty()) { - state.topLevelProject().projectNamesUsedInOverrides.insert(projectName); + state.topLevelProject().addProjectNameUsedInOverrides(projectName); return; } const QString &productName = extract(StringConstants::productsOverridePrefix()); if (!productName.isEmpty()) { - state.topLevelProject().productNamesUsedInOverrides.insert(productName.left( + state.topLevelProject().addProductNameUsedInOverrides(productName.left( productName.indexOf(StringConstants::dot()))); return; } @@ -1262,869 +491,67 @@ void ProjectResolver::Private::loadTopLevelProjectItem() void ProjectResolver::Private::buildProjectTree() { - state.topLevelProject().buildDirectory = TopLevelProject::deriveBuildDirectory( + state.topLevelProject().setBuildDirectory(TopLevelProject::deriveBuildDirectory( state.parameters().buildRoot(), - TopLevelProject::deriveId(state.parameters().finalBuildConfigurationTree())); + TopLevelProject::deriveId(state.parameters().finalBuildConfigurationTree()))); rootProjectItem->setProperty(StringConstants::sourceDirectoryProperty(), VariantValue::create(QFileInfo(rootProjectItem->file()->filePath()) .absolutePath())); rootProjectItem->setProperty(StringConstants::buildDirectoryProperty(), - VariantValue::create(state.topLevelProject().buildDirectory)); + VariantValue::create(state.topLevelProject().buildDirectory())); rootProjectItem->setProperty(StringConstants::profileProperty(), VariantValue::create(state.parameters().topLevelProfile())); - productsCollector.run(rootProjectItem); - productsHandler.run(); + ProductsCollector(state).run(rootProjectItem); - state.itemReader().clearExtraSearchPathsStack(); // TODO: Unneeded? AccumulatingTimer timer(state.parameters().logElapsedTime() - ? &elapsedTimePropertyChecking : nullptr); - checkPropertyDeclarations(rootProjectItem, state.topLevelProject().disabledItems, - state.parameters(), state.logger()); -} - -void ProjectResolver::Private::resolveShadowProduct(Item *item, ResolverProjectContext *) -{ - if (!productContext->product->enabled) - return; - for (const auto &m : item->modules()) { - if (m.name.toString() != productContext->product->name) - continue; - collectPropertiesForExportItem(m.item); - for (const auto &dep : m.item->modules()) - collectPropertiesForModuleInExportItem(dep); - break; - } - try { - adaptExportedPropertyValues(item); - } catch (const ErrorInfo &) {} - productExportInfo.emplace_back(productContext->product, item); -} - -void ProjectResolver::Private::setupExportedProperties( - const Item *item, const QString &namePrefix, std::vector<ExportedProperty> &properties) -{ - const auto &props = item->properties(); - for (auto it = props.cbegin(); it != props.cend(); ++it) { - const QString qualifiedName = namePrefix.isEmpty() - ? it.key() : namePrefix + QLatin1Char('.') + it.key(); - if ((item->type() == ItemType::Export || item->type() == ItemType::Properties) - && qualifiedName == StringConstants::prefixMappingProperty()) { - continue; - } - const ValuePtr &v = it.value(); - if (v->type() == Value::ItemValueType) { - setupExportedProperties(std::static_pointer_cast<ItemValue>(v)->item(), - qualifiedName, properties); - continue; - } - ExportedProperty exportedProperty; - exportedProperty.fullName = qualifiedName; - exportedProperty.type = item->propertyDeclaration(it.key()).type(); - if (v->type() == Value::VariantValueType) { - exportedProperty.sourceCode = toJSLiteral( - std::static_pointer_cast<VariantValue>(v)->value()); - } else { - QBS_CHECK(v->type() == Value::JSSourceValueType); - const JSSourceValue * const sv = static_cast<JSSourceValue *>(v.get()); - exportedProperty.sourceCode = sv->sourceCode().toString(); - } - const ItemDeclaration itemDecl - = BuiltinDeclarations::instance().declarationsForType(item->type()); - PropertyDeclaration propertyDecl; - const auto itemProperties = itemDecl.properties(); - for (const PropertyDeclaration &decl : itemProperties) { - if (decl.name() == it.key()) { - propertyDecl = decl; - exportedProperty.isBuiltin = true; - break; - } - } - - // Do not add built-in properties that were left at their default value. - if (!exportedProperty.isBuiltin || evaluator.isNonDefaultValue(item, it.key())) - properties.push_back(exportedProperty); - } - - // Order the list of properties, so the output won't look so random. - static const auto less = [](const ExportedProperty &p1, const ExportedProperty &p2) -> bool { - const int p1ComponentCount = p1.fullName.count(QLatin1Char('.')); - const int p2ComponentCount = p2.fullName.count(QLatin1Char('.')); - if (p1.isBuiltin && !p2.isBuiltin) - return true; - if (!p1.isBuiltin && p2.isBuiltin) - return false; - if (p1ComponentCount < p2ComponentCount) - return true; - if (p1ComponentCount > p2ComponentCount) - return false; - return p1.fullName < p2.fullName; - }; - std::sort(properties.begin(), properties.end(), less); -} - -static bool usesImport(const ExportedProperty &prop, const QRegularExpression ®ex) -{ - return prop.sourceCode.indexOf(regex) != -1; -} - -static bool usesImport(const ExportedItem &item, const QRegularExpression ®ex) -{ - return any_of(item.properties, - [regex](const ExportedProperty &p) { return usesImport(p, regex); }) - || any_of(item.children, - [regex](const ExportedItemPtr &child) { return usesImport(*child, regex); }); -} - -static bool usesImport(const ExportedModule &module, const QString &name) -{ - // Imports are used in three ways: - // (1) var f = new TextFile(...); - // (2) var path = FileInfo.joinPaths(...) - // (3) var obj = DataCollection; - const QString pattern = QStringLiteral("\\b%1\\b"); - - const QRegularExpression regex(pattern.arg(name)); // std::regex is much slower - return any_of(module.m_properties, - [regex](const ExportedProperty &p) { return usesImport(p, regex); }) - || any_of(module.children, - [regex](const ExportedItemPtr &child) { return usesImport(*child, regex); }); -} - -static QString getLineAtLocation(const CodeLocation &loc, const QString &content) -{ - int pos = 0; - int currentLine = 1; - while (currentLine < loc.line()) { - while (content.at(pos++) != QLatin1Char('\n')) - ; - ++currentLine; - } - const int eolPos = content.indexOf(QLatin1Char('\n'), pos); - return content.mid(pos, eolPos - pos); -} - -void ProjectResolver::Private::resolveExport(Item *exportItem, ResolverProjectContext *) -{ - ExportedModule &exportedModule = productContext->product->exportedModule; - setupExportedProperties(exportItem, QString(), exportedModule.m_properties); - static const auto cmpFunc = [](const ExportedProperty &p1, const ExportedProperty &p2) { - return p1.fullName < p2.fullName; - }; - std::sort(exportedModule.m_properties.begin(), exportedModule.m_properties.end(), cmpFunc); - - transform(exportItem->children(), exportedModule.children, - [&exportedModule, this](const auto &child) { - return resolveExportChild(child, exportedModule); }); - - for (const JsImport &jsImport : exportItem->file()->jsImports()) { - if (usesImport(exportedModule, jsImport.scopeName)) { - exportedModule.importStatements << getLineAtLocation(jsImport.location, - exportItem->file()->content()); - } - } - const auto builtInImports = JsExtensions::extensionNames(); - for (const QString &builtinImport: builtInImports) { - if (usesImport(exportedModule, builtinImport)) - exportedModule.importStatements << QStringLiteral("import qbs.") + builtinImport; - } - exportedModule.importStatements.sort(); -} - -// TODO: This probably wouldn't be necessary if we had item serialization. -std::unique_ptr<ExportedItem> ProjectResolver::Private::resolveExportChild( - const Item *item, const ExportedModule &module) -{ - std::unique_ptr<ExportedItem> exportedItem(new ExportedItem); - - // This is the type of the built-in base item. It may turn out that we need to support - // derived items under Export. In that case, we probably need a new Item member holding - // the original type name. - exportedItem->name = item->typeName(); - - transform(item->children(), exportedItem->children, [&module, this](const auto &child) { - return resolveExportChild(child, module); }); - - setupExportedProperties(item, QString(), exportedItem->properties); - return exportedItem; -} - -QString ProjectResolver::Private::sourceCodeAsFunction(const JSSourceValueConstPtr &value, - const PropertyDeclaration &decl) const -{ - QString &scriptFunction = scriptFunctions[std::make_pair(value->sourceCode(), - decl.functionArgumentNames())]; - if (!scriptFunction.isNull()) - return scriptFunction; - const QString args = decl.functionArgumentNames().join(QLatin1Char(',')); - if (value->hasFunctionForm()) { - // Insert the argument list. - scriptFunction = value->sourceCodeForEvaluation(); - scriptFunction.insert(10, args); - // Remove the function application "()" that has been - // added in ItemReaderASTVisitor::visitStatement. - scriptFunction.chop(2); - } else { - scriptFunction = QLatin1String("(function(") + args + QLatin1String("){return ") - + value->sourceCode().toString() + QLatin1String(";})"); - } - return scriptFunction; -} - -QString ProjectResolver::Private::sourceCodeForEvaluation(const JSSourceValueConstPtr &value) const -{ - QString &code = sourceCode[value->sourceCode()]; - if (!code.isNull()) - return code; - code = value->sourceCodeForEvaluation(); - return code; -} - -ScriptFunctionPtr ProjectResolver::Private::scriptFunctionValue( - Item *item, const QString &name) const -{ - JSSourceValuePtr value = item->sourceProperty(name); - ScriptFunctionPtr &script = scriptFunctionMap[value ? value->location() : CodeLocation()]; - if (!script.get()) { - script = ScriptFunction::create(); - const PropertyDeclaration decl = item->propertyDeclaration(name); - script->sourceCode = sourceCodeAsFunction(value, decl); - script->location = value->location(); - script->fileContext = resolvedFileContext(value->file()); - } - return script; -} - -ResolvedFileContextPtr ProjectResolver::Private::resolvedFileContext( - const FileContextConstPtr &ctx) const -{ - ResolvedFileContextPtr &result = fileContextMap[ctx]; - if (!result) - result = ResolvedFileContext::create(*ctx); - return result; -} - -void ProjectResolver::Private::resolveRule(Item *item, ResolverProjectContext *projectContext) -{ - checkCancelation(); - - if (!evaluator.boolValue(item, StringConstants::conditionProperty())) - return; - - RulePtr rule = Rule::create(); - - // read artifacts - bool hasArtifactChildren = false; - for (Item * const child : item->children()) { - if (Q_UNLIKELY(child->type() != ItemType::Artifact)) { - throw ErrorInfo(Tr::tr("'Rule' can only have children of type 'Artifact'."), - child->location()); - } - hasArtifactChildren = true; - resolveRuleArtifact(rule, child); - } - - rule->name = evaluator.stringValue(item, StringConstants::nameProperty()); - rule->prepareScript.initialize( - scriptFunctionValue(item, StringConstants::prepareProperty())); - rule->outputArtifactsScript.initialize( - scriptFunctionValue(item, StringConstants::outputArtifactsProperty())); - rule->outputFileTags = evaluator.fileTagsValue(item, StringConstants::outputFileTagsProperty()); - if (rule->outputArtifactsScript.isValid()) { - if (hasArtifactChildren) - throw ErrorInfo(Tr::tr("The Rule.outputArtifacts script is not allowed in rules " - "that contain Artifact items."), - item->location()); - } - if (!hasArtifactChildren && rule->outputFileTags.empty()) { - throw ErrorInfo(Tr::tr("A rule needs to have Artifact items or a non-empty " - "outputFileTags property."), item->location()); - } - rule->multiplex = evaluator.boolValue(item, StringConstants::multiplexProperty()); - rule->alwaysRun = evaluator.boolValue(item, StringConstants::alwaysRunProperty()); - rule->inputs = evaluator.fileTagsValue(item, StringConstants::inputsProperty()); - rule->inputsFromDependencies - = evaluator.fileTagsValue(item, StringConstants::inputsFromDependenciesProperty()); - bool requiresInputsSet = false; - rule->requiresInputs = evaluator.boolValue(item, StringConstants::requiresInputsProperty(), - &requiresInputsSet); - if (!requiresInputsSet) - rule->requiresInputs = rule->declaresInputs(); - rule->auxiliaryInputs - = evaluator.fileTagsValue(item, StringConstants::auxiliaryInputsProperty()); - rule->excludedInputs - = evaluator.fileTagsValue(item, StringConstants::excludedInputsProperty()); - if (rule->excludedInputs.empty()) { - rule->excludedInputs = evaluator.fileTagsValue( - item, StringConstants::excludedAuxiliaryInputsProperty()); - } - rule->explicitlyDependsOn - = evaluator.fileTagsValue(item, StringConstants::explicitlyDependsOnProperty()); - rule->explicitlyDependsOnFromDependencies = evaluator.fileTagsValue( - item, StringConstants::explicitlyDependsOnFromDependenciesProperty()); - rule->module = moduleContext ? moduleContext->module : projectContext->dummyModule; - if (!rule->multiplex && !rule->declaresInputs()) { - throw ErrorInfo(Tr::tr("Rule has no inputs, but is not a multiplex rule."), - item->location()); - } - if (!rule->multiplex && !rule->requiresInputs) { - throw ErrorInfo(Tr::tr("Rule.requiresInputs is false for non-multiplex rule."), - item->location()); - } - if (!rule->declaresInputs() && rule->requiresInputs) { - throw ErrorInfo(Tr::tr("Rule.requiresInputs is true, but the rule " - "does not declare any input tags."), item->location()); - } - if (productContext) { - rule->product = productContext->product.get(); - productContext->product->rules.push_back(rule); - } else { - projectContext->rules.push_back(rule); - } -} - -void ProjectResolver::Private::resolveRuleArtifact(const RulePtr &rule, Item *item) -{ - RuleArtifactPtr artifact = RuleArtifact::create(); - rule->artifacts.push_back(artifact); - artifact->location = item->location(); - - if (const auto sourceProperty = item->sourceProperty(StringConstants::filePathProperty())) - artifact->filePathLocation = sourceProperty->location(); - - artifact->filePath = verbatimValue(item, StringConstants::filePathProperty()); - artifact->fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty()); - artifact->alwaysUpdated = evaluator.boolValue(item, - StringConstants::alwaysUpdatedProperty()); - - QualifiedIdSet seenBindings; - for (Item *obj = item; obj; obj = obj->prototype()) { - for (QMap<QString, ValuePtr>::const_iterator it = obj->properties().constBegin(); - it != obj->properties().constEnd(); ++it) - { - if (it.value()->type() != Value::ItemValueType) - continue; - resolveRuleArtifactBinding(artifact, - std::static_pointer_cast<ItemValue>(it.value())->item(), - QStringList(it.key()), &seenBindings); - } - } -} - -void ProjectResolver::Private::resolveRuleArtifactBinding( - const RuleArtifactPtr &ruleArtifact, Item *item, const QStringList &namePrefix, - QualifiedIdSet *seenBindings) -{ - for (QMap<QString, ValuePtr>::const_iterator it = item->properties().constBegin(); - it != item->properties().constEnd(); ++it) - { - const QStringList name = QStringList(namePrefix) << it.key(); - if (it.value()->type() == Value::ItemValueType) { - resolveRuleArtifactBinding(ruleArtifact, - std::static_pointer_cast<ItemValue>(it.value())->item(), name, - seenBindings); - } else if (it.value()->type() == Value::JSSourceValueType) { - const auto insertResult = seenBindings->insert(name); - if (!insertResult.second) - continue; - JSSourceValuePtr sourceValue = std::static_pointer_cast<JSSourceValue>(it.value()); - RuleArtifact::Binding rab; - rab.name = name; - rab.code = sourceCodeForEvaluation(sourceValue); - rab.location = sourceValue->location(); - ruleArtifact->bindings.push_back(rab); - } else { - QBS_ASSERT(!"unexpected value type", continue); - } - } -} - -void ProjectResolver::Private::resolveFileTagger(Item *item, ResolverProjectContext *projectContext) -{ - checkCancelation(); - if (!evaluator.boolValue(item, StringConstants::conditionProperty())) - return; - std::vector<FileTaggerConstPtr> &fileTaggers = productContext - ? productContext->product->fileTaggers - : projectContext->fileTaggers; - const QStringList patterns = evaluator.stringListValue(item, - StringConstants::patternsProperty()); - if (patterns.empty()) - throw ErrorInfo(Tr::tr("FileTagger.patterns must be a non-empty list."), item->location()); - - const FileTags fileTags = evaluator.fileTagsValue(item, StringConstants::fileTagsProperty()); - if (fileTags.empty()) - throw ErrorInfo(Tr::tr("FileTagger.fileTags must not be empty."), item->location()); - - for (const QString &pattern : patterns) { - if (pattern.isEmpty()) - throw ErrorInfo(Tr::tr("A FileTagger pattern must not be empty."), item->location()); - } - - const int priority = evaluator.intValue(item, StringConstants::priorityProperty()); - fileTaggers.push_back(FileTagger::create(patterns, fileTags, priority)); -} - -void ProjectResolver::Private::resolveJobLimit(Item *item, ResolverProjectContext *projectContext) -{ - if (!evaluator.boolValue(item, StringConstants::conditionProperty())) - return; - const QString jobPool = evaluator.stringValue(item, StringConstants::jobPoolProperty()); - if (jobPool.isEmpty()) - throw ErrorInfo(Tr::tr("A JobLimit item needs to have a non-empty '%1' property.") - .arg(StringConstants::jobPoolProperty()), item->location()); - bool jobCountWasSet; - const int jobCount = evaluator.intValue(item, StringConstants::jobCountProperty(), -1, - &jobCountWasSet); - if (!jobCountWasSet) { - throw ErrorInfo(Tr::tr("A JobLimit item needs to have a '%1' property.") - .arg(StringConstants::jobCountProperty()), item->location()); - } - if (jobCount < 0) { - throw ErrorInfo(Tr::tr("A JobLimit item must have a non-negative '%1' property.") - .arg(StringConstants::jobCountProperty()), item->location()); - } - JobLimits &jobLimits = moduleContext - ? moduleContext->jobLimits - : productContext ? productContext->product->jobLimits - : projectContext->jobLimits; - JobLimit jobLimit(jobPool, jobCount); - const int oldLimit = jobLimits.getLimit(jobPool); - if (oldLimit == -1 || oldLimit > jobCount) - jobLimits.setJobLimit(jobLimit); -} - -void ProjectResolver::Private::resolveScanner(Item *item, ResolverProjectContext *projectContext) -{ - checkCancelation(); - if (!evaluator.boolValue(item, StringConstants::conditionProperty())) { - qCDebug(lcProjectResolver) << "scanner condition is false"; - return; - } - - ResolvedScannerPtr scanner = ResolvedScanner::create(); - scanner->module = moduleContext ? moduleContext->module : projectContext->dummyModule; - scanner->inputs = evaluator.fileTagsValue(item, StringConstants::inputsProperty()); - scanner->recursive = evaluator.boolValue(item, StringConstants::recursiveProperty()); - scanner->searchPathsScript.initialize( - scriptFunctionValue(item, StringConstants::searchPathsProperty())); - scanner->scanScript.initialize( - scriptFunctionValue(item, StringConstants::scanProperty())); - productContext->product->scanners.push_back(scanner); -} - -void ProjectResolver::Private::matchArtifactProperties(const ResolvedProductPtr &product, - const std::vector<SourceArtifactPtr> &artifacts) -{ - for (const SourceArtifactPtr &artifact : artifacts) { - for (const auto &artifactProperties : product->artifactProperties) { - if (!artifact->isTargetOfModule() - && artifact->fileTags.intersects(artifactProperties->fileTagsFilter())) { - artifact->properties = artifactProperties->propertyMap(); - } - } - } + ? &state.topLevelProject().timingData().propertyChecking : nullptr); + checkPropertyDeclarations(rootProjectItem, state); } void ProjectResolver::Private::printProfilingInfo() { if (!setupParams.logElapsedTime()) return; - logger.qbsLog(LoggerInfo, true) - << " " - << Tr::tr("Project file loading and parsing took %1.") - .arg(elapsedTimeString(state.itemReader().elapsedTime())); - productsCollector.printProfilingInfo(2); - productsHandler.printProfilingInfo(2); - state.dependenciesResolver().printProfilingInfo(4); - state.moduleInstantiator().printProfilingInfo(6); - state.propertyMerger().printProfilingInfo(6); - state.probesResolver().printProfilingInfo(4); - state.logger().qbsLog(LoggerInfo, true) - << " " - << Tr::tr("Property checking took %1.") - .arg(elapsedTimeString(elapsedTimePropertyChecking)); - logger.qbsLog(LoggerInfo, true) - << " " << Tr::tr("All property evaluation took %1.") - .arg(elapsedTimeString(elapsedTimeAllPropEval)); - logger.qbsLog(LoggerInfo, true) - << " " << Tr::tr("Module property evaluation took %1.") - .arg(elapsedTimeString(elapsedTimeModPropEval)); - logger.qbsLog(LoggerInfo, true) - << " " << Tr::tr("Resolving groups (without module property " - "evaluation) took %1.") - .arg(elapsedTimeString(elapsedTimeGroups)); -} - -class TempScopeSetter -{ -public: - TempScopeSetter(const ValuePtr &value, Item *newScope) : m_value(value), m_oldScope(value->scope()) - { - value->setScope(newScope, {}); - } - ~TempScopeSetter() { if (m_value) m_value->setScope(m_oldScope, {}); } - - TempScopeSetter(const TempScopeSetter &) = delete; - TempScopeSetter &operator=(const TempScopeSetter &) = delete; - TempScopeSetter &operator=(TempScopeSetter &&) = delete; - - TempScopeSetter(TempScopeSetter &&other) noexcept - : m_value(std::move(other.m_value)), m_oldScope(other.m_oldScope) - { - other.m_value.reset(); - other.m_oldScope = nullptr; - } - -private: - ValuePtr m_value; - Item *m_oldScope; -}; - -void ProjectResolver::Private::collectPropertiesForExportItem(Item *productModuleInstance) -{ - if (!productModuleInstance->isPresentModule()) - return; - Item * const exportItem = productModuleInstance->prototype(); - QBS_CHECK(exportItem); - QBS_CHECK(exportItem->type() == ItemType::Export); - const ItemDeclaration::Properties exportDecls = BuiltinDeclarations::instance() - .declarationsForType(ItemType::Export).properties(); - ExportedModule &exportedModule = productContext->product->exportedModule; - const auto &props = exportItem->properties(); - for (auto it = props.begin(); it != props.end(); ++it) { - const auto match - = [it](const PropertyDeclaration &decl) { return decl.name() == it.key(); }; - if (it.key() != StringConstants::prefixMappingProperty() && - std::find_if(exportDecls.begin(), exportDecls.end(), match) != exportDecls.end()) { - continue; - } - if (it.value()->type() == Value::ItemValueType) { - collectPropertiesForExportItem(it.key(), it.value(), productModuleInstance, - exportedModule.modulePropertyValues); - } else { - TempScopeSetter tss(it.value(), productModuleInstance); - evaluateProperty(exportItem, it.key(), it.value(), exportedModule.propertyValues, - false); - } - } -} - -// Collects module properties assigned to in other (higher-level) modules. -void ProjectResolver::Private::collectPropertiesForModuleInExportItem(const Item::Module &module) -{ - if (!module.item->isPresentModule()) - return; - ExportedModule &exportedModule = productContext->product->exportedModule; - if (module.productInfo || module.name.first() == StringConstants::qbsModule()) - return; - const auto checkName = [module](const ExportedModuleDependency &d) { - return module.name.toString() == d.name; + const auto print = [this](int indent, const QString &pattern, qint64 time) { + logger.qbsLog(LoggerInfo, true) << QByteArray(indent, ' ') + << pattern.arg(elapsedTimeString(time)); }; - if (any_of(exportedModule.moduleDependencies, checkName)) - return; - - Item *modulePrototype = module.item->prototype(); - while (modulePrototype && modulePrototype->type() != ItemType::Module) - modulePrototype = modulePrototype->prototype(); - if (!modulePrototype) // Can happen for broken products in relaxed mode. - return; - const Item::PropertyMap &props = modulePrototype->properties(); - ExportedModuleDependency dep; - dep.name = module.name.toString(); - for (auto it = props.begin(); it != props.end(); ++it) { - if (it.value()->type() == Value::ItemValueType) - collectPropertiesForExportItem(it.key(), it.value(), module.item, dep.moduleProperties); - } - exportedModule.moduleDependencies.push_back(dep); - - for (const auto &dep : module.item->modules()) - collectPropertiesForModuleInExportItem(dep); -} - -void ProjectResolver::Private::resolveProductDependencies() -{ - for (auto it = productsByItem.cbegin(); it != productsByItem.cend(); ++it) { - const ResolvedProductPtr &product = it.value(); - for (const Item::Module &module : it.key()->modules()) { - if (!module.productInfo) - continue; - const ResolvedProductPtr &dep = productsByItem.value(module.productInfo->item); - QBS_CHECK(dep); - QBS_CHECK(dep != product); - it.value()->dependencies << dep; - it.value()->dependencyParameters.insert(dep, module.parameters); // TODO: Streamline this with normal module dependencies? - } - - // TODO: We might want to keep the topological sorting and get rid of "module module dependencies". - std::sort(product->dependencies.begin(),product->dependencies.end(), - [](const ResolvedProductPtr &p1, const ResolvedProductPtr &p2) { - return p1->fullDisplayName() < p2->fullDisplayName(); - }); - } -} - -void ProjectResolver::Private::postProcess(const ResolvedProductPtr &product, - ResolverProjectContext *projectContext) const -{ - product->fileTaggers << projectContext->fileTaggers; - std::sort(std::begin(product->fileTaggers), std::end(product->fileTaggers), - [] (const FileTaggerConstPtr &a, const FileTaggerConstPtr &b) { - return a->priority() > b->priority(); - }); - for (const RulePtr &rule : projectContext->rules) { - RulePtr clonedRule = rule->clone(); - clonedRule->product = product.get(); - product->rules.push_back(clonedRule); - } -} - -void ProjectResolver::Private::applyFileTaggers(const ResolvedProductPtr &product) const -{ - for (const SourceArtifactPtr &artifact : product->allEnabledFiles()) - applyFileTaggers(artifact, product); -} - -void ProjectResolver::Private::applyFileTaggers(const SourceArtifactPtr &artifact, - const ResolvedProductConstPtr &product) -{ - if (!artifact->overrideFileTags || artifact->fileTags.empty()) { - const QString fileName = FileInfo::fileName(artifact->absoluteFilePath); - const FileTags fileTags = product->fileTagsForFileName(fileName); - artifact->fileTags.unite(fileTags); - if (artifact->fileTags.empty()) - artifact->fileTags.insert(unknownFileTag()); - qCDebug(lcProjectResolver) << "adding file tags" << artifact->fileTags - << "to" << fileName; - } -} - -QVariantMap ProjectResolver::Private::evaluateModuleValues(Item *item, bool lookupPrototype) -{ - AccumulatingTimer modPropEvalTimer(setupParams.logElapsedTime() - ? &elapsedTimeModPropEval : nullptr); - QVariantMap moduleValues; - for (const Item::Module &module : item->modules()) { - if (!module.item->isPresentModule()) - continue; - const QString fullName = module.name.toString(); - moduleValues[fullName] = evaluateProperties(module.item, lookupPrototype, true); - } - - return moduleValues; -} - -QVariantMap ProjectResolver::Private::evaluateProperties(Item *item, bool lookupPrototype, - bool checkErrors) -{ - const QVariantMap tmplt; - return evaluateProperties(item, item, tmplt, lookupPrototype, checkErrors); -} - -QVariantMap ProjectResolver::Private::evaluateProperties( - const Item *item, const Item *propertiesContainer, const QVariantMap &tmplt, - bool lookupPrototype, bool checkErrors) -{ - AccumulatingTimer propEvalTimer(setupParams.logElapsedTime() - ? &elapsedTimeAllPropEval : nullptr); - QVariantMap result = tmplt; - for (QMap<QString, ValuePtr>::const_iterator it = propertiesContainer->properties().begin(); - it != propertiesContainer->properties().end(); ++it) { - checkCancelation(); - evaluateProperty(item, it.key(), it.value(), result, checkErrors); - } - return lookupPrototype && propertiesContainer->prototype() - ? evaluateProperties(item, propertiesContainer->prototype(), result, true, checkErrors) - : result; -} - -void ProjectResolver::Private::evaluateProperty( - const Item *item, const QString &propName, const ValuePtr &propValue, QVariantMap &result, - bool checkErrors) -{ - JSContext * const ctx = engine->context(); - switch (propValue->type()) { - case Value::ItemValueType: - { - // Ignore items. Those point to module instances - // and are handled in evaluateModuleValues(). - break; - } - case Value::JSSourceValueType: - { - if (result.contains(propName)) - break; - const PropertyDeclaration pd = item->propertyDeclaration(propName); - if (pd.flags().testFlag(PropertyDeclaration::PropertyNotAvailableInConfig)) { - break; - } - const ScopedJsValue scriptValue(ctx, evaluator.property(item, propName)); - if (JsException ex = evaluator.engine()->checkAndClearException(propValue->location())) { - if (checkErrors) - throw ex.toErrorInfo(); - } - - // NOTE: Loses type information if scriptValue.isUndefined == true, - // as such QScriptValues become invalid QVariants. - QVariant v; - if (JS_IsFunction(ctx, scriptValue)) { - v = getJsString(ctx, scriptValue); - } else { - v = getJsVariant(ctx, scriptValue); - QVariantMap m = v.toMap(); - if (m.contains(StringConstants::importScopeNamePropertyInternal())) { - QVariantMap tmp = m; - const ScopedJsValue proto(ctx, JS_GetPrototype(ctx, scriptValue)); - m = getJsVariant(ctx, proto).toMap(); - for (auto it = tmp.begin(); it != tmp.end(); ++it) - m.insert(it.key(), it.value()); - v = m; - } - } - - if (pd.type() == PropertyDeclaration::Path && v.isValid()) { - v = v.toString(); - } else if (pd.type() == PropertyDeclaration::PathList - || pd.type() == PropertyDeclaration::StringList) { - v = v.toStringList(); - } else if (pd.type() == PropertyDeclaration::VariantList) { - v = v.toList(); - } - checkAllowedValues(v, propValue->location(), pd, propName); - result[propName] = v; - break; - } - case Value::VariantValueType: - { - if (result.contains(propName)) - break; - VariantValuePtr vvp = std::static_pointer_cast<VariantValue>(propValue); - QVariant v = vvp->value(); - - const PropertyDeclaration pd = item->propertyDeclaration(propName); - if (v.isNull() && !pd.isScalar()) // QTBUG-51237 - v = QStringList(); - - checkAllowedValues(v, propValue->location(), pd, propName); - result[propName] = v; - break; - } - } -} - -void ProjectResolver::Private::checkAllowedValues( - const QVariant &value, const CodeLocation &loc, const PropertyDeclaration &decl, - const QString &key) const -{ - const auto type = decl.type(); - if (type != PropertyDeclaration::String && type != PropertyDeclaration::StringList) - return; - - if (value.isNull()) - return; - - const auto &allowedValues = decl.allowedValues(); - if (allowedValues.isEmpty()) - return; - - const auto checkValue = [this, &loc, &allowedValues, &key](const QString &value) - { - if (!allowedValues.contains(value)) { - const auto message = Tr::tr("Value '%1' is not allowed for property '%2'.") - .arg(value, key); - ErrorInfo error(message, loc); - handlePropertyError(error, setupParams, logger); - } - }; - - if (type == PropertyDeclaration::StringList) { - const auto strings = value.toStringList(); - for (const auto &string: strings) { - checkValue(string); - } - } else if (type == PropertyDeclaration::String) { - checkValue(value.toString()); - } -} - -void ProjectResolver::Private::collectPropertiesForExportItem(const QualifiedId &moduleName, - const ValuePtr &value, Item *moduleInstance, QVariantMap &moduleProps) -{ - QBS_CHECK(value->type() == Value::ItemValueType); - Item * const itemValueItem = std::static_pointer_cast<ItemValue>(value)->item(); - if (itemValueItem->propertyDeclarations().isEmpty()) { - for (const Item::Module &module : moduleInstance->modules()) { - if (module.name == moduleName) { - itemValueItem->setPropertyDeclarations(module.item->propertyDeclarations()); - break; - } - } - } - if (itemValueItem->type() == ItemType::ModuleInstancePlaceholder) { - struct EvalPreparer { - EvalPreparer(Item *valueItem, const QualifiedId &moduleName) - : valueItem(valueItem), - hadName(!!valueItem->variantProperty(StringConstants::nameProperty())) - { - if (!hadName) { - // Evaluator expects a name here. - valueItem->setProperty(StringConstants::nameProperty(), - VariantValue::create(moduleName.toString())); - } - } - ~EvalPreparer() - { - if (!hadName) - valueItem->setProperty(StringConstants::nameProperty(), VariantValuePtr()); - } - Item * const valueItem; - const bool hadName; - }; - EvalPreparer ep(itemValueItem, moduleName); - std::vector<TempScopeSetter> tss; - for (const ValuePtr &v : itemValueItem->properties()) - tss.emplace_back(v, moduleInstance); - moduleProps.insert(moduleName.toString(), evaluateProperties(itemValueItem, false, false)); - return; - } - QBS_CHECK(itemValueItem->type() == ItemType::ModulePrefix); - const Item::PropertyMap &props = itemValueItem->properties(); - for (auto it = props.begin(); it != props.end(); ++it) { - QualifiedId fullModuleName = moduleName; - fullModuleName << it.key(); - collectPropertiesForExportItem(fullModuleName, it.value(), moduleInstance, moduleProps); - } -} - -void ProjectResolver::Private::createProductConfig(ResolvedProduct *product) -{ - EvalCacheEnabler cachingEnabler(&evaluator, productContext->product->sourceDirectory); - product->moduleProperties->setValue(evaluateModuleValues(productContext->item)); - product->productProperties = evaluateProperties(productContext->item, productContext->item, - QVariantMap(), true, true); -} - -void ProjectResolver::Private::callItemFunction(const ItemFuncMap &mappings, Item *item, - ResolverProjectContext *projectContext) -{ - const ItemFuncPtr f = mappings.value(item->type()); - QBS_CHECK(f); - if (item->type() == ItemType::Project) { - ResolverProjectContext subProjectContext = createProjectContext(projectContext); - (this->*f)(item, &subProjectContext); - } else { - (this->*f)(item, projectContext); - } -} - -ResolverProjectContext ProjectResolver::Private::createProjectContext( - ResolverProjectContext *parentProjectContext) const -{ - ResolverProjectContext subProjectContext; - subProjectContext.parentContext = parentProjectContext; - subProjectContext.project = ResolvedProject::create(); - parentProjectContext->project->subProjects.push_back(subProjectContext.project); - subProjectContext.project->parentProject = parentProjectContext->project; - return subProjectContext; + print(2, Tr::tr("Project file loading and parsing took %1."), state.itemReader().elapsedTime()); + print(2, Tr::tr("Preparing products took %1."), + state.topLevelProject().timingData().preparingProducts); + print(2, Tr::tr("Setting up Groups took %1."), + state.topLevelProject().timingData().groupsSetup); + print(2, Tr::tr("Scheduling products took %1."), + state.topLevelProject().timingData().schedulingProducts); + print(2, Tr::tr("Resolving products took %1."), + state.topLevelProject().timingData().resolvingProducts); + print(4, Tr::tr("Property evaluation took %1."), + state.topLevelProject().timingData().propertyEvaluation); + print(4, Tr::tr("Resolving groups (without module property evaluation) took %1."), + state.topLevelProject().timingData().groupsResolving); + print(4, Tr::tr("Setting up product dependencies took %1."), + state.topLevelProject().timingData().dependenciesResolving); + print(6, Tr::tr("Running module providers took %1."), + state.topLevelProject().timingData().moduleProviders); + print(6, Tr::tr("Instantiating modules took %1."), + state.topLevelProject().timingData().moduleInstantiation); + print(6, Tr::tr("Merging module property values took %1."), + state.topLevelProject().timingData().propertyMerging); + logger.qbsLog(LoggerInfo, true) << QByteArray(4, ' ') << "There were " + << state.topLevelProject().productDeferrals() + << " product deferrals with a total of " + << state.topLevelProject().productCount() << " products."; + print(2, Tr::tr("Running Probes took %1."), state.topLevelProject().timingData().probes); + state.logger().qbsLog(LoggerInfo, true) + << " " + << Tr::tr("%1 probes encountered, %2 configure scripts executed, " + "%3 re-used from current run, %4 re-used from earlier run.") + .arg(state.topLevelProject().probesEncounteredCount()) + .arg(state.topLevelProject().probesRunCount()) + .arg(state.topLevelProject().reusedCurrentProbesCount()) + .arg(state.topLevelProject().reusedOldProbesCount()); + print(2, Tr::tr("Property checking took %1."), + state.topLevelProject().timingData().propertyChecking); } } // namespace Internal diff --git a/src/lib/corelib/logging/categories.cpp b/src/lib/corelib/logging/categories.cpp index 0f844f5b4..5738dc21d 100644 --- a/src/lib/corelib/logging/categories.cpp +++ b/src/lib/corelib/logging/categories.cpp @@ -50,6 +50,7 @@ Q_LOGGING_CATEGORY(lcModuleLoader, "qbs.moduleloader", QtCriticalMsg) Q_LOGGING_CATEGORY(lcPluginManager, "qbs.pluginmanager", QtCriticalMsg) Q_LOGGING_CATEGORY(lcProjectResolver, "qbs.projectresolver", QtCriticalMsg) Q_LOGGING_CATEGORY(lcUpToDateCheck, "qbs.uptodate", QtCriticalMsg) +Q_LOGGING_CATEGORY(lcLoaderScheduling, "qbs.loader.scheduling", QtCriticalMsg) } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/logging/categories.h b/src/lib/corelib/logging/categories.h index 40c69845e..c8873c30c 100644 --- a/src/lib/corelib/logging/categories.h +++ b/src/lib/corelib/logging/categories.h @@ -53,6 +53,7 @@ Q_DECLARE_LOGGING_CATEGORY(lcModuleLoader) Q_DECLARE_LOGGING_CATEGORY(lcPluginManager) Q_DECLARE_LOGGING_CATEGORY(lcProjectResolver) Q_DECLARE_LOGGING_CATEGORY(lcUpToDateCheck) +Q_DECLARE_LOGGING_CATEGORY(lcLoaderScheduling) } // namespace Internal } // namespace qbs diff --git a/src/lib/corelib/tools/error.cpp b/src/lib/corelib/tools/error.cpp index 963089fe8..f1b90b71e 100644 --- a/src/lib/corelib/tools/error.cpp +++ b/src/lib/corelib/tools/error.cpp @@ -317,6 +317,13 @@ bool ErrorInfo::hasLocation() const return ei.codeLocation().isValid(); }); } +bool ErrorInfo::isCancelException() const +{ + return Internal::any_of(d->items, [](const ErrorItem &ei) { + return ei.description() == QLatin1String("interrupted"); + }); +} + void ErrorInfo::load(Internal::PersistentPool &pool) { pool.load(*d); diff --git a/src/lib/corelib/tools/error.h b/src/lib/corelib/tools/error.h index 4c6370b1e..ba600a558 100644 --- a/src/lib/corelib/tools/error.h +++ b/src/lib/corelib/tools/error.h @@ -113,6 +113,7 @@ public: QJsonObject toJson() const; bool isInternalError() const; bool hasLocation() const; + bool isCancelException() const; void load(Internal::PersistentPool &pool); void store(Internal::PersistentPool &pool) const; diff --git a/src/lib/corelib/tools/executablefinder.cpp b/src/lib/corelib/tools/executablefinder.cpp index 0bdc861fa..b5d9c0151 100644 --- a/src/lib/corelib/tools/executablefinder.cpp +++ b/src/lib/corelib/tools/executablefinder.cpp @@ -120,7 +120,7 @@ QString ExecutableFinder::findInPath(const QString &filePath, const QString &wor fullProgramPath = filePath; qCDebug(lcExec) << "looking for executable in PATH" << fullProgramPath; QStringList pathEnv = m_environment.value(StringConstants::pathEnvVar()) - .split(HostOsInfo::pathListSeparator(), QBS_SKIP_EMPTY_PARTS); + .split(HostOsInfo::pathListSeparator(), Qt::SkipEmptyParts); if (HostOsInfo::isWindowsHost()) pathEnv.prepend(StringConstants::dot()); for (QString directory : std::as_const(pathEnv)) { diff --git a/src/lib/corelib/tools/launchersocket.cpp b/src/lib/corelib/tools/launchersocket.cpp index 4b72d7580..7d4788ee3 100644 --- a/src/lib/corelib/tools/launchersocket.cpp +++ b/src/lib/corelib/tools/launchersocket.cpp @@ -82,12 +82,7 @@ void LauncherSocket::setSocket(QLocalSocket *socket) QBS_ASSERT(!m_socket, return); m_socket.store(socket); m_packetParser.setDevice(m_socket); - connect(m_socket, -#if (QT_VERSION < QT_VERSION_CHECK(5, 15, 0)) - static_cast<void(QLocalSocket::*)(QLocalSocket::LocalSocketError)>(&QLocalSocket::error), -#else - &QLocalSocket::errorOccurred, -#endif + connect(m_socket, &QLocalSocket::errorOccurred, this, &LauncherSocket::handleSocketError); connect(m_socket, &QLocalSocket::readyRead, this, &LauncherSocket::handleSocketDataAvailable); diff --git a/src/lib/corelib/tools/profiling.cpp b/src/lib/corelib/tools/profiling.cpp index db64a73c6..62815912d 100644 --- a/src/lib/corelib/tools/profiling.cpp +++ b/src/lib/corelib/tools/profiling.cpp @@ -72,7 +72,7 @@ void TimedActivityLogger::finishActivity() { if (!d) return; - const QString timeString = elapsedTimeString(d->timer.elapsed()); + const QString timeString = elapsedTimeString(d->timer.nsecsElapsed()); d->logger.qbsLog(LoggerInfo, true) << Tr::tr("Activity '%2' took %3.").arg(d->activity, timeString); d.reset(); @@ -98,13 +98,13 @@ void AccumulatingTimer::stop() { if (!m_timer.isValid()) return; - *m_elapsedTime += m_timer.elapsed(); + *m_elapsedTime += m_timer.nsecsElapsed(); m_timer.invalidate(); } -QString elapsedTimeString(qint64 elapsedTimeInMs) +QString elapsedTimeString(qint64 elapsedTimeInNs) { - qint64 ms = elapsedTimeInMs; + qint64 ms = elapsedTimeInNs / (1000 * 1000); qint64 s = ms/1000; ms -= s*1000; qint64 m = s/60; diff --git a/src/lib/corelib/tools/profiling.h b/src/lib/corelib/tools/profiling.h index c6fc9ede7..3220c2a82 100644 --- a/src/lib/corelib/tools/profiling.h +++ b/src/lib/corelib/tools/profiling.h @@ -52,7 +52,7 @@ namespace qbs { namespace Internal { class Logger; -QString elapsedTimeString(qint64 elapsedTimeInMs); +QString elapsedTimeString(qint64 elapsedTimeInNs); class TimedActivityLogger { diff --git a/src/lib/corelib/tools/progressobserver.h b/src/lib/corelib/tools/progressobserver.h index 9acb4b30c..73a61de37 100644 --- a/src/lib/corelib/tools/progressobserver.h +++ b/src/lib/corelib/tools/progressobserver.h @@ -41,6 +41,8 @@ #include <QtCore/qglobal.h> +#include <vector> + QT_BEGIN_NAMESPACE class QString; QT_END_NAMESPACE @@ -66,13 +68,13 @@ public: // Call this to ensure that the progress bar always goes to 100%. void setFinished(); - void setScriptEngine(ScriptEngine *engine) { m_scriptEngine = engine; } + void addScriptEngine(ScriptEngine *engine) { m_scriptEngines.push_back(engine); } protected: - ScriptEngine *scriptEngine() const { return m_scriptEngine; } + const std::vector<ScriptEngine *> &scriptEngines() const { return m_scriptEngines; } private: - ScriptEngine *m_scriptEngine = nullptr; + std::vector<ScriptEngine *> m_scriptEngines; }; } // namespace Internal diff --git a/src/lib/corelib/tools/qttools.h b/src/lib/corelib/tools/qttools.h index bc1210d53..88ada73d4 100644 --- a/src/lib/corelib/tools/qttools.h +++ b/src/lib/corelib/tools/qttools.h @@ -55,19 +55,7 @@ QT_BEGIN_NAMESPACE class QProcessEnvironment; QT_END_NAMESPACE -#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) -#define QBS_SKIP_EMPTY_PARTS QString::SkipEmptyParts -#else -#define QBS_SKIP_EMPTY_PARTS Qt::SkipEmptyParts -#endif - namespace std { -#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) -template<> struct hash<QString> { - std::size_t operator()(const QString &s) const { return qHash(s); } -}; -#endif - template<typename T1, typename T2> struct hash<std::pair<T1, T2>> { size_t operator()(const pair<T1, T2> &x) const @@ -161,12 +149,6 @@ inline qbs::QHashValueType qHash(const QVariantHash &v) return std::hash<QVariantHash>()(v) % std::numeric_limits<uint>::max(); } -#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) -namespace Qt { -inline QTextStream &endl(QTextStream &stream) { return stream << QT_PREPEND_NAMESPACE(endl); } -} // namespace Qt -#endif - QT_END_NAMESPACE namespace qbs { @@ -174,32 +156,20 @@ namespace qbs { template <class T> QSet<T> toSet(const QList<T> &list) { -#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) - return list.toSet(); -#else return QSet<T>(list.begin(), list.end()); -#endif } template<class T> QList<T> toList(const QSet<T> &set) { -#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) - return set.toList(); -#else return QList<T>(set.begin(), set.end()); -#endif } template<typename K, typename V> QHash<K, V> &unite(QHash<K, V> &h, const QHash<K, V> &other) { -#if (QT_VERSION < QT_VERSION_CHECK(5, 15, 0)) - return h.unite(other); -#else h.insert(other); return h; -#endif } inline void setupDefaultCodec(QTextStream &stream) diff --git a/src/lib/corelib/tools/scripttools.cpp b/src/lib/corelib/tools/scripttools.cpp index efa942fd5..6262e7cf7 100644 --- a/src/lib/corelib/tools/scripttools.cpp +++ b/src/lib/corelib/tools/scripttools.cpp @@ -231,44 +231,19 @@ QStringList getJsStringList(JSContext *ctx, JSValue val) return l; } -JSValue makeJsVariant(JSContext *ctx, const QVariant &v) +JSValue makeJsVariant(JSContext *ctx, const QVariant &v, quintptr id) { - switch (static_cast<QMetaType::Type>(v.userType())) { - case QMetaType::QByteArray: - return makeJsArrayBuffer(ctx, v.toByteArray()); - case QMetaType::QString: - return makeJsString(ctx, v.toString()); - case QMetaType::QStringList: - return makeJsStringList(ctx, v.toStringList()); - case QMetaType::QVariantList: - return makeJsVariantList(ctx, v.toList()); - case QMetaType::Int: - case QMetaType::UInt: - return JS_NewInt32(ctx, v.toInt()); - case QMetaType::Long: - case QMetaType::ULong: - case QMetaType::LongLong: - case QMetaType::ULongLong: - return JS_NewInt64(ctx, v.toInt()); - case QMetaType::Bool: - return JS_NewBool(ctx, v.toBool()); - case QMetaType::QDateTime: - return JS_NewDate(ctx, v.toDateTime().toString(Qt::ISODateWithMs).toUtf8().constData()); - case QMetaType::QVariantMap: - return makeJsVariantMap(ctx, v.toMap()); - default: - return JS_UNDEFINED; - } + return ScriptEngine::engineForContext(ctx)->asJsValue(v, id); } -JSValue makeJsVariantList(JSContext *ctx, const QVariantList &l) +JSValue makeJsVariantList(JSContext *ctx, const QVariantList &l, quintptr id) { - return ScriptEngine::engineForContext(ctx)->asJsValue(l); + return ScriptEngine::engineForContext(ctx)->asJsValue(l, id); } -JSValue makeJsVariantMap(JSContext *ctx, const QVariantMap &m) +JSValue makeJsVariantMap(JSContext *ctx, const QVariantMap &m, quintptr id) { - return ScriptEngine::engineForContext(ctx)->asJsValue(m); + return ScriptEngine::engineForContext(ctx)->asJsValue(m, id); } static QVariant getJsVariantImpl(JSContext *ctx, JSValue val, QList<JSValue> path) diff --git a/src/lib/corelib/tools/scripttools.h b/src/lib/corelib/tools/scripttools.h index 955c37af5..ac7ed9928 100644 --- a/src/lib/corelib/tools/scripttools.h +++ b/src/lib/corelib/tools/scripttools.h @@ -75,9 +75,9 @@ QBS_AUTOTEST_EXPORT QVariant getJsVariant(JSContext *ctx, JSValueConst val); JSValue makeJsArrayBuffer(JSContext *ctx, const QByteArray &s); JSValue makeJsString(JSContext *ctx, const QString &s); JSValue makeJsStringList(JSContext *ctx, const QStringList &l); -JSValue makeJsVariant(JSContext *ctx, const QVariant &v); -JSValue makeJsVariantList(JSContext *ctx, const QVariantList &l); -JSValue makeJsVariantMap(JSContext *ctx, const QVariantMap &m); +JSValue makeJsVariant(JSContext *ctx, const QVariant &v, quintptr id = 0); +JSValue makeJsVariantList(JSContext *ctx, const QVariantList &l, quintptr id = 0); +JSValue makeJsVariantMap(JSContext *ctx, const QVariantMap &m, quintptr id = 0); QStringList getJsStringList(JSContext *ctx, JSValueConst val); JSValue throwError(JSContext *ctx, const QString &message); using PropertyHandler = std::function<void(const JSAtom &, const JSPropertyDescriptor &)>; diff --git a/src/lib/corelib/tools/settingsmodel.cpp b/src/lib/corelib/tools/settingsmodel.cpp index 89d923496..e3c995db3 100644 --- a/src/lib/corelib/tools/settingsmodel.cpp +++ b/src/lib/corelib/tools/settingsmodel.cpp @@ -329,7 +329,7 @@ void SettingsModel::SettingsModelPrivate::readSettings() addNodeFromSettings(&rootNode, topLevelKey); for (QVariantMap::ConstIterator it = additionalProperties.constBegin(); it != additionalProperties.constEnd(); ++it) { - const QStringList nameAsList = it.key().split(QLatin1Char('.'), QBS_SKIP_EMPTY_PARTS); + const QStringList nameAsList = it.key().split(QLatin1Char('.'), Qt::SkipEmptyParts); addNode(&rootNode, nameAsList.front(), nameAsList.mid(1), it.value()); } dirty = false; diff --git a/src/lib/corelib/tools/setupprojectparameters.cpp b/src/lib/corelib/tools/setupprojectparameters.cpp index bc07acfef..28ad745ce 100644 --- a/src/lib/corelib/tools/setupprojectparameters.cpp +++ b/src/lib/corelib/tools/setupprojectparameters.cpp @@ -38,6 +38,8 @@ ****************************************************************************/ #include "setupprojectparameters.h" +#include "buildoptions.h" + #include <logging/logger.h> #include <logging/translator.h> #include <tools/buildgraphlocker.h> @@ -90,6 +92,7 @@ public: mutable QVariantMap buildConfigurationTree; mutable QVariantMap overriddenValuesTree; mutable QVariantMap finalBuildConfigTree; + int maxJobCount = 0; bool overrideBuildGraphData; bool dryRun; bool logElapsedTime; @@ -154,6 +157,9 @@ SetupProjectParameters SetupProjectParameters::fromJson(const QJsonObject &data) setValueFromJson(params.d->projectFilePath, data, "project-file-path"); setValueFromJson(params.d->buildRoot, data, "build-root"); setValueFromJson(params.d->settingsBaseDir, data, "settings-directory"); + setValueFromJson(params.d->maxJobCount, data, "max-job-count"); + if (params.maxJobCount() <= 0) + params.setMaxJobCount(BuildOptions::defaultMaxJobCount()); setValueFromJson(params.d->overriddenValues, data, "overridden-properties"); setValueFromJson(params.d->dryRun, data, "dry-run"); setValueFromJson(params.d->logElapsedTime, data, "log-time"); @@ -373,6 +379,27 @@ void SetupProjectParameters::setSettingsDirectory(const QString &settingsBaseDir } /*! + * \brief Returns the maximum number of threads to employ when resolving the project. + * If the value is not valid (i.e. <= 0), a sensible one will be derived from the number of + * available processor cores. + * The default is 0. + * \sa BuildOptions::defaultMaxJobCount + */ +int SetupProjectParameters::maxJobCount() const +{ + return d->maxJobCount; +} + +/*! + * \brief Controls how many threads to employ when resolving the project. + * A value <= 0 leaves the decision to qbs. + */ +void SetupProjectParameters::setMaxJobCount(int jobCount) +{ + d->maxJobCount = jobCount; +} + +/*! * Returns the overridden values of the build configuration. */ QVariantMap SetupProjectParameters::overriddenValues() const diff --git a/src/lib/corelib/tools/setupprojectparameters.h b/src/lib/corelib/tools/setupprojectparameters.h index 67bb5298a..76b038752 100644 --- a/src/lib/corelib/tools/setupprojectparameters.h +++ b/src/lib/corelib/tools/setupprojectparameters.h @@ -99,6 +99,9 @@ public: QString settingsDirectory() const; void setSettingsDirectory(const QString &settingsBaseDir); + int maxJobCount() const; + void setMaxJobCount(int jobCount); + QVariantMap overriddenValues() const; void setOverriddenValues(const QVariantMap &values); QVariantMap overriddenValuesTree() const; diff --git a/src/lib/corelib/tools/shellutils.cpp b/src/lib/corelib/tools/shellutils.cpp index d032aecac..5fff254f6 100644 --- a/src/lib/corelib/tools/shellutils.cpp +++ b/src/lib/corelib/tools/shellutils.cpp @@ -57,7 +57,7 @@ QString shellInterpreter(const QString &filePath) { const QString shebang = ts.readLine(); if (shebang.startsWith(QLatin1String("#!"))) { return (shebang.mid(2).split(QRegularExpression(QStringLiteral("\\s")), - QBS_SKIP_EMPTY_PARTS) << QString()).front(); + Qt::SkipEmptyParts) << QString()).front(); } } diff --git a/src/lib/corelib/tools/stringconstants.h b/src/lib/corelib/tools/stringconstants.h index 45903032c..799a140d9 100644 --- a/src/lib/corelib/tools/stringconstants.h +++ b/src/lib/corelib/tools/stringconstants.h @@ -110,6 +110,7 @@ public: QBS_STRING_CONSTANT(installDirProperty, "installDir") QBS_STRING_CONSTANT(installSourceBaseProperty, "installSourceBase") QBS_STRING_CONSTANT(isEnabledKey, "is-enabled") + QBS_STRING_CONSTANT(isEagerProperty, "isEager") QBS_STRING_CONSTANT(jobCountProperty, "jobCount") QBS_STRING_CONSTANT(jobPoolProperty, "jobPool") QBS_STRING_CONSTANT(lengthProperty, "length") diff --git a/src/lib/msbuild/io/visualstudiosolutionwriter.cpp b/src/lib/msbuild/io/visualstudiosolutionwriter.cpp index 625489ac6..874fbc71f 100644 --- a/src/lib/msbuild/io/visualstudiosolutionwriter.cpp +++ b/src/lib/msbuild/io/visualstudiosolutionwriter.cpp @@ -108,20 +108,6 @@ bool VisualStudioSolutionWriter::write(const VisualStudioSolution *solution) << project->guid().toString().toStdString() << u8"\"\n"; - const auto dependencies = solution->dependencies(project); - if (!dependencies.empty()) { - out << u8"\tProjectSection(ProjectDependencies) = postProject\n"; - - for (const auto &dependency : dependencies) - out << u8"\t\t" - << dependency->guid().toString().toStdString() - << u8" = " - << dependency->guid().toString().toStdString() - << u8"\n"; - - out << u8"\tEndProjectSection\n"; - } - out << u8"EndProject\n"; } diff --git a/src/libexec/qbs_processlauncher/launchersockethandler.cpp b/src/libexec/qbs_processlauncher/launchersockethandler.cpp index 637362b88..037f061a5 100644 --- a/src/libexec/qbs_processlauncher/launchersockethandler.cpp +++ b/src/libexec/qbs_processlauncher/launchersockethandler.cpp @@ -121,12 +121,7 @@ void LauncherSocketHandler::start() connect(m_socket, &QLocalSocket::disconnected, this, &LauncherSocketHandler::handleSocketClosed); connect(m_socket, &QLocalSocket::readyRead, this, &LauncherSocketHandler::handleSocketData); - connect(m_socket, -#if (QT_VERSION < QT_VERSION_CHECK(5, 15, 0)) - static_cast<void(QLocalSocket::*)(QLocalSocket::LocalSocketError)>(&QLocalSocket::error), -#else - &QLocalSocket::errorOccurred, -#endif + connect(m_socket, &QLocalSocket::errorOccurred, this, &LauncherSocketHandler::handleSocketError); m_socket->connectToServer(m_serverPath); } diff --git a/src/shared/quickjs/quickjs.c b/src/shared/quickjs/quickjs.c index 3665eea5e..61b9816e7 100644 --- a/src/shared/quickjs/quickjs.c +++ b/src/shared/quickjs/quickjs.c @@ -37162,6 +37162,13 @@ exception: return JS_EXCEPTION; } +JSValue JS_ObjectSeal(JSContext *ctx, JSValueConst obj, int freeze) +{ + JSValueConst argv[] = {obj}; + JS_FreeValue(ctx, js_object_seal(ctx, JS_UNDEFINED, 1, argv, freeze)); + return obj; +} + static JSValue js_object_fromEntries(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { diff --git a/src/shared/quickjs/quickjs.h b/src/shared/quickjs/quickjs.h index f2caef137..aa4a74c9c 100644 --- a/src/shared/quickjs/quickjs.h +++ b/src/shared/quickjs/quickjs.h @@ -708,6 +708,8 @@ int JS_DeleteProperty(JSContext *ctx, JSValueConst obj, JSAtom prop, int flags); int JS_SetPrototype(JSContext *ctx, JSValueConst obj, JSValueConst proto_val); JSValue JS_GetPrototype(JSContext *ctx, JSValueConst val); +JSValue JS_ObjectSeal(JSContext *ctx, JSValueConst obj, int freeze); + #define JS_GPN_STRING_MASK (1 << 0) #define JS_GPN_SYMBOL_MASK (1 << 1) #define JS_GPN_PRIVATE_MASK (1 << 2) diff --git a/tests/auto/api/testdata/link-dynamiclibs-staticlibs/link-dynamiclibs-staticlibs.qbs b/tests/auto/api/testdata/link-dynamiclibs-staticlibs/link-dynamiclibs-staticlibs.qbs index 4ba829a5b..0c86d05c6 100644 --- a/tests/auto/api/testdata/link-dynamiclibs-staticlibs/link-dynamiclibs-staticlibs.qbs +++ b/tests/auto/api/testdata/link-dynamiclibs-staticlibs/link-dynamiclibs-staticlibs.qbs @@ -40,6 +40,11 @@ Project { name: "static2" files: [ "static2.cpp", "static2.h" ] Depends { name: "cpp" } + Probe { + id: tcPrinter + property bool isGcc: qbs.toolchain.contains("gcc") + configure: { console.info("is gcc: " + isGcc); } + } } } diff --git a/tests/auto/api/testdata/link-staticlibs-dynamiclibs/link-staticlibs-dynamiclibs.qbs b/tests/auto/api/testdata/link-staticlibs-dynamiclibs/link-staticlibs-dynamiclibs.qbs index 1bcc2876f..c30cf40f9 100644 --- a/tests/auto/api/testdata/link-staticlibs-dynamiclibs/link-staticlibs-dynamiclibs.qbs +++ b/tests/auto/api/testdata/link-staticlibs-dynamiclibs/link-staticlibs-dynamiclibs.qbs @@ -16,7 +16,11 @@ Project { id: osCheck property bool isNormalUnix: qbs.targetOS.includes("unix") && !qbs.targetOS.includes("darwin") - configure: { console.info("is normal unix: " + (isNormalUnix ? "yes" : "no")); } + property bool isGcc: qbs.toolchain.contains("gcc") + configure: { + console.info("is normal unix: " + (isNormalUnix ? "yes" : "no")); + console.info("is gcc: " + isGcc); + } } } diff --git a/tests/auto/api/testdata/restored-warnings/restored-warnings.qbs b/tests/auto/api/testdata/restored-warnings/restored-warnings.qbs index bbdfbeadb..f6a68f27c 100644 --- a/tests/auto/api/testdata/restored-warnings/restored-warnings.qbs +++ b/tests/auto/api/testdata/restored-warnings/restored-warnings.qbs @@ -1,14 +1,21 @@ import qbs.Process 1.5 -CppApplication { - name: "theProduct" +Project { + CppApplication { + name: "theProduct" - property bool moreFiles: false - cpp.blubb: true + property bool moreFiles: false + cpp.blubb: true - files: ["file.cpp", "main.cpp"] - Group { - condition: moreFiles - files: ["blubb.cpp"] + files: ["file.cpp", "main.cpp"] + Group { + condition: moreFiles + files: ["blubb.cpp"] + } + } + + Product { + name: "theOtherProduct" + property bool dummy: { throw "this one comes from a thread"; } } } diff --git a/tests/auto/api/tst_api.cpp b/tests/auto/api/tst_api.cpp index 94d02c15c..140cadeb9 100644 --- a/tests/auto/api/tst_api.cpp +++ b/tests/auto/api/tst_api.cpp @@ -928,16 +928,12 @@ void TestApi::dependencyOnMultiplexedType() } else { QVERIFY(p.name() == "p2"); ++p2Count; - - // FIXME: This is an odd effect of our current algorithm: We collect the products - // matching the requested type and add Depends items with their names ("p1" in - // this case). Later, the algorithm checking for compatibility regarding the - // multiplexing axes picks the aggregate. However, the aggregate does not have - // a matching type... It's not entirely clear what the real expected - // result should be here. - QCOMPARE(p.dependencies().size(), 2); + QVERIFY(p.dependencies().contains("dep")); } } + QCOMPARE(depCount, 1); + QCOMPARE(p1Count, 3); + QCOMPARE(p2Count, 1); std::unique_ptr<qbs::BuildJob> buildJob(project.buildAllProducts(qbs::BuildOptions())); waitForFinished(buildJob.get()); QVERIFY2(!buildJob->error().hasError(), qPrintable(buildJob->error().toString())); @@ -1554,27 +1550,30 @@ void TestApi::linkDynamicAndStaticLibs() BuildDescriptionReceiver bdr; qbs::BuildOptions options; options.setEchoMode(qbs::CommandEchoModeCommandLine); + m_logSink->output.clear(); const qbs::ErrorInfo errorInfo = doBuildProject("link-dynamiclibs-staticlibs", &bdr, nullptr, nullptr, options); VERIFY_NO_ERROR(errorInfo); + const bool isGcc = m_logSink->output.contains("is gcc: true"); + const bool isNotGcc = m_logSink->output.contains("is gcc: false"); + if (isNotGcc) + QSKIP("The remainder of this test applies only to GCC"); + QVERIFY(isGcc); + // The dependent static libs should not appear in the link command for the executable. - const SettingsPtr s = settings(); - const qbs::Profile buildProfile(profileName(), s.get()); - if (profileToolchain(buildProfile).contains("gcc")) { - static const std::regex appLinkCmdRex(" -o [^ ]*/HelloWorld" QBS_HOST_EXE_SUFFIX " "); - QString appLinkCmd; - for (const QString &line : std::as_const(bdr.descriptionLines)) { - const auto ln = line.toStdString(); - if (std::regex_search(ln, appLinkCmdRex)) { - appLinkCmd = line; - break; - } + static const std::regex appLinkCmdRex(" -o [^ ]*/HelloWorld" QBS_HOST_EXE_SUFFIX " "); + QString appLinkCmd; + for (const QString &line : std::as_const(bdr.descriptionLines)) { + const auto ln = line.toStdString(); + if (std::regex_search(ln, appLinkCmdRex)) { + appLinkCmd = line; + break; } - QVERIFY(!appLinkCmd.isEmpty()); - QVERIFY(!appLinkCmd.contains("static1")); - QVERIFY(!appLinkCmd.contains("static2")); } + QVERIFY(!appLinkCmd.isEmpty()); + QVERIFY(!appLinkCmd.contains("static1")); + QVERIFY(!appLinkCmd.contains("static2")); } void TestApi::linkStaticAndDynamicLibs() @@ -1589,31 +1588,32 @@ void TestApi::linkStaticAndDynamicLibs() const bool isNormalUnix = m_logSink->output.contains("is normal unix: yes"); const bool isNotNormalUnix = m_logSink->output.contains("is normal unix: no"); QVERIFY2(isNormalUnix != isNotNormalUnix, qPrintable(m_logSink->output)); + const bool isGcc = m_logSink->output.contains("is gcc: true"); + const bool isNotGcc = m_logSink->output.contains("is gcc: false"); + if (isNotGcc) + QSKIP("The remainder of this test applies only to GCC"); + QVERIFY(isGcc); // The dependencies libdynamic1.so and libstatic2.a must not appear in the link command for the // executable. The -rpath-link line for libdynamic1.so must be there. - const SettingsPtr s = settings(); - const qbs::Profile buildProfile(profileName(), s.get()); - if (profileToolchain(buildProfile).contains("gcc")) { - static const std::regex appLinkCmdRex(" -o [^ ]*/HelloWorld" QBS_HOST_EXE_SUFFIX " "); - QString appLinkCmd; - for (const QString &line : std::as_const(bdr.descriptionLines)) { - const auto ln = line.toStdString(); - if (std::regex_search(ln, appLinkCmdRex)) { - appLinkCmd = line; - break; - } - } - QVERIFY(!appLinkCmd.isEmpty()); - if (isNormalUnix) { - const std::regex rpathLinkRex("-rpath-link=\\S*/" - + relativeProductBuildDir("dynamic2").toStdString()); - const auto ln = appLinkCmd.toStdString(); - QVERIFY(std::regex_search(ln, rpathLinkRex)); + static const std::regex appLinkCmdRex(" -o [^ ]*/HelloWorld" QBS_HOST_EXE_SUFFIX " "); + QString appLinkCmd; + for (const QString &line : std::as_const(bdr.descriptionLines)) { + const auto ln = line.toStdString(); + if (std::regex_search(ln, appLinkCmdRex)) { + appLinkCmd = line; + break; } - QVERIFY(!appLinkCmd.contains("libstatic2.a")); - QVERIFY(!appLinkCmd.contains("libdynamic2.so")); } + QVERIFY(!appLinkCmd.isEmpty()); + if (isNormalUnix) { + const std::regex rpathLinkRex("-rpath-link=\\S*/" + + relativeProductBuildDir("dynamic2").toStdString()); + const auto ln = appLinkCmd.toStdString(); + QVERIFY(std::regex_search(ln, rpathLinkRex)); + } + QVERIFY(!appLinkCmd.contains("libstatic2.a")); + QVERIFY(!appLinkCmd.contains("libdynamic2.so")); } void TestApi::listBuildSystemFiles() @@ -2217,7 +2217,7 @@ void TestApi::newPatternMatch() void TestApi::nonexistingProjectPropertyFromProduct() { qbs::SetupProjectParameters setupParams - = defaultSetupParameters("nonexistingprojectproperties"); + = defaultSetupParameters("nonexistingprojectproperties/invalidaccessfromproduct.qbs"); std::unique_ptr<qbs::SetupProjectJob> job(qbs::Project().setupProject(setupParams, m_logSink, nullptr)); waitForFinished(job.get()); @@ -2526,6 +2526,7 @@ qbs::SetupProjectParameters TestApi::defaultSetupParameters(const QString &proje setupParams.setLibexecPath(QDir::cleanPath(QCoreApplication::applicationDirPath() + QLatin1String("/" QBS_RELATIVE_LIBEXEC_PATH))); setupParams.setTopLevelProfile(profileName()); + setupParams.setMaxJobCount(2); setupParams.setConfigurationName(QStringLiteral("default")); setupParams.setSettingsDirectory(settings()->baseDirectory()); return setupParams; @@ -2710,12 +2711,14 @@ void TestApi::restoredWarnings() waitForFinished(job.get()); QVERIFY2(!job->error().hasError(), qPrintable(job->error().toString())); job.reset(nullptr); - QCOMPARE(toSet(m_logSink->warnings).size(), 3); + QCOMPARE(toSet(m_logSink->warnings).size(), 5); const auto beforeErrors = m_logSink->warnings; for (const qbs::ErrorInfo &e : beforeErrors) { const QString msg = e.toString(); QVERIFY2(msg.contains("Superfluous version") || msg.contains("Property 'blubb' is not declared") + || msg.contains("this one comes from a thread") + || msg.contains("Product 'theOtherProduct' had errors and was disabled") || msg.contains("Product 'theProduct' had errors and was disabled"), qPrintable(msg)); } @@ -2726,7 +2729,7 @@ void TestApi::restoredWarnings() waitForFinished(job.get()); QVERIFY2(!job->error().hasError(), qPrintable(job->error().toString())); job.reset(nullptr); - QCOMPARE(toSet(m_logSink->warnings).size(), 3); + QCOMPARE(toSet(m_logSink->warnings).size(), 5); m_logSink->warnings.clear(); // Re-resolving with changes: Errors come from the re-resolving, stored ones must be suppressed. @@ -2737,13 +2740,15 @@ void TestApi::restoredWarnings() waitForFinished(job.get()); QVERIFY2(!job->error().hasError(), qPrintable(job->error().toString())); job.reset(nullptr); - QCOMPARE(toSet(m_logSink->warnings).size(), 4); // One more for the additional group + QCOMPARE(toSet(m_logSink->warnings).size(), 6); // One more for the additional group const auto afterErrors = m_logSink->warnings; for (const qbs::ErrorInfo &e : afterErrors) { const QString msg = e.toString(); QVERIFY2(msg.contains("Superfluous version") || msg.contains("Property 'blubb' is not declared") || msg.contains("blubb.cpp' does not exist") + || msg.contains("this one comes from a thread") + || msg.contains("Product 'theOtherProduct' had errors and was disabled") || msg.contains("Product 'theProduct' had errors and was disabled"), qPrintable(msg)); } diff --git a/tests/auto/blackbox/testdata-providers/broken-provider/broken-provider.qbs b/tests/auto/blackbox/testdata-providers/broken-provider/broken-provider.qbs new file mode 100644 index 000000000..461c7e30e --- /dev/null +++ b/tests/auto/blackbox/testdata-providers/broken-provider/broken-provider.qbs @@ -0,0 +1,13 @@ +Project { + qbsModuleProviders: "provider_a" + name: "project" + Project { + name: "innerProject" + Product { + name: "p1" + Depends { name: "qbsothermodule"; required: false } + Depends { name: "qbsmetatestmodule" } + } + } + +} diff --git a/tests/auto/blackbox/testdata-providers/broken-provider/module-providers/provider_a.qbs b/tests/auto/blackbox/testdata-providers/broken-provider/module-providers/provider_a.qbs new file mode 100644 index 000000000..f446d2d13 --- /dev/null +++ b/tests/auto/blackbox/testdata-providers/broken-provider/module-providers/provider_a.qbs @@ -0,0 +1,5 @@ +ModuleProvider { + relativeSearchPaths: { + throw "This provider is broken"; + } +} diff --git a/tests/auto/blackbox/testdata-providers/non-eager-provider/module-providers/provider_a.qbs b/tests/auto/blackbox/testdata-providers/non-eager-provider/module-providers/provider_a.qbs new file mode 100644 index 000000000..6cd9177db --- /dev/null +++ b/tests/auto/blackbox/testdata-providers/non-eager-provider/module-providers/provider_a.qbs @@ -0,0 +1,11 @@ +import "../../qbs-module-providers-helpers.js" as Helpers + +ModuleProvider { + isEager: false + relativeSearchPaths: { + if (moduleName === "nonexistentmodule") + return undefined; + Helpers.writeModule(outputBaseDir, moduleName, "from_provider_a"); + return ""; + } +} diff --git a/tests/auto/blackbox/testdata-providers/non-eager-provider/non-eager-provider.qbs b/tests/auto/blackbox/testdata-providers/non-eager-provider/non-eager-provider.qbs new file mode 100644 index 000000000..bd3662de3 --- /dev/null +++ b/tests/auto/blackbox/testdata-providers/non-eager-provider/non-eager-provider.qbs @@ -0,0 +1,13 @@ +Project { + Product { + name: "p1" + Depends { name: "qbsmetatestmodule" } + Depends { name: "qbsothermodule" } + Depends { name: "nonexistentmodule"; required: false } + property bool dummy: { + console.info("p1.qbsmetatestmodule.prop: " + qbsmetatestmodule.prop); + console.info("p1.qbsothermodule.prop: " + qbsothermodule.prop); + } + qbsModuleProviders: "provider_a" + } +} diff --git a/tests/auto/blackbox/testdata-providers/removal-version/module-providers/provider_a.qbs b/tests/auto/blackbox/testdata-providers/removal-version/module-providers/provider_a.qbs new file mode 100644 index 000000000..7f358acbc --- /dev/null +++ b/tests/auto/blackbox/testdata-providers/removal-version/module-providers/provider_a.qbs @@ -0,0 +1,14 @@ +import "../../qbs-module-providers-helpers.js" as Helpers + +ModuleProvider { + isEager: false + property bool deprecated: false + PropertyOptions { + name: "deprecated" + removalVersion: "2.2.0" + } + relativeSearchPaths: { + Helpers.writeModule(outputBaseDir, moduleName, "from_provider_a"); + return ""; + } +} diff --git a/tests/auto/blackbox/testdata-providers/removal-version/removal-version.qbs b/tests/auto/blackbox/testdata-providers/removal-version/removal-version.qbs new file mode 100644 index 000000000..1aa5e2ce9 --- /dev/null +++ b/tests/auto/blackbox/testdata-providers/removal-version/removal-version.qbs @@ -0,0 +1,12 @@ +Project { + qbsModuleProviders: "provider_a" + name: "project" + Project { + name: "innerProject" + Product { + name: "p1" + Depends { name: "qbsmetatestmodule" } + } + } + +} diff --git a/tests/auto/blackbox/testdata-qt/qml-debugging/qml-debugging.qbs b/tests/auto/blackbox/testdata-qt/qml-debugging/qml-debugging.qbs index 8176a7c3e..0885e6b0b 100644 --- a/tests/auto/blackbox/testdata-qt/qml-debugging/qml-debugging.qbs +++ b/tests/auto/blackbox/testdata-qt/qml-debugging/qml-debugging.qbs @@ -4,4 +4,9 @@ QtApplication { Depends { name: "Qt.quick" } Qt.quick.qmlDebugging: true files: "main.cpp" + Probe { + id: checker + property bool isGcc: qbs.toolchain.contains("gcc") + configure: { console.info("is gcc: " + isGcc); } + } } diff --git a/tests/auto/blackbox/testdata/escaped-linker-flags/escaped-linker-flags.qbs b/tests/auto/blackbox/testdata/escaped-linker-flags/escaped-linker-flags.qbs index f48bf8d1f..b4fb0df4d 100644 --- a/tests/auto/blackbox/testdata/escaped-linker-flags/escaped-linker-flags.qbs +++ b/tests/auto/blackbox/testdata/escaped-linker-flags/escaped-linker-flags.qbs @@ -10,4 +10,9 @@ CppApplication { cpp.linkerFlags: ["-s"] } files: ["main.cpp"] + Probe { + id: checker + property bool isUnixGcc: qbs.toolchain.contains("gcc") && !qbs.targetOS.contains("macos") + configure: { console.info("is gcc: " + isUnixGcc); } + } } diff --git a/tests/auto/blackbox/testdata/linker-library-duplicates/setup-run-environment.qbs b/tests/auto/blackbox/testdata/linker-library-duplicates/setup-run-environment.qbs index e69cde064..c41e8f1d7 100644 --- a/tests/auto/blackbox/testdata/linker-library-duplicates/setup-run-environment.qbs +++ b/tests/auto/blackbox/testdata/linker-library-duplicates/setup-run-environment.qbs @@ -6,6 +6,11 @@ Project { files: ["lib1.cpp"] Depends { name: "bundle" } bundle.isBundle: false + Probe { + id: checker + property bool isGcc: qbs.toolchain.contains("gcc") + configure: { console.info("is gcc: " + isGcc); } + } } DynamicLibrary { diff --git a/tests/auto/blackbox/testdata/linkerscripts/linkerscripts.qbs b/tests/auto/blackbox/testdata/linkerscripts/linkerscripts.qbs index 0b4de0ab9..6d068b6a2 100644 --- a/tests/auto/blackbox/testdata/linkerscripts/linkerscripts.qbs +++ b/tests/auto/blackbox/testdata/linkerscripts/linkerscripts.qbs @@ -55,6 +55,13 @@ DynamicLibrary { } } + Probe { + id: checker + property bool isGcc: qbs.toolchain.contains("gcc") + property bool isLinux: qbs.targetOS.contains("linux") + configure: { console.info("is Linux gcc: " + (isGcc && isLinux)) } + } + qbs.installPrefix: "" install: true installDir: "" diff --git a/tests/auto/blackbox/testdata/reproducible-build/reproducible-build.qbs b/tests/auto/blackbox/testdata/reproducible-build/reproducible-build.qbs index f7ed8e61a..fabdf48db 100644 --- a/tests/auto/blackbox/testdata/reproducible-build/reproducible-build.qbs +++ b/tests/auto/blackbox/testdata/reproducible-build/reproducible-build.qbs @@ -2,4 +2,9 @@ CppApplication { name: "the product" files: ["file1.cpp", "file2.cpp", "main.cpp"] cpp.cxxFlags: ["-flto"] + Probe { + id: checker + property bool isGcc: qbs.toolchain.contains("gcc") && !qbs.toolchain.contains("clang") + configure: { console.info("is gcc: " + isGcc); } + } } diff --git a/tests/auto/blackbox/testdata/separate-debug-info/separate-debug-info.qbs b/tests/auto/blackbox/testdata/separate-debug-info/separate-debug-info.qbs index 9f57b7f01..4198b863f 100644 --- a/tests/auto/blackbox/testdata/separate-debug-info/separate-debug-info.qbs +++ b/tests/auto/blackbox/testdata/separate-debug-info/separate-debug-info.qbs @@ -8,10 +8,13 @@ Project { Probe { id: osProbe property stringList targetOS: qbs.targetOS + property stringList toolchain: qbs.toolchain configure: { console.info("is windows: " + (targetOS.includes("windows") ? "yes" : "no")); console.info("is macos: " + (targetOS.includes("macos") ? "yes" : "no")); console.info("is darwin: " + (targetOS.includes("darwin") ? "yes" : "no")); + console.info("is gcc: " + (toolchain.includes("gcc") ? "yes" : "no")); + console.info("is msvc: " + (toolchain.includes("msvc") ? "yes" : "no")); } } } diff --git a/tests/auto/blackbox/testdata/system-run-paths/system-run-paths.qbs b/tests/auto/blackbox/testdata/system-run-paths/system-run-paths.qbs index 6e2137173..35dd7a00f 100644 --- a/tests/auto/blackbox/testdata/system-run-paths/system-run-paths.qbs +++ b/tests/auto/blackbox/testdata/system-run-paths/system-run-paths.qbs @@ -20,4 +20,9 @@ Project { cpp.rpaths: qbs.installRoot + "/lib" cpp.systemRunPaths: project.setRunPaths ? [qbs.installRoot + "/lib"] : [] } + Probe { + id: checker + property bool isUnix: qbs.targetOS.contains("unix") + configure: { console.info("is unix: " + isUnix); } + } } diff --git a/tests/auto/blackbox/testdata/trackExternalProductChanges/trackExternalProductChanges.qbs b/tests/auto/blackbox/testdata/trackExternalProductChanges/trackExternalProductChanges.qbs index 534f49ff2..0e28e5687 100644 --- a/tests/auto/blackbox/testdata/trackExternalProductChanges/trackExternalProductChanges.qbs +++ b/tests/auto/blackbox/testdata/trackExternalProductChanges/trackExternalProductChanges.qbs @@ -11,4 +11,9 @@ CppApplication { name: "file that needs help from the environment to find a header" files: "including.cpp" } + Probe { + id: checker + property bool isGcc: qbs.toolchain.contains("gcc") + configure: { console.info("is gcc: " + isGcc); } + } } diff --git a/tests/auto/blackbox/testdata/versionscript/versionscript.qbs b/tests/auto/blackbox/testdata/versionscript/versionscript.qbs index cc5c7b1cc..fcb4314a2 100644 --- a/tests/auto/blackbox/testdata/versionscript/versionscript.qbs +++ b/tests/auto/blackbox/testdata/versionscript/versionscript.qbs @@ -20,6 +20,12 @@ DynamicLibrary { return [cmd]; } } + Probe { + id: checker + property bool isLinux: qbs.targetOS.includes("linux") + property bool isGcc: qbs.toolchain.contains("gcc") + configure: { console.info("is gcc for Linux: " + (isLinux && isGcc)); } + } qbs.installPrefix: "" install: true diff --git a/tests/auto/blackbox/tst_blackbox.cpp b/tests/auto/blackbox/tst_blackbox.cpp index ed76636e8..2befc8db8 100644 --- a/tests/auto/blackbox/tst_blackbox.cpp +++ b/tests/auto/blackbox/tst_blackbox.cpp @@ -63,7 +63,6 @@ #define WAIT_FOR_NEW_TIMESTAMP() waitForNewTimestamp(testDataDir) using qbs::Internal::HostOsInfo; -using qbs::Profile; class MacosTarHealer { public: @@ -139,7 +138,7 @@ QString TestBlackbox::findArchiver(const QString &fileName, int *status) QString binary = findExecutable(QStringList(fileName)); if (binary.isEmpty()) { const SettingsPtr s = settings(); - Profile p(profileName(), s.get()); + qbs::Profile p(profileName(), s.get()); binary = findExecutable(p.value("archiver.command").toStringList()); } return binary; @@ -299,7 +298,7 @@ void TestBlackbox::textTemplate() static QStringList sortedFileList(const QByteArray &ba) { - auto list = QString::fromUtf8(ba).split(QRegularExpression("[\r\n]"), QBS_SKIP_EMPTY_PARTS); + auto list = QString::fromUtf8(ba).split(QRegularExpression("[\r\n]"), Qt::SkipEmptyParts); std::sort(list.begin(), list.end()); return list; } @@ -699,7 +698,7 @@ void TestBlackbox::buildDirectories() QDir::setCurrent(projectDir); QCOMPARE(runQbs(), 0); const QStringList outputLines - = QString::fromLocal8Bit(m_qbsStdout.trimmed()).split('\n', QBS_SKIP_EMPTY_PARTS); + = QString::fromLocal8Bit(m_qbsStdout.trimmed()).split('\n', Qt::SkipEmptyParts); QVERIFY2(outputLines.contains(projectDir + '/' + relativeProductBuildDir("p1")), m_qbsStdout.constData()); QVERIFY2(outputLines.contains(projectDir + '/' + relativeProductBuildDir("p2")), @@ -1594,14 +1593,14 @@ void TestBlackbox::versionCheck_data() void TestBlackbox::versionScript() { - const SettingsPtr s = settings(); - Profile buildProfile(profileName(), s.get()); - QStringList toolchain = profileToolchain(buildProfile); - if (!toolchain.contains("gcc") || targetOs() != HostOsInfo::HostOsLinux) - QSKIP("version script test only applies to Linux"); QDir::setCurrent(testDataDir + "/versionscript"); - QCOMPARE(runQbs(QbsRunParameters(QStringList("-q") - << ("qbs.installRoot:" + QDir::currentPath()))), 0); + QCOMPARE(runQbs(QbsRunParameters("resolve", {"qbs.installRoot:" + QDir::currentPath()})), 0); + const bool isLinuxGcc = m_qbsStdout.contains("is gcc for Linux: true"); + const bool isNotLinuxGcc = m_qbsStdout.contains("is gcc for Linux: false"); + if (isNotLinuxGcc) + QSKIP("version script test only applies to Linux"); + QVERIFY(isLinuxGcc); + QCOMPARE(runQbs(QbsRunParameters(QStringList("-q"))), 0); const QString output = QString::fromLocal8Bit(m_qbsStderr); const QRegularExpression pattern(QRegularExpression::anchoredPattern(".*---(.*)---.*"), QRegularExpression::DotMatchesEverythingOption); @@ -2068,10 +2067,13 @@ void TestBlackbox::separateDebugInfo() const bool isDarwin = m_qbsStdout.contains("is darwin: yes"); const bool isNotDarwin = m_qbsStdout.contains("is darwin: no"); QVERIFY(isDarwin != isNotDarwin); + const bool isGcc = m_qbsStdout.contains("is gcc: yes"); + const bool isNotGcc = m_qbsStdout.contains("is gcc: no"); + QVERIFY(isGcc != isNotGcc); + const bool isMsvc = m_qbsStdout.contains("is msvc: yes"); + const bool isNotMsvc = m_qbsStdout.contains("is msvc: no"); + QVERIFY(isMsvc != isNotMsvc); - const SettingsPtr s = settings(); - Profile buildProfile(profileName(), s.get()); - QStringList toolchain = profileToolchain(buildProfile); if (isDarwin) { QVERIFY(directoryExists(relativeProductBuildDir("app1") + "/app1.app.dSYM")); QVERIFY(regularFileExists(relativeProductBuildDir("app1") @@ -2151,7 +2153,7 @@ void TestBlackbox::separateDebugInfo() + "/bar4.bundle.dSYM/Contents/Resources/DWARF") .entryInfoList(QDir::NoDotAndDotDot | QDir::AllEntries).size(), 1); QVERIFY(regularFileExists(relativeProductBuildDir("bar5") + "/bar5.bundle.dwarf")); - } else if (toolchain.contains("gcc")) { + } else if (isGcc) { const QString exeSuffix = isWindows ? ".exe" : ""; const QString dllPrefix = isWindows ? "" : "lib"; const QString dllSuffix = isWindows ? ".dll" : ".so"; @@ -2165,7 +2167,7 @@ void TestBlackbox::separateDebugInfo() + '/' + dllPrefix + "bar1" + dllSuffix + ".debug")); QVERIFY(!QFile::exists(relativeProductBuildDir("bar2") + '/' + dllPrefix + "bar2" + dllSuffix + ".debug")); - } else if (toolchain.contains("msvc")) { + } else if (isMsvc) { QVERIFY(QFile::exists(relativeProductBuildDir("app1") + "/app1.pdb")); QVERIFY(QFile::exists(relativeProductBuildDir("foo1") + "/foo1.pdb")); QVERIFY(QFile::exists(relativeProductBuildDir("bar1") + "/bar1.pdb")); @@ -2222,6 +2224,9 @@ void TestBlackbox::trackExternalProductChanges() QVERIFY(!m_qbsStdout.contains("compiling jsFileChange.cpp")); QVERIFY(!m_qbsStdout.contains("compiling fileExists.cpp")); + const bool isGcc = m_qbsStdout.contains("is gcc: true"); + const bool isNotGcc = m_qbsStdout.contains("is gcc: false"); + QbsRunParameters params; params.environment.insert("QBS_TEST_PULL_IN_FILE_VIA_ENV", "1"); QCOMPARE(runQbs(params), 0); @@ -2271,12 +2276,11 @@ void TestBlackbox::trackExternalProductChanges() QVERIFY(!m_qbsStdout.contains("compiling jsFileChange.cpp")); QVERIFY(m_qbsStdout.contains("compiling fileExists.cpp")); + if (isNotGcc) + QSKIP("The remainder of this test requires a GCC-like toolchain"); + QVERIFY(isGcc); + rmDirR(relativeBuildDir()); - const SettingsPtr s = settings(); - const Profile profile(profileName(), s.get()); - const QStringList toolchainTypes = profileToolchain(profile); - if (!toolchainTypes.contains("gcc")) - QSKIP("Need GCC-like compiler to run this test"); params.environment = QbsRunParameters::defaultEnvironment(); params.environment.insert("INCLUDE_PATH_TEST", "1"); params.expectFailure = true; @@ -2569,22 +2573,21 @@ void TestBlackbox::removeDuplicateLibraries() void TestBlackbox::reproducibleBuild() { - const SettingsPtr s = settings(); - const Profile profile(profileName(), s.get()); - const QStringList toolchains = profileToolchain(profile); - if (!toolchains.contains("gcc")) - QSKIP("reproducible builds only supported for gcc"); - if (toolchains.contains("clang")) - QSKIP("reproducible builds are not supported for clang"); - QFETCH(bool, reproducible); QDir::setCurrent(testDataDir + "/reproducible-build"); - QbsRunParameters params; + QbsRunParameters params("resolve"); params.arguments << QString("modules.cpp.enableReproducibleBuilds:") + (reproducible ? "true" : "false"); rmDirR(relativeBuildDir()); QCOMPARE(runQbs(params), 0); + const bool isGcc = m_qbsStdout.contains("is gcc: true"); + const bool isNotGcc = m_qbsStdout.contains("is gcc: false"); + if (isNotGcc) + QSKIP("reproducible builds only supported for gcc"); + QVERIFY(isGcc); + + QCOMPARE(runQbs(), 0); QFile object(relativeProductBuildDir("the product") + '/' + inputDirHash(".") + '/' + objectFileName("file1.cpp", profileName())); QVERIFY2(object.open(QIODevice::ReadOnly), qPrintable(object.fileName())); @@ -2592,6 +2595,7 @@ void TestBlackbox::reproducibleBuild() object.close(); QCOMPARE(runQbs(QbsRunParameters("clean")), 0); QVERIFY(!object.exists()); + params.command = "build"; QCOMPARE(runQbs(params), 0); if (reproducible) { QVERIFY(object.open(QIODevice::ReadOnly)); @@ -3802,7 +3806,7 @@ void TestBlackbox::emptyProfile() QDir::setCurrent(testDataDir + "/empty-profile"); const SettingsPtr s = settings(); - const Profile buildProfile(profileName(), s.get()); + const qbs::Profile buildProfile(profileName(), s.get()); bool isMsvc = false; auto toolchainType = buildProfile.value(QStringLiteral("qbs.toolchainType")).toString(); QbsRunParameters params; @@ -3823,7 +3827,7 @@ void TestBlackbox::emptyProfile() QDir::toNativeSeparators( buildProfile.value(QStringLiteral("cpp.toolchainInstallPath")).toString()); auto paths = params.environment.value(QStringLiteral("PATH")) - .split(HostOsInfo::pathListSeparator(), QBS_SKIP_EMPTY_PARTS); + .split(HostOsInfo::pathListSeparator(), Qt::SkipEmptyParts); if (!tcPath.isEmpty() && !paths.contains(tcPath)) { paths.prepend(tcPath); params.environment.insert( @@ -3923,15 +3927,16 @@ void TestBlackbox::errorInfo() void TestBlackbox::escapedLinkerFlags() { - const SettingsPtr s = settings(); - const Profile buildProfile(profileName(), s.get()); - const QStringList toolchain = profileToolchain(buildProfile); - if (!toolchain.contains("gcc")) - QSKIP("escaped linker flags test only applies with gcc and GNU ld"); - if (targetOs() == HostOsInfo::HostOsMacos) - QSKIP("Does not apply on macOS"); QDir::setCurrent(testDataDir + "/escaped-linker-flags"); - QbsRunParameters params(QStringList("products.app.escapeLinkerFlags:false")); + QbsRunParameters params("resolve", QStringList("products.app.escapeLinkerFlags:false")); + QCOMPARE(runQbs(params), 0); + const bool isGcc = m_qbsStdout.contains("is gcc: true"); + const bool isNotGcc = m_qbsStdout.contains("is gcc: false"); + if (isNotGcc) + QSKIP("escaped linker flags test only applies on plain unix with gcc and GNU ld"); + QVERIFY(isGcc); + + params.command = "build"; QCOMPARE(runQbs(params), 0); params.command = "resolve"; params.arguments = QStringList() << "products.app.escapeLinkerFlags:true"; @@ -3988,26 +3993,24 @@ void TestBlackbox::exportedPropertyInDisabledProduct_data() void TestBlackbox::systemRunPaths() { - const SettingsPtr s = settings(); - const Profile buildProfile(profileName(), s.get()); - switch (targetOs()) { - case HostOsInfo::HostOsLinux: - case HostOsInfo::HostOsMacos: - case HostOsInfo::HostOsOtherUnix: - break; - default: - QSKIP("only applies on Unix"); - } - const QString lddFilePath = findExecutable(QStringList() << "ldd"); if (lddFilePath.isEmpty()) QSKIP("ldd not found"); + QDir::setCurrent(testDataDir + "/system-run-paths"); QFETCH(bool, setRunPaths); rmDirR(relativeBuildDir()); - QbsRunParameters params; + QbsRunParameters params("resolve"); params.arguments << QString("project.setRunPaths:") + (setRunPaths ? "true" : "false"); QCOMPARE(runQbs(params), 0); + const bool isUnix = m_qbsStdout.contains("is unix: true"); + const bool isNotUnix = m_qbsStdout.contains("is unix: false"); + if (isNotUnix) + QSKIP("only applies on Unix"); + QVERIFY(isUnix); + + params.command = "build"; + QCOMPARE(runQbs(params), 0); QProcess ldd; ldd.start(lddFilePath, QStringList() << relativeExecutableFilePath("app")); QVERIFY2(ldd.waitForStarted(), qPrintable(ldd.errorString())); @@ -4115,7 +4118,8 @@ void TestBlackbox::exportsQbs() paramsExternalBuild.buildDirectory = QDir::currentPath() + "/external-consumer-profile"; paramsExternalBuild.expectFailure = true; QVERIFY(runQbs(paramsExternalBuild) != 0); - QVERIFY2(m_qbsStderr.contains("MyLib could not be loaded"), m_qbsStderr.constData()); + QVERIFY2(m_qbsStderr.contains("Dependency 'MyLib' not found for product 'consumer'"), + m_qbsStderr.constData()); // Removing the condition from the generated module leaves us with two conflicting // candidates. @@ -4564,7 +4568,7 @@ void TestBlackbox::cli() QCOMPARE(status, 0); const SettingsPtr s = settings(); - Profile p("qbs_autotests-cli", s.get()); + qbs::Profile p("qbs_autotests-cli", s.get()); const QStringList toolchain = profileToolchain(p); if (!p.exists() || !(toolchain.contains("dotnet") || toolchain.contains("mono"))) QSKIP("No suitable Common Language Infrastructure test profile"); @@ -4844,7 +4848,8 @@ void TestBlackbox::lastModuleCandidateBroken() QbsRunParameters params; params.expectFailure = true; QVERIFY(runQbs(params) != 0); - QVERIFY2(m_qbsStderr.contains("Module Foo could not be loaded"), m_qbsStderr); + QVERIFY2(m_qbsStderr.contains("Dependency 'Foo' not found for product " + "'last-module-candidate-broken'"), m_qbsStderr); } void TestBlackbox::ld() @@ -5083,22 +5088,20 @@ void TestBlackbox::lexyaccOutputs_data() void TestBlackbox::linkerLibraryDuplicates() { - const SettingsPtr s = settings(); - Profile buildProfile(profileName(), s.get()); - QStringList toolchain = profileToolchain(buildProfile); - if (!toolchain.contains("gcc")) - QSKIP("linkerLibraryDuplicates test only applies to GCC toolchain"); - QDir::setCurrent(testDataDir + "/linker-library-duplicates"); rmDirR(relativeBuildDir()); - QFETCH(QString, removeDuplicateLibraries); QStringList runParams; - if (!removeDuplicateLibraries.isEmpty()) { + if (!removeDuplicateLibraries.isEmpty()) runParams.append(removeDuplicateLibraries); - } QCOMPARE(runQbs(QbsRunParameters("resolve", runParams)), 0); + const bool isGcc = m_qbsStdout.contains("is gcc: true"); + const bool isNotGcc = m_qbsStdout.contains("is gcc: false"); + if (isNotGcc) + QSKIP("linkerLibraryDuplicates test only applies to GCC toolchain"); + QVERIFY(isGcc); + QCOMPARE(runQbs(QStringList { "--command-echo-mode", "command-line" }), 0); const QByteArrayList output = m_qbsStdout.split('\n'); QByteArray linkLine; @@ -5163,20 +5166,20 @@ void TestBlackbox::linkerLibraryDuplicates_data() void TestBlackbox::linkerScripts() { - const SettingsPtr s = settings(); - Profile buildProfile(profileName(), s.get()); - QStringList toolchain = profileToolchain(buildProfile); - if (!toolchain.contains("gcc") || targetOs() != HostOsInfo::HostOsLinux) - QSKIP("linker script test only applies to Linux "); - - QbsRunParameters runParams(QStringList() -// << "--log-level" << "debug" - << ("qbs.installRoot:" + QDir::currentPath())); const QString sourceDir = QDir::cleanPath(testDataDir + "/linkerscripts"); + QbsRunParameters runParams("resolve", {"qbs.installRoot:" + QDir::currentPath()}); runParams.buildDirectory = sourceDir + "/build"; runParams.workingDir = sourceDir; QCOMPARE(runQbs(runParams), 0); + const bool isGcc = m_qbsStdout.contains("is Linux gcc: true"); + const bool isNotGcc = m_qbsStdout.contains("is Linux gcc: false"); + if (isNotGcc) + QSKIP("linker script test only applies to Linux"); + QVERIFY(isGcc); + + runParams.command = "build"; + QCOMPARE(runQbs(runParams), 0); const QString output = QString::fromLocal8Bit(m_qbsStderr); const QRegularExpression pattern(QRegularExpression::anchoredPattern(".*---(.*)---.*"), QRegularExpression::DotMatchesEverythingOption); @@ -6343,7 +6346,7 @@ void TestBlackbox::qbsSession() // Wait for and verify hello packet. QJsonObject receivedMessage = getNextSessionPacket(sessionProc, incomingData); QCOMPARE(receivedMessage.value("type"), "hello"); - QCOMPARE(receivedMessage.value("api-level").toInt(), 3); + QCOMPARE(receivedMessage.value("api-level").toInt(), 4); QCOMPARE(receivedMessage.value("api-compat-level").toInt(), 2); // Resolve & verify structure @@ -6359,6 +6362,7 @@ void TestBlackbox::qbsSession() resolveMessage.insert("overridden-properties", overriddenValues); resolveMessage.insert("environment", envToJson(QbsRunParameters::defaultEnvironment())); resolveMessage.insert("data-mode", "only-if-changed"); + resolveMessage.insert("max-job-count", 2); resolveMessage.insert("log-time", true); resolveMessage.insert("module-properties", QJsonArray::fromStringList({"cpp.cxxLanguageVersion"})); @@ -7432,7 +7436,7 @@ static bool haveMakeNsis() << QStringLiteral("HKEY_LOCAL_MACHINE\\SOFTWARE\\NSIS"); QStringList paths = QProcessEnvironment::systemEnvironment().value("PATH") - .split(HostOsInfo::pathListSeparator(), QBS_SKIP_EMPTY_PARTS); + .split(HostOsInfo::pathListSeparator(), Qt::SkipEmptyParts); for (const QString &key : std::as_const(regKeys)) { QSettings settings(key, QSettings::NativeFormat); @@ -7670,7 +7674,7 @@ void TestBlackbox::generator_data() void TestBlackbox::nodejs() { const SettingsPtr s = settings(); - Profile p(profileName(), s.get()); + qbs::Profile p(profileName(), s.get()); int status; findNodejs(&status); @@ -7711,7 +7715,7 @@ void TestBlackbox::typescript() QSKIP("Skip this test when running on GitHub"); const SettingsPtr s = settings(); - Profile p(profileName(), s.get()); + qbs::Profile p(profileName(), s.get()); int status; findTypeScript(&status); diff --git a/tests/auto/blackbox/tst_blackboxapple.cpp b/tests/auto/blackbox/tst_blackboxapple.cpp index fdb656c72..02b56f603 100644 --- a/tests/auto/blackbox/tst_blackboxapple.cpp +++ b/tests/auto/blackbox/tst_blackboxapple.cpp @@ -1131,7 +1131,7 @@ void TestBlackboxApple::xcode() QVERIFY2(xcodebuildShowSdks.exitCode() == 0, qPrintable(xcodebuildShowSdks.readAllStandardError().constData())); const auto lines = QString::fromLocal8Bit(xcodebuildShowSdks.readAllStandardOutput().trimmed()) - .split('\n', QBS_SKIP_EMPTY_PARTS); + .split('\n', Qt::SkipEmptyParts); for (const QString &line : lines) { static const std::regex regexp("^.+\\s+\\-sdk\\s+([a-z]+)([0-9]+\\.[0-9]+)$"); const auto ln = line.toStdString(); diff --git a/tests/auto/blackbox/tst_blackboxbase.cpp b/tests/auto/blackbox/tst_blackboxbase.cpp index 8e85640db..17652e779 100644 --- a/tests/auto/blackbox/tst_blackboxbase.cpp +++ b/tests/auto/blackbox/tst_blackboxbase.cpp @@ -221,7 +221,7 @@ void TestBlackboxBase::validateTestProfile() QString TestBlackboxBase::findExecutable(const QStringList &fileNames) { const QStringList path = QString::fromLocal8Bit(qgetenv("PATH")) - .split(HostOsInfo::pathListSeparator(), QBS_SKIP_EMPTY_PARTS); + .split(HostOsInfo::pathListSeparator(), Qt::SkipEmptyParts); for (const QString &fileName : fileNames) { QFileInfo fi(fileName); diff --git a/tests/auto/blackbox/tst_blackboxproviders.cpp b/tests/auto/blackbox/tst_blackboxproviders.cpp index 9c408919c..38bc42672 100644 --- a/tests/auto/blackbox/tst_blackboxproviders.cpp +++ b/tests/auto/blackbox/tst_blackboxproviders.cpp @@ -49,6 +49,18 @@ TestBlackboxProviders::TestBlackboxProviders() { } +void TestBlackboxProviders::brokenProvider() +{ + QDir::setCurrent(testDataDir + "/broken-provider"); + QbsRunParameters params; + params.expectFailure = true; + QVERIFY(runQbs(params) != 0); + + QVERIFY(m_qbsStderr.contains("Error executing provider for module 'qbsothermodule'")); + QVERIFY(m_qbsStderr.contains("Error executing provider for module 'qbsmetatestmodule'")); + QCOMPARE(m_qbsStderr.count("This provider is broken"), 2); +} + void TestBlackboxProviders::fallbackModuleProvider_data() { QTest::addColumn<bool>("fallbacksEnabledGlobally"); @@ -179,6 +191,22 @@ void TestBlackboxProviders::moduleProvidersCache() QCOMPARE(m_qbsStderr.count("Re-using provider \"provider_a\" from cache"), 2); } +void TestBlackboxProviders::nonEagerModuleProvider() +{ + QDir::setCurrent(testDataDir + "/non-eager-provider"); + + QbsRunParameters params("resolve"); + QCOMPARE(runQbs(params), 0); + QVERIFY2(m_qbsStdout.contains(("Running setup script for qbsmetatestmodule")), m_qbsStdout); + QVERIFY2(m_qbsStdout.contains(("Running setup script for qbsothermodule")), m_qbsStdout); + QVERIFY2(!m_qbsStdout.contains(("Running setup script for nonexistentmodule")), m_qbsStdout); + + QVERIFY2(m_qbsStdout.contains(("p1.qbsmetatestmodule.prop: from_provider_a")), + m_qbsStdout); + QVERIFY2(m_qbsStdout.contains(("p1.qbsothermodule.prop: from_provider_a")), + m_qbsStdout); +} + void TestBlackboxProviders::probeInModuleProvider() { QDir::setCurrent(testDataDir + "/probe-in-module-provider"); @@ -366,4 +394,12 @@ void TestBlackboxProviders::qbspkgconfigModuleProvider() QCOMPARE(runQbs(params), 0); } +void TestBlackboxProviders::removalVersion() +{ + QDir::setCurrent(testDataDir + "/removal-version"); + QCOMPARE(runQbs(), 0); + QVERIFY(m_qbsStderr.contains( + "Property 'deprecated' was scheduled for removal in version 2.2.0, but is still present")); +} + QTEST_MAIN(TestBlackboxProviders) diff --git a/tests/auto/blackbox/tst_blackboxproviders.h b/tests/auto/blackbox/tst_blackboxproviders.h index 017cc1c5e..413df682d 100644 --- a/tests/auto/blackbox/tst_blackboxproviders.h +++ b/tests/auto/blackbox/tst_blackboxproviders.h @@ -41,10 +41,12 @@ public: TestBlackboxProviders(); private slots: + void brokenProvider(); void fallbackModuleProvider_data(); void fallbackModuleProvider(); void moduleProviders(); void moduleProvidersCache(); + void nonEagerModuleProvider(); void probeInModuleProvider(); void providersProperties(); void qbsModulePropertiesInProviders(); @@ -55,6 +57,7 @@ private slots: void qbsModuleProvidersCompatibility(); void qbsModuleProvidersCompatibility_data(); void qbspkgconfigModuleProvider(); + void removalVersion(); }; #endif // TST_BLACKBOXPROVIDERS_H diff --git a/tests/auto/blackbox/tst_blackboxqt.cpp b/tests/auto/blackbox/tst_blackboxqt.cpp index ef08f8ee9..d0f0a3831 100644 --- a/tests/auto/blackbox/tst_blackboxqt.cpp +++ b/tests/auto/blackbox/tst_blackboxqt.cpp @@ -38,7 +38,6 @@ #define WAIT_FOR_NEW_TIMESTAMP() waitForNewTimestamp(testDataDir) using qbs::Internal::HostOsInfo; -using qbs::Profile; TestBlackboxQt::TestBlackboxQt() : TestBlackboxBase (SRCDIR "/testdata-qt", "blackbox-qt") { @@ -240,7 +239,7 @@ void TestBlackboxQt::mixedBuildVariants() { QDir::setCurrent(testDataDir + "/mixed-build-variants"); const SettingsPtr s = settings(); - Profile profile(profileName(), s.get()); + qbs::Profile profile(profileName(), s.get()); if (profileToolchain(profile).contains("msvc")) { QbsRunParameters params; params.arguments << "qbs.buildVariant:debug"; @@ -484,18 +483,22 @@ void TestBlackboxQt::qmlDebugging() { QDir::setCurrent(testDataDir + "/qml-debugging"); QCOMPARE(runQbs(), 0); - const SettingsPtr s = settings(); - Profile profile(profileName(), s.get()); - if (!profileToolchain(profile).contains("gcc")) - return; + + const bool isGcc = m_qbsStdout.contains("is gcc: true"); + const bool isNotGcc = m_qbsStdout.contains("is gcc: false"); + if (isNotGcc) + QSKIP("The remainder of this test only applies to gcc"); + QVERIFY(isGcc); + QProcess nm; nm.start("nm", QStringList(relativeExecutableFilePath("debuggable-app"))); - if (nm.waitForStarted()) { // Let's ignore hosts without nm. - QVERIFY2(nm.waitForFinished(), qPrintable(nm.errorString())); - QVERIFY2(nm.exitCode() == 0, nm.readAllStandardError().constData()); - const QByteArray output = nm.readAllStandardOutput(); - QVERIFY2(output.toLower().contains("debugginghelper"), output.constData()); - } + if (!nm.waitForStarted()) + QSKIP("The remainder of this test requires nm"); + + QVERIFY2(nm.waitForFinished(), qPrintable(nm.errorString())); + QVERIFY2(nm.exitCode() == 0, nm.readAllStandardError().constData()); + const QByteArray output = nm.readAllStandardOutput(); + QVERIFY2(output.toLower().contains("debugginghelper"), output.constData()); } void TestBlackboxQt::qobjectInObjectiveCpp() diff --git a/tests/auto/blackbox/tst_blackboxwindows.cpp b/tests/auto/blackbox/tst_blackboxwindows.cpp index 1c487bc52..57bd7f947 100644 --- a/tests/auto/blackbox/tst_blackboxwindows.cpp +++ b/tests/auto/blackbox/tst_blackboxwindows.cpp @@ -228,7 +228,7 @@ static bool haveWiX(const Profile &profile) << QStringLiteral("HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Installer XML\\"); QStringList paths = QProcessEnvironment::systemEnvironment().value("PATH") - .split(HostOsInfo::pathListSeparator(), QBS_SKIP_EMPTY_PARTS); + .split(HostOsInfo::pathListSeparator(), Qt::SkipEmptyParts); for (const QString &key : std::as_const(regKeys)) { const QStringList versions = QSettings(key, QSettings::NativeFormat).childGroups(); diff --git a/tests/auto/cmdlineparser/tst_cmdlineparser.cpp b/tests/auto/cmdlineparser/tst_cmdlineparser.cpp index f617d41c7..6a8b44f89 100644 --- a/tests/auto/cmdlineparser/tst_cmdlineparser.cpp +++ b/tests/auto/cmdlineparser/tst_cmdlineparser.cpp @@ -26,6 +26,8 @@ ** ****************************************************************************/ +#include "../shared.h" + #include <app/qbs/parser/commandlineparser.h> #include <app/shared/logging/consolelogger.h> #include <tools/buildoptions.h> @@ -55,6 +57,25 @@ private slots: m_fileArgs = QStringList() << "-f" << m_projectFile.fileName(); } + void testResolve_data() + { + QTest::addColumn<QStringList>("args"); + QTest::addColumn<int>("expectedJobCount"); + + QTest::newRow("default job count") << QStringList() << BuildOptions::defaultMaxJobCount(); + QTest::newRow("explicit job count") << QStringList("-j5") << 5; + } + void testResolve() + { + QFETCH(QStringList, args); + QFETCH(int, expectedJobCount); + + CommandLineParser parser; + QVERIFY(parser.parseCommandLine(QStringList("resolve") << args << m_fileArgs)); + QCOMPARE(parser.command(), ResolveCommandType); + QCOMPARE(parser.jobCount(profileName()), expectedJobCount); + } + void testValidCommandLine() { QStringList args; diff --git a/tests/auto/language/testdata/erroneous/dependency_cycle3a.qbs b/tests/auto/language/testdata/erroneous/dependency_cycle3a.qbs new file mode 100644 index 000000000..a0660c074 --- /dev/null +++ b/tests/auto/language/testdata/erroneous/dependency_cycle3a.qbs @@ -0,0 +1,11 @@ +Project { + Product { + name: "B" + Depends { productTypes: ["a"] } + } + Product { + type: ["a"] + name: "A" + Depends { name: "B" } + } +} diff --git a/tests/auto/language/testdata/erroneous/frozen-object-list.qbs b/tests/auto/language/testdata/erroneous/frozen-object-list.qbs new file mode 100644 index 000000000..8bbd2b413 --- /dev/null +++ b/tests/auto/language/testdata/erroneous/frozen-object-list.qbs @@ -0,0 +1,17 @@ + +Product { + Probe { + id: probe + property varList output + configure: { + output = [{"key": "value"}]; + found = true; + } + } + + property var test: { + var result = probe.output; + result.push({}); + return result; + } +} diff --git a/tests/auto/language/testdata/erroneous/frozen-object.qbs b/tests/auto/language/testdata/erroneous/frozen-object.qbs new file mode 100644 index 000000000..0f891d04b --- /dev/null +++ b/tests/auto/language/testdata/erroneous/frozen-object.qbs @@ -0,0 +1,18 @@ + +Product { + Probe { + id: probe + property var output + configure: { + output = {"key": "value"} + found = true + } + } + + property var test: { + "use strict" + var result = probe.output; + result.key = "newValue"; + return result; + } +} diff --git a/tests/auto/language/testdata/local-profile-as-top-level-profile.qbs b/tests/auto/language/testdata/local-profile-as-top-level-profile.qbs new file mode 100644 index 000000000..9bca3c599 --- /dev/null +++ b/tests/auto/language/testdata/local-profile-as-top-level-profile.qbs @@ -0,0 +1,7 @@ +Product { + Profile { + name: "test-profile" + qbs.architecture: "arm" + qbs.targetPlatform: "macos" + } +} diff --git a/tests/auto/language/tst_language.cpp b/tests/auto/language/tst_language.cpp index 554550299..d732b211c 100644 --- a/tests/auto/language/tst_language.cpp +++ b/tests/auto/language/tst_language.cpp @@ -86,6 +86,27 @@ static QString testProject(const char *fileName) { return testDataDir() + QLatin1Char('/') + QLatin1String(fileName); } +class JSSourceValueCreator +{ + FileContextPtr m_fileContext; + std::vector<std::unique_ptr<QString>> m_strings; +public: + JSSourceValueCreator(const FileContextPtr &fileContext) + : m_fileContext(fileContext) + { + } + + JSSourceValuePtr create(const QString &sourceCode) + { + JSSourceValuePtr value = JSSourceValue::create(); + value->setFile(m_fileContext); + auto str = std::make_unique<QString>(sourceCode); + value->setSourceCode(*str.get()); + m_strings.push_back(std::move(str)); + return value; + } +}; + TestLanguage::TestLanguage(ILogSink *logSink, Settings *settings) : m_logSink(logSink) , m_settings(settings) @@ -165,12 +186,16 @@ TopLevelProjectPtr TestLanguage::resolveProject(const char *relProjectFilePath) void TestLanguage::init() { + // clear caches, otherwise StoredVariantValues may end up being at the same address + // as the destroyed value + m_engine->reset(); m_logSink->setLogLevel(LoggerInfo); defaultParameters = {}; defaultParameters.setBuildRoot(m_tempDir.path() + "/buildroot"); defaultParameters.setPropertyCheckingMode(ErrorHandlingMode::Strict); defaultParameters.setSettingsDirectory(m_settings->baseDirectory()); defaultParameters.setTopLevelProfile(profileName()); + defaultParameters.setMaxJobCount(1); defaultParameters.setConfigurationName("default"); defaultParameters.setEnvironment(QProcessEnvironment::systemEnvironment()); defaultParameters.setSearchPaths({SRCDIR "/../../../share/qbs"}); @@ -497,6 +522,29 @@ void TestLanguage::conditionalDepends() QCOMPARE(exceptionCaught, false); } +void TestLanguage::convertStringList() +{ + FileContextPtr fileContext = FileContext::create(); + fileContext->setFilePath("/dev/null"); + JSSourceValueCreator sourceValueCreator(fileContext); + ItemPool pool; + Item *scope = Item::create(&pool, ItemType::Scope); + scope->setProperty("x", sourceValueCreator.create("[\"a\", \"b\"]")); + + Evaluator evaluator(m_engine.get()); + auto variantValue = evaluator.variantValue(scope, "x"); + // despite we have a stringList prop, we evaluate it as a QVariantList + QCOMPARE(variantValue.userType(), QMetaType::Type::QVariantList); + // and we have to convert it explicitly +#if (QT_VERSION >= QT_VERSION_CHECK(6, 0, 0)) + variantValue.convert(QMetaType(QMetaType::QStringList)); +#else + variantValue.convert(QMetaType::QStringList); +#endif + QCOMPARE(variantValue.userType(), QMetaType::Type::QStringList); + QCOMPARE(variantValue, QStringList({"a", "b"})); +} + void TestLanguage::delayedError() { QFETCH(bool, productEnabled); @@ -854,6 +902,8 @@ void TestLanguage::erroneousFiles_data() << "Cyclic dependencies detected."; QTest::newRow("dependency_cycle3") << "Cyclic dependencies detected."; + QTest::newRow("dependency_cycle3a") + << "Cyclic dependencies detected."; QTest::newRow("dependency_cycle4") << "Cyclic dependencies detected."; QTest::newRow("references_cycle") @@ -967,6 +1017,8 @@ void TestLanguage::erroneousFiles_data() << "invalid-references.qbs:2:17.*Cannot open '.*nosuchproject.qbs'"; QTest::newRow("missing-js-file") << "missing-js-file-module.qbs.*Cannot open '.*javascriptfile.js'"; + QTest::newRow("frozen-object") << "'key' is read-only"; + QTest::newRow("frozen-object-list") << "object is not extensible"; } void TestLanguage::erroneousFiles() @@ -1038,6 +1090,7 @@ void TestLanguage::exports() product = products.value("B"); QVERIFY(!!product); QVERIFY(product->dependencies.empty()); + QCOMPARE(product->exportedModule.productDependencies, std::vector<QString>{"C"}); product = products.value("C"); QVERIFY(!!product); QVERIFY(product->dependencies.empty()); @@ -1559,27 +1612,6 @@ void TestLanguage::invalidOverrides_data() << "products.MyOtherProduct.cpp.useRPaths" << QString(); } -class JSSourceValueCreator -{ - FileContextPtr m_fileContext; - std::vector<std::unique_ptr<QString>> m_strings; -public: - JSSourceValueCreator(const FileContextPtr &fileContext) - : m_fileContext(fileContext) - { - } - - JSSourceValuePtr create(const QString &sourceCode) - { - JSSourceValuePtr value = JSSourceValue::create(); - value->setFile(m_fileContext); - auto str = std::make_unique<QString>(sourceCode); - value->setSourceCode(*str.get()); - m_strings.push_back(std::move(str)); - return value; - } -}; - void TestLanguage::itemPrototype() { FileContextPtr fileContext = FileContext::create(); @@ -1672,6 +1704,24 @@ void TestLanguage::jsImportUsedInMultipleScopes() QVERIFY(!exceptionCaught); } +void TestLanguage::localProfileAsTopLevelProfile() +{ + bool exceptionCaught = false; + try { + defaultParameters.setTopLevelProfile("test-profile"); + resolveProject("local-profile-as-top-level-profile.qbs"); + QVERIFY(!!project); + QCOMPARE(int(project->products.size()), 1); + const PropertyMapConstPtr &props = project->products.front()->moduleProperties; + QCOMPARE(props->qbsPropertyValue("architecture"), "arm"); + QCOMPARE(props->qbsPropertyValue("targetPlatform"), "macos"); + } catch (const ErrorInfo &e) { + exceptionCaught = true; + qDebug() << e.toString(); + } + QCOMPARE(exceptionCaught, false); +} + void TestLanguage::moduleMergingVariantValues() { bool exceptionCaught = false; @@ -2789,6 +2839,7 @@ void TestLanguage::qbsPropertiesInProjectCondition() try { resolveProject("qbs-properties-in-project-condition.qbs"); QVERIFY(!!project); + QVERIFY(!project->enabled); const QHash<QString, ResolvedProductPtr> products = productsFromProject(project); QCOMPARE(products.size(), 0); } catch (const ErrorInfo &e) { diff --git a/tests/auto/language/tst_language.h b/tests/auto/language/tst_language.h index 708c79cce..c6695f7c9 100644 --- a/tests/auto/language/tst_language.h +++ b/tests/auto/language/tst_language.h @@ -71,6 +71,7 @@ private slots: void chainedProbes(); void canonicalArchitecture(); void conditionalDepends(); + void convertStringList(); void delayedError(); void delayedError_data(); void dependencyOnAllProfiles(); @@ -114,6 +115,7 @@ private slots: void jsExtensions(); void jsImportUsedInMultipleScopes_data(); void jsImportUsedInMultipleScopes(); + void localProfileAsTopLevelProfile(); void moduleMergingVariantValues(); void modulePrioritizationBySearchPath_data(); void modulePrioritizationBySearchPath(); diff --git a/tests/fuzzy-test/fuzzytester.cpp b/tests/fuzzy-test/fuzzytester.cpp index f38ad4736..e601e006f 100644 --- a/tests/fuzzy-test/fuzzytester.cpp +++ b/tests/fuzzy-test/fuzzytester.cpp @@ -151,12 +151,7 @@ QStringList FuzzyTester::findAllCommits(const QString &startCommit) QString allCommitsString; runGit(QStringList() << "log" << (startCommit + "~1.." + m_headCommit) << "--format=format:%h", &allCommitsString); - return allCommitsString.simplified().split(QLatin1Char(' '), -#if (QT_VERSION < QT_VERSION_CHECK(5, 14, 0)) - QString::SkipEmptyParts); -#else - Qt::SkipEmptyParts); -#endif + return allCommitsString.simplified().split(QLatin1Char(' '), Qt::SkipEmptyParts); } QString FuzzyTester::findWorkingStartCommit(const QString &startCommit) |