Refactor the GCC/Clang/MinGW compilers auto-detection

Right now the qbs-setup-toolchains can auto detect all installed
compiler versions from the directories specified in the PATH
environment variable.

Also it tries to ignore all duplicate compiler executables
found in same directory.

F.e. if some directory contains two compiler names
'arm-none-eabi-gcc-9.1.0' and 'arm-none-eabi-gcc', and they
are equal, then the compiler 'arm-none-eabi-gcc-9.1.0' with a
longer name will be chose.

Change-Id: I395dcbf79f4fc8d4653b1a51a751388a8d8d3f12
Reviewed-by: Christian Kandeler <christian.kandeler@qt.io>

4 files changed, 321 insertions, 90 deletions

diff --git a/src/app/qbs-setup-toolchains/gccprobe.cpp b/src/app/qbs-setup-toolchains/gccprobe.cpp
index 2f2a22465..e72e39bbb 100644
--- a/src/app/qbs-setup-toolchains/gccprobe.cpp
+++ b/src/app/qbs-setup-toolchains/gccprobe.cpp
@@ -85,10 +85,10 @@ static QStringList validMinGWMachines()
             QStringLiteral("amd64-mingw32msvc")};
 }
 
-static QString gccMachineName(const QString &compilerFilePath)
+static QString gccMachineName(const QFileInfo &compiler)
 {
-    return qsystem(compilerFilePath, QStringList()
-                   << QStringLiteral("-dumpmachine")).trimmed();
+    return qsystem(compiler.absoluteFilePath(), {QStringLiteral("-dumpmachine")})
+            .trimmed();
 }
 
 static QStringList standardCompilerFileNames()
@@ -99,63 +99,116 @@ static QStringList standardCompilerFileNames()
             HostOsInfo::appendExecutableSuffix(QStringLiteral("clang++"))};
 }
 
-static void setCommonProperties(Profile &profile, const QString &compilerFilePath,
-        const QStringList &toolchainTypes)
+class ToolchainDetails
 {
-    const QFileInfo cfi(compilerFilePath);
-    const QString compilerName = cfi.fileName();
+public:
+    explicit ToolchainDetails(const QFileInfo &compiler)
+    {
+        auto baseName = compiler.completeBaseName();
+        if (baseName.contains(QRegExp(QLatin1String("\\d$")))) {
+            const auto dashIndex = baseName.lastIndexOf(QLatin1Char('-'));
+            version = baseName.mid(dashIndex + 1);
+            baseName = baseName.left(dashIndex);
+        }
+
+        const auto dashIndex = baseName.lastIndexOf(QLatin1Char('-'));
+        suffix = baseName.mid(dashIndex + 1);
+        prefix = baseName.left(dashIndex + 1);
+    }
 
+    QString prefix;
+    QString suffix;
+    QString version;
+};
+
+static void setCommonProperties(Profile &profile, const QFileInfo &compiler,
+        const QStringList &toolchainTypes, const ToolchainDetails &details)
+{
     if (toolchainTypes.contains(QStringLiteral("mingw")))
         profile.setValue(QStringLiteral("qbs.targetPlatform"),
                          QStringLiteral("windows"));
 
-    const QString prefix = compilerName.left(
-                compilerName.lastIndexOf(QLatin1Char('-')) + 1);
-    if (!prefix.isEmpty())
-        profile.setValue(QStringLiteral("cpp.toolchainPrefix"), prefix);
+    if (!details.prefix.isEmpty())
+        profile.setValue(QStringLiteral("cpp.toolchainPrefix"), details.prefix);
 
     profile.setValue(QStringLiteral("cpp.toolchainInstallPath"),
-                     cfi.absolutePath());
+                     compiler.absolutePath());
     profile.setValue(QStringLiteral("qbs.toolchain"), toolchainTypes);
 
-    const QString suffix = compilerName.right(compilerName.size() - prefix.size());
-    if (!standardCompilerFileNames().contains(suffix))
+    if (!standardCompilerFileNames().contains(
+                HostOsInfo::appendExecutableSuffix(details.suffix))) {
         qWarning("%s", qPrintable(
                      QStringLiteral("'%1' is not a standard compiler file name; "
                                     "you must set the cpp.cCompilerName and "
                                     "cpp.cxxCompilerName properties of this profile "
-                                    "manually").arg(compilerName)));
+                                    "manually").arg(compiler.fileName())));
+    }
 }
 
 class ToolPathSetup
 {
 public:
-    explicit ToolPathSetup(Profile *profile, QString path, QString toolchainPrefix)
+    explicit ToolPathSetup(Profile *profile, QString path, ToolchainDetails details)
         : m_profile(profile),
           m_compilerDirPath(std::move(path)),
-          m_toolchainPrefix(std::move(toolchainPrefix))
+          m_details(std::move(details))
     {
     }
 
     void apply(const QString &toolName, const QString &propertyName) const
     {
-        QString toolFileName = m_toolchainPrefix
-                + HostOsInfo::appendExecutableSuffix(toolName);
-        if (QFile::exists(m_compilerDirPath + QLatin1Char('/') + toolFileName))
-            return;
-        const QString toolFilePath = findExecutable(toolFileName);
-        if (toolFilePath.isEmpty()) {
+        // Check for full tool name at first (includes suffix and version).
+        QString filePath = toolFilePath(toolName, UseFullToolName);
+        if (filePath.isEmpty()) {
+            // Check for base tool name at second (without of suffix and version).
+            filePath = toolFilePath(toolName, UseBaseToolName);
+            if (filePath.isEmpty()) {
+                // Check for short tool name at third (only a tool name).
+                filePath = toolFilePath(toolName, UseOnlyShortToolName);
+            }
+        }
+
+        if (filePath.isEmpty()) {
             qWarning("%s", qPrintable(
-                         QStringLiteral("'%1' exists neither in '%2' nor in PATH.")
-                         .arg(toolFileName, m_compilerDirPath)));
+                         QStringLiteral("'%1' not found in '%2'. "
+                                        "Qbs will try to find it in PATH at build time.")
+                         .arg(toolName, m_compilerDirPath)));
         }
-        m_profile->setValue(propertyName, toolFilePath);
+
+        m_profile->setValue(propertyName, filePath);
     }
 
 private:
+    enum ToolNameParts : quint8 {
+        UseOnlyShortToolName = 0x0,
+        UseToolPrefix = 0x01,
+        UseToolSuffix = 0x02,
+        UseToolVersion = 0x04,
+        UseFullToolName = UseToolPrefix | UseToolSuffix | UseToolVersion,
+        UseBaseToolName = UseToolPrefix,
+    };
+
+    QString toolFilePath(const QString &toolName, int parts) const
+    {
+        QString fileName;
+        if ((parts & UseToolPrefix) && !m_details.prefix.isEmpty())
+            fileName += m_details.prefix;
+        if ((parts & UseToolSuffix) && !m_details.suffix.isEmpty())
+            fileName += m_details.suffix + QLatin1Char('-');
+        fileName += toolName;
+        if ((parts & UseToolVersion) && !m_details.version.isEmpty())
+            fileName += QLatin1Char('-') + m_details.version;
+
+        fileName = HostOsInfo::appendExecutableSuffix(fileName);
+        QString filePath = QDir(m_compilerDirPath).absoluteFilePath(fileName);
+        if (QFile::exists(filePath))
+            return filePath;
+        return {};
+    }
+
     Profile * const m_profile;
     QString m_compilerDirPath;
-    QString m_toolchainPrefix;
+    ToolchainDetails m_details;
 };
 
 static bool doesProfileTargetOS(const Profile &profile, const QString &os)
@@ -169,12 +222,48 @@ static bool doesProfileTargetOS(const Profile &profile, const QString &os)
     return Internal::contains(HostOsInfo::hostOSIdentifiers(), os.toStdString());
 }
 
+static QString buildProfileName(const QFileInfo &cfi)
+{
+    // We need to replace a dot-separated compiler version string
+    // with a underscore-separated string, because the profile
+    // name does not allow a dots.
+    auto result = cfi.completeBaseName();
+    result.replace(QLatin1Char('.'), QLatin1Char('_'));
+    return result;
+}
+
+static QStringList buildCompilerNameFilters(const QString &compilerName)
+{
+    QStringList filters = {
+        // "clang", "gcc"
+        compilerName,
+        // "clang-8", "gcc-5"
+        compilerName + QLatin1String("-[1-9]*"),
+        // "avr-gcc"
+        QLatin1String("*-") + compilerName,
+        // "arm-none-eabi-gcc"
+        QLatin1String("*-*-*-") + compilerName,
+        // "arm-none-eabi-gcc-9.1.0"
+        QLatin1String("*-*-*-") + compilerName + QLatin1String("-[1-9]*"),
+         // "x86_64-pc-linux-gnu-gcc"
+        QLatin1String("*-*-*-*-") + compilerName,
+        // "x86_64-pc-linux-gnu-gcc-7.4.1"
+        QLatin1String("*-*-*-*-") + compilerName + QLatin1String("-[1-9]*")
+    };
+
+    std::transform(filters.begin(), filters.end(), filters.begin(),
+                   [](const auto &filter) {
+        return HostOsInfo::appendExecutableSuffix(filter);
+    });
+
+    return filters;
+}
+
 Profile createGccProfile(const QFileInfo &compiler, Settings *settings,
                          const QStringList &toolchainTypes,
                          const QString &profileName)
 {
-    const auto compilerFilePath = compiler.absoluteFilePath();
-    const QString machineName = gccMachineName(compilerFilePath);
+    const QString machineName = gccMachineName(compiler);
 
     if (toolchainTypes.contains(QLatin1String("mingw"))) {
         if (!validMinGWMachines().contains(machineName)) {
@@ -185,29 +274,31 @@ Profile createGccProfile(const QFileInfo &compiler, Settings *settings,
 
     Profile profile(!profileName.isEmpty() ? profileName : machineName, settings);
     profile.removeProfile();
-    setCommonProperties(profile, compilerFilePath, toolchainTypes);
-
-    // Check whether auxiliary tools reside within the toolchain's install path.
-    // This might not be the case when using icecc or another compiler wrapper.
-    const QString compilerDirPath = compiler.absolutePath();
-    const ToolPathSetup toolPathSetup(
-                &profile, compilerDirPath,
-                profile.value(QStringLiteral("cpp.toolchainPrefix"))
-                .toString());
-    toolPathSetup.apply(QStringLiteral("ar"), QStringLiteral("cpp.archiverPath"));
-    toolPathSetup.apply(QStringLiteral("as"), QStringLiteral("cpp.assemblerPath"));
-    toolPathSetup.apply(QStringLiteral("nm"), QStringLiteral("cpp.nmPath"));
-    if (doesProfileTargetOS(profile, QStringLiteral("darwin")))
-        toolPathSetup.apply(QStringLiteral("dsymutil"),
-                            QStringLiteral("cpp.dsymutilPath"));
-    else
-        toolPathSetup.apply(QStringLiteral("objcopy"),
-                            QStringLiteral("cpp.objcopyPath"));
-    toolPathSetup.apply(QStringLiteral("strip"),
-                        QStringLiteral("cpp.stripPath"));
+
+    const ToolchainDetails details(compiler);
+
+    setCommonProperties(profile, compiler, toolchainTypes, details);
+
+    if (!toolchainTypes.contains(QLatin1String("clang"))) {
+        // Check whether auxiliary tools reside within the toolchain's install path.
+        // This might not be the case when using icecc or another compiler wrapper.
+        const QString compilerDirPath = compiler.absolutePath();
+        const ToolPathSetup toolPathSetup(&profile, compilerDirPath, details);
+        toolPathSetup.apply(QStringLiteral("ar"), QStringLiteral("cpp.archiverPath"));
+        toolPathSetup.apply(QStringLiteral("as"), QStringLiteral("cpp.assemblerPath"));
+        toolPathSetup.apply(QStringLiteral("nm"), QStringLiteral("cpp.nmPath"));
+        if (doesProfileTargetOS(profile, QStringLiteral("darwin")))
+            toolPathSetup.apply(QStringLiteral("dsymutil"),
+                                QStringLiteral("cpp.dsymutilPath"));
+        else
+            toolPathSetup.apply(QStringLiteral("objcopy"),
+                                QStringLiteral("cpp.objcopyPath"));
+        toolPathSetup.apply(QStringLiteral("strip"),
+                            QStringLiteral("cpp.stripPath"));
+    }
 
     qbsInfo() << Tr::tr("Profile '%1' created for '%2'.")
-                 .arg(profile.name(), compilerFilePath);
+                 .arg(profile.name(), compiler.absoluteFilePath());
     return profile;
 }
 
@@ -215,38 +306,54 @@ void gccProbe(Settings *settings, QList<Profile> &profiles, const QString &compi
 {
     qbsInfo() << Tr::tr("Trying to detect %1...").arg(compilerName);
 
-    const QString crossCompilePrefix = QString::fromLocal8Bit(qgetenv("CROSS_COMPILE"));
-    const QString compilerFilePath = findExecutable(crossCompilePrefix + compilerName);
-    const QFileInfo cfi(compilerFilePath);
-    if (!cfi.exists()) {
-        qbsError() << Tr::tr("%1 not found.").arg(compilerName);
-        return;
-    }
-    const QString profileName = cfi.completeBaseName();
-    const QStringList toolchainTypes = toolchainTypeFromCompilerName(compilerName);
-    profiles.push_back(createGccProfile(compilerFilePath, settings,
-                                        toolchainTypes, profileName));
-}
+    const QStringList searchPaths = systemSearchPaths();
 
-void mingwProbe(Settings *settings, QList<Profile> &profiles)
-{
-    // List of possible compiler binary names for this platform
-    QStringList compilerNames;
-    if (HostOsInfo::isWindowsHost()) {
-        compilerNames << QStringLiteral("gcc");
-    } else {
-        const auto machineNames = validMinGWMachines();
-        for (const QString &machineName : machineNames) {
-            compilerNames << machineName + QLatin1String("-gcc");
+    std::vector<QFileInfo> candidates;
+    const auto filters = buildCompilerNameFilters(compilerName);
+    for (const auto &searchPath : searchPaths) {
+        const QDir dir(searchPath);
+        const QStringList fileNames = dir.entryList(
+                    filters, QDir::Files | QDir::Executable);
+        for (const QString &fileName : fileNames) {
+            // Ignore unexpected compiler names.
+            if (fileName.startsWith(QLatin1String("c89-gcc"))
+                    || fileName.startsWith(QLatin1String("c99-gcc"))) {
+                continue;
+            }
+            const QFileInfo candidate = dir.filePath(fileName);
+            // Filter duplicates.
+            const auto existingEnd = candidates.end();
+            const auto existingIt = std::find_if(
+                        candidates.begin(), existingEnd,
+                        [candidate](const QFileInfo &existing) {
+                return isSameExecutable(candidate.absoluteFilePath(),
+                                        existing.absoluteFilePath());
+            });
+            if (existingIt == existingEnd) {
+                // No duplicates are found, just add a new candidate.
+                candidates.push_back(candidate);
+            } else {
+                // Replace the existing entry if a candidate name more than
+                // an existing name.
+                const auto candidateName = candidate.completeBaseName();
+                const auto existingName = existingIt->completeBaseName();
+                if (candidateName > existingName)
+                    *existingIt = candidate;
+            }
         }
     }
 
-    for (const QString &compilerName : qAsConst(compilerNames)) {
-        const QString gccPath
-                = findExecutable(HostOsInfo::appendExecutableSuffix(compilerName));
-        if (!gccPath.isEmpty())
-            profiles.push_back(createGccProfile(
-                                   gccPath, settings,
-                                   canonicalToolchain(QStringLiteral("mingw"))));
+    if (candidates.empty()) {
+        qbsInfo() << Tr::tr("No %1 toolchains found.").arg(compilerName);
+        return;
+    }
+
+    for (const auto &candidate : qAsConst(candidates)) {
+        const QStringList toolchainTypes = toolchainTypeFromCompilerName(
+                    candidate.baseName());
+        const QString profileName = buildProfileName(candidate);
+        auto profile = createGccProfile(candidate, settings,
+                                        toolchainTypes, profileName);
+        profiles.push_back(std::move(profile));
     }
 }
diff --git a/src/app/qbs-setup-toolchains/gccprobe.h b/src/app/qbs-setup-toolchains/gccprobe.h
index 6ce5938d0..4344c5836 100644
--- a/src/app/qbs-setup-toolchains/gccprobe.h
+++ b/src/app/qbs-setup-toolchains/gccprobe.h
@@ -58,6 +58,5 @@ qbs::Profile createGccProfile(const QFileInfo &compiler,
 
 void gccProbe(qbs::Settings *settings, QList<qbs::Profile> &profiles,
               const QString &compilerName);
-void mingwProbe(qbs::Settings *settings, QList<qbs::Profile> &profiles);
 
 #endif // Header guard
diff --git a/src/app/qbs-setup-toolchains/probe.cpp b/src/app/qbs-setup-toolchains/probe.cpp
index d1b473e0a..f041568a9 100644
--- a/src/app/qbs-setup-toolchains/probe.cpp
+++ b/src/app/qbs-setup-toolchains/probe.cpp
@@ -54,8 +54,20 @@
 #include <tools/toolchains.h>
 
 #include <QtCore/qdir.h>
+#include <QtCore/qoperatingsystemversion.h>
 #include <QtCore/qtextstream.h>
 
+#ifdef Q_OS_WIN
+// We need defines for Windows 8.
+#undef _WIN32_WINNT
+#define _WIN32_WINNT _WIN32_WINNT_WIN8
+
+#include <qt_windows.h>
+#include <ShlObj.h>
+#else
+#include <qplatformdefs.h>
+#endif // Q_OS_WIN
+
 using namespace qbs;
 using Internal::HostOsInfo;
 using Internal::Tr;
@@ -63,6 +75,11 @@ using Internal::Tr;
 static QTextStream qStdout(stdout);
 static QTextStream qStderr(stderr);
 
+QStringList systemSearchPaths()
+{
+    return QString::fromLocal8Bit(qgetenv("PATH")).split(HostOsInfo::pathListSeparator());
+}
+
 QString findExecutable(const QString &fileName)
 {
     QString fullFileName = fileName;
@@ -70,8 +87,7 @@ QString findExecutable(const QString &fileName)
             && !fileName.endsWith(QLatin1String(".exe"), Qt::CaseInsensitive)) {
         fullFileName += QLatin1String(".exe");
     }
-    const QString path = QString::fromLocal8Bit(qgetenv("PATH"));
-    const auto ppaths = path.split(HostOsInfo::pathListSeparator());
+    const auto ppaths = systemSearchPaths();
     for (const QString &ppath : ppaths) {
         const QString fullPath = ppath + QLatin1Char('/') + fullFileName;
         if (QFileInfo::exists(fullPath))
@@ -109,16 +125,13 @@ void probe(Settings *settings)
     if (HostOsInfo::isWindowsHost()) {
         msvcProbe(settings, profiles);
         clangClProbe(settings, profiles);
-    } else {
-        gccProbe(settings, profiles, QStringLiteral("gcc"));
-        gccProbe(settings, profiles, QStringLiteral("clang"));
-
-        if (HostOsInfo::isMacosHost()) {
-            xcodeProbe(settings, profiles);
-        }
+    } else if (HostOsInfo::isMacosHost()) {
+        xcodeProbe(settings, profiles);
     }
 
-    mingwProbe(settings, profiles);
+    gccProbe(settings, profiles, QStringLiteral("gcc"));
+    gccProbe(settings, profiles, QStringLiteral("clang"));
+
     iarProbe(settings, profiles);
     keilProbe(settings, profiles);
     sdccProbe(settings, profiles);
@@ -180,3 +193,111 @@ int extractVersion(const QByteArray &macroDump, const QByteArray &keyToken)
             .toInt();
     return version;
 }
+
+static QString resolveSymlinks(const QString &filePath)
+{
+    QFileInfo fi(filePath);
+    int links = 16;
+    while (links-- && fi.isSymLink())
+        fi.setFile(fi.dir(), fi.symLinkTarget());
+    if (links <= 0)
+        return {};
+    return fi.filePath();
+}
+
+// Copied from qfilesystemengine_win.cpp.
+#ifdef Q_OS_WIN
+
+// File ID for Windows up to version 7.
+static inline QByteArray fileIdWin7(HANDLE handle)
+{
+    BY_HANDLE_FILE_INFORMATION info;
+    if (GetFileInformationByHandle(handle, &info)) {
+        char buffer[sizeof "01234567:0123456701234567\0"];
+        qsnprintf(buffer, sizeof(buffer), "%lx:%08lx%08lx",
+                  info.dwVolumeSerialNumber,
+                  info.nFileIndexHigh,
+                  info.nFileIndexLow);
+        return QByteArray(buffer);
+    }
+    return {};
+}
+
+// File ID for Windows starting from version 8.
+static QByteArray fileIdWin8(HANDLE handle)
+{
+    QByteArray result;
+    FILE_ID_INFO infoEx = {};
+    if (::GetFileInformationByHandleEx(
+                handle,
+                static_cast<FILE_INFO_BY_HANDLE_CLASS>(18), // FileIdInfo in Windows 8
+                &infoEx, sizeof(FILE_ID_INFO))) {
+        result = QByteArray::number(infoEx.VolumeSerialNumber, 16);
+        result += ':';
+        // Note: MinGW-64's definition of FILE_ID_128 differs from the MSVC one.
+        result += QByteArray(reinterpret_cast<const char *>(&infoEx.FileId),
+                             int(sizeof(infoEx.FileId))).toHex();
+    }
+    return result;
+}
+
+static QByteArray fileIdWin(HANDLE fHandle)
+{
+    return QOperatingSystemVersion::current() >= QOperatingSystemVersion::Windows8
+            ? fileIdWin8(HANDLE(fHandle))
+            : fileIdWin7(HANDLE(fHandle));
+}
+
+static QByteArray fileId(const QString &filePath)
+{
+    QByteArray result;
+    const HANDLE handle = ::CreateFile(
+                reinterpret_cast<const wchar_t*>(filePath.utf16()), 0,
+                FILE_SHARE_READ, nullptr, OPEN_EXISTING,
+                FILE_FLAG_BACKUP_SEMANTICS, nullptr);
+    if (handle != INVALID_HANDLE_VALUE) {
+        result = fileIdWin(handle);
+        ::CloseHandle(handle);
+    }
+    return result;
+}
+
+static qint64 fileSize(const QString &filePath)
+{
+    return QFileInfo(filePath).size();
+}
+
+#else
+
+static QByteArray fileId(const QString &filePath)
+{
+    QByteArray result;
+    if (Q_UNLIKELY(filePath.isEmpty()))
+        return {};
+    QT_STATBUF statResult = {};
+    if (QT_STAT(filePath.toLocal8Bit().constData(), &statResult))
+        return {};
+    result = QByteArray::number(quint64(statResult.st_dev), 16);
+    result += ':';
+    result += QByteArray::number(quint64(statResult.st_ino));
+    return result;
+}
+
+#endif // Q_OS_WIN
+
+bool isSameExecutable(const QString &filePath1, const QString &filePath2)
+{
+    if (filePath1 == filePath2)
+        return true;
+    if (resolveSymlinks(filePath1) == resolveSymlinks(filePath2))
+        return true;
+    if (fileId(filePath1) == fileId(filePath2))
+        return true;
+
+#ifdef Q_OS_WIN
+    if (fileSize(filePath1) == fileSize(filePath2))
+        return true;
+#endif
+
+    return false;
+}
diff --git a/src/app/qbs-setup-toolchains/probe.h b/src/app/qbs-setup-toolchains/probe.h
index 3dfb3f103..235d7a899 100644
--- a/src/app/qbs-setup-toolchains/probe.h
+++ b/src/app/qbs-setup-toolchains/probe.h
@@ -53,6 +53,8 @@ QT_END_NAMESPACE
 
 namespace qbs { class Settings; }
 
+QStringList systemSearchPaths();
+
 QString findExecutable(const QString &fileName);
 
 QStringList toolchainTypeFromCompilerName(const QString &compilerName);
@@ -77,4 +79,6 @@ inline bool operator<(const ToolchainInstallInfo &lhs, const ToolchainInstallInf
 
 int extractVersion(const QByteArray &macroDump, const QByteArray &keyToken);
 
+bool isSameExecutable(const QString &exe1, const QString &exe2);
+
 #endif // Header guard