Fix BiC tests to be able to test multiple modules at once

Task-number: QTBUG-100698
Change-Id: If4b84cc030faafcab8566dc02ef86d253f1eb3a2
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

3 files changed, 118 insertions, 30 deletions

diff --git a/tests/postbuild/bic/README b/tests/postbuild/bic/README
index 9dae447e..ff78bbaf 100644
--- a/tests/postbuild/bic/README
+++ b/tests/postbuild/bic/README
@@ -65,3 +65,33 @@ cd $QTDIR/../qtdeclarative/tests/auto/bic/data
 $QTDIR/../qtqa/tests/postbuild/bic/gen.sh "QtQml QtQuick QtQuickTest" 5.4.0.linux-gcc-amd64
 
 To verify, run the test as described above.
+
+The new way
+-----------
+
+It's also possible to test multiple modules at once. This avoids the problem
+we've had occasionally in Qt 6, namely that modules can include headers from
+other modules, and then those cross-module includes may later be refactored
+away. This used to cause false positives in BiC tests, because the symbols
+are no longer there when tested against old data that had them.
+
+This is facilitated by running the test with QTDIR set as explained above,
+and then listing all the modules in QT_MODULE_TO_TEST, while also providing
+the directory where the test data resides (the old way automatically
+gets the test data location from the module directory, in the new way we
+need to provide the location). Looks like this:
+
+QT_MODULE_TO_TEST="qt=QtConcurrent,QtCore,QtDBus,QtDesigner,QtGui,QtHelp,QtNetwork,QtOpenGL,QtPrintSupport,QtQml,QtQuick,QtQuickTest,QtSql,QtSvg,QtTest,QtWidgets,QtXml;/your-qt-sources/qtbase" ./tst_bic
+
+In other words, the format of QT_MODULE_TEST is
+qt=module1,module1;path-to-test-data
+
+The corresponding new way to generate test data is similar; you again
+set QTDIR, and then invoke the script with slightly different
+arguments:
+
+gen.sh qt=QtConcurrent,QtCore,QtDBus,QtDesigner,QtGui,QtHelp,QtNetwork,QtOpenGL,QtPrintSupport,QtQml,QtQuick,QtQuickTest,QtSql,QtSvg,QtTest,QtWidgets,QtXml 6.3.0.linux-gcc-amd64
+
+The format of the invocation is
+
+gen.sh qt=mod1,mod2,... platform
diff --git a/tests/postbuild/bic/gen.sh b/tests/postbuild/bic/gen.sh
index dd148988..15de2b79 100755
--- a/tests/postbuild/bic/gen.sh
+++ b/tests/postbuild/bic/gen.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 #############################################################################
 ##
-## Copyright (C) 2017 The Qt Company Ltd.
+## Copyright (C) 2022 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is the build configuration utility of the Qt Toolkit.
@@ -29,9 +29,10 @@
 
 if [ "$#" -lt 2 ]; then
     echo "$0 - Generates reference files for b/c autotest"
-    echo "Usage: $0 [module|-all] [platform]"
+    echo "Usage: $0 [module|qt=mod1,mod2,...|-all] [platform]"
     echo "Examples: $0 -all 5.1.0.macx-gcc-ppc32"
     echo "          $0 QtGui 5.0.0.linux-gcc-ia32"
+    echo "          $0 qt=QtGui,QtQuick 5.0.0.linux-gcc-amd64"
     exit 1
 fi
 
@@ -41,6 +42,21 @@ else
     modules="$1"
 fi
 
+use_qt=""
+
+case "$1" in
+    qt=*) use_qt="yes"
+    # split qt=foo,bar,baz into ["qt", "foo,bar,baz"]
+    IFS="=" read -r -a split_qt <<-_EOF_
+$1
+_EOF_
+    # split "foo,bar,baz" into ["foo", "bar", "baz"]
+    IFS="," read -r -a qt_modules <<-_EOF_
+${split_qt[1]}
+_EOF_
+;;
+esac
+
 GCC_MAJOR=`echo '__GNUC__' | gcc -E - | tail -1`
 
 if [ $GCC_MAJOR -ge 8 ]; then
@@ -49,15 +65,27 @@ else
     DUMP_CMDLINE=-fdump-class-hierarchy
 fi
 
-for module in $modules; do
-    echo "#include <$module/$module>" >test.cpp
-    g++ -c -std=c++17 -I$QTDIR/include -DQT_NO_STL $DUMP_CMDLINE -fPIC test.cpp
-    mv test.cpp*.class $module.$2.txt
+function remove_templates {
     # Remove template classes from the output
     perl -pi -e '$skip = 1 if (/^(Class|Vtable).*</);
         if ($skip) {
             $skip = 0 if (/^$/);
             $_ = "";
-        }' $module.$2.txt
-done
+        }' $1
+}
 
+if [ "$use_qt" == "yes" ]; then
+    for qt_module in "${qt_modules[@]}"; do
+        echo -e "#include <$qt_module/$qt_module>\n" >> test.cpp
+    done
+    g++ -c -std=c++17 -I$QTDIR/include -DQT_NO_STL $DUMP_CMDLINE -fPIC test.cpp
+    mv test.cpp*.class qt.$2.txt
+    remove_templates qt.$2.txt
+else
+    for module in $modules; do
+        echo "#include <$module/$module>" >test.cpp
+        g++ -c -std=c++17 -I$QTDIR/include -DQT_NO_STL $DUMP_CMDLINE -fPIC test.cpp
+        mv test.cpp*.class $module.$2.txt
+        remove_templates "$module.$2.txt"
+    done
+fi
diff --git a/tests/postbuild/bic/tst_bic.cpp b/tests/postbuild/bic/tst_bic.cpp
index 40aad19f..c4b8f941 100644
--- a/tests/postbuild/bic/tst_bic.cpp
+++ b/tests/postbuild/bic/tst_bic.cpp
@@ -174,6 +174,9 @@ private slots:
 private:
     QBic bic;
     QString qtModuleDir;
+    QString qtDir;
+    QString qtSourceDir;
+    QStringList qtModules;
     QHash<QString, QString> modules;
     QStringList incPaths;
     QString m_compiler;
@@ -361,10 +364,25 @@ void tst_Bic::initTestCase()
 
     qWarning("This test needs the correct qmake in PATH, we need it to generate INCPATH for qt modules.");
 
-    qtModuleDir = QDir::cleanPath(QFile::decodeName(qgetenv(moduleVar)));
-    if (qtModuleDir.isEmpty()) {
-        QSKIP("$QT_MODULE_TO_TEST is unset - nothing to test.  "
-              "Set QT_MODULE_TO_TEST to the absolute path of a Qt module to test.");
+    QString moduleEnvVar = QFile::decodeName(qgetenv(moduleVar));
+    bool qt = moduleEnvVar.startsWith(QLatin1String("qt="));
+    if (qt) {
+        qtDir = QFile::decodeName(qgetenv("QTDIR"));
+        if (qtDir.isEmpty())
+            QSKIP("QTDIR is unset. QTDIR must point to Qt headers when testing in qt mode.");
+        QStringList qtComponents = moduleEnvVar.split(QLatin1Char(';'));
+        QStringList qtSplit = qtComponents[0].split(QLatin1Char('='));
+        if (qtSplit.size() == 2) {
+            qtModules = qtSplit[1].split(QLatin1Char(','));
+            if (qtComponents.size() == 2)
+                qtSourceDir = qtComponents[1];
+        }
+    } else {
+        qtModuleDir = QDir::cleanPath(QFile::decodeName(qgetenv(moduleVar)));
+        if (qtModuleDir.isEmpty()) {
+            QSKIP("$QT_MODULE_TO_TEST is unset - nothing to test.  "
+                  "Set QT_MODULE_TO_TEST to the absolute path of a Qt module to test.");
+        }
     }
     if (m_compiler != QLatin1String("g++")) {
         const QString message = QLatin1String("Support for \"")
@@ -375,23 +393,25 @@ void tst_Bic::initTestCase()
     if (qgetenv("PATH").contains("teambuilder"))
         qWarning("This test might not work with teambuilder, consider switching it off.");
 
-    QString configFile = qtModuleDir + "/tests/global/global.cfg";
-
-    if (!QFile(configFile).exists()) {
-        QSKIP(
-            qPrintable(QString(
-                "%1 does not exist.  Create it if you want to run this test."
-            ).arg(configFile))
-        );
-    }
+    if (!qt) {
+        QString configFile = qtModuleDir + "/tests/global/global.cfg";
 
-    QString workDir = qtModuleDir + QStringLiteral("/tests/global");
-    modules = qt_tests_shared_global_get_modules(workDir, configFile);
+        if (!QFile(configFile).exists()) {
+            QSKIP(
+                  qPrintable(QString(
+                      "%1 does not exist.  Create it if you want to run this test."
+                  ).arg(configFile))
+            );
+        }
 
-    if (!modules.size())
-        QSKIP("No modules found.");
+        QString workDir = qtModuleDir + QStringLiteral("/tests/global");
+        modules = qt_tests_shared_global_get_modules(workDir, configFile);
 
-    incPaths = qt_tests_shared_global_get_include_paths(workDir, modules);
+        if (!modules.size())
+            QSKIP("No modules found.");
+        incPaths = qt_tests_shared_global_get_include_paths(workDir, modules);
+    } else
+        incPaths << QLatin1String("-I") + qtDir + QLatin1String("/include");
 
     QVERIFY2(incPaths.size() > 0, "Parse INCPATH failed.");
     m_compilerArguments = compilerArguments(m_compiler, incPaths);
@@ -445,12 +465,16 @@ void tst_Bic::sizesAndVTables_data()
     QTest::addColumn<QString>("oldLib");
     QTest::addColumn<bool>("isPatchRelease");
 
-    const QStringList keys = modules.keys();
+    QStringList keys;
+    if (qtModules.isEmpty())
+        keys = modules.keys();
+    else
+        keys << QLatin1String("qt");
     int major = QT_VERSION_MAJOR;
     int minor = QT_VERSION_MINOR;
     int patch = QT_VERSION_PATCH;
 
-    QFile qmakeConf(qtModuleDir + "/.qmake.conf");
+    QFile qmakeConf(qtSourceDir.isEmpty() ? qtModuleDir : qtSourceDir + "/.qmake.conf");
     if (qmakeConf.open(QIODevice::ReadOnly)) {
         const QString contents = QString::fromUtf8(qmakeConf.readAll());
         qmakeConf.close();
@@ -478,7 +502,7 @@ void tst_Bic::sizesAndVTables_data()
                 QTest::newRow(key.toLatin1() + ":5." + QByteArray::number(i))
 #endif
                     << key
-                    << (QString(qtModuleDir + "/tests/auto/bic/data/%1.")
+                    << (QString(qtSourceDir.isEmpty() ? qtModuleDir : qtSourceDir + "/tests/auto/bic/data/%1.")
                         + QString::number(major)
                         + QLatin1Char('.')
                         + QString::number(i)
@@ -500,7 +524,13 @@ QBic::Info tst_Bic::getCurrentInfo(const QString &libName)
     tmpQFile.open();
     QString tmpFileName = tmpQFile.fileName();
 
-    QByteArray tmpFileContents = "#include<" + libName.toLatin1() + "/" + libName.toLatin1() + ">\n";
+    QByteArray tmpFileContents;
+    if (qtModules.isEmpty()) {
+        tmpFileContents = "#include<" + libName.toLatin1() + "/" + libName.toLatin1() + ">\n";
+    } else {
+        for (auto &&x : qtModules)
+            tmpFileContents += "#include<" + x.toLatin1() + "/" + x.toLatin1() + ">\n";
+    }
     tmpQFile.write(tmpFileContents);
     tmpQFile.flush();