Add blackbox examples test

It is necessary to build examples with different profiles pretty much
like it is done with other tests.

Thus, build examples as a separate blackbox test.

Change-Id: If4e910fb60a6d51e1a0c690e7a4ece4b17bd5b95
Reviewed-by: Christian Kandeler <christian.kandeler@qt.io>

9 files changed, 199 insertions, 2 deletions

diff --git a/.travis.yml b/.travis.yml
index 486ee6a76..5ca075d10 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -176,7 +176,6 @@ jobs:
           modules.qbsbuildconfig.enableUnitTests:true
           modules.cpp.treatWarningsAsErrors:true
           project.withDocumentation:true
-          project.withExamples:true
           config:release-64 profile:qt64
           config:release profile:qt
       after_script:
diff --git a/scripts/build-qbs-with-qbs.sh b/scripts/build-qbs-with-qbs.sh
index 91a35f41f..d1c1916f2 100755
--- a/scripts/build-qbs-with-qbs.sh
+++ b/scripts/build-qbs-with-qbs.sh
@@ -63,7 +63,6 @@ BUILD_OPTIONS="\
     modules.qbsbuildconfig.enableUnitTests:true \
     modules.cpp.treatWarningsAsErrors:true \
     modules.cpp.separateDebugInformation:true \
-    project.withExamples:true \
     ${BUILD_OPTIONS} \
     config:release \
 "
diff --git a/tests/auto/auto.pro b/tests/auto/auto.pro
index 1afe48176..ba312c2a6 100644
--- a/tests/auto/auto.pro
+++ b/tests/auto/auto.pro
@@ -13,6 +13,7 @@ SUBDIRS += \
     blackbox/blackbox-android.pro \
     blackbox/blackbox-apple.pro \
     blackbox/blackbox-clangdb.pro \
+    blackbox/blackbox-examples.pro \
     blackbox/blackbox-java.pro \
     blackbox/blackbox-joblimits.pro \
     blackbox/blackbox-qt.pro \
diff --git a/tests/auto/auto.qbs b/tests/auto/auto.qbs
index bf75d0f23..1f72a2116 100644
--- a/tests/auto/auto.qbs
+++ b/tests/auto/auto.qbs
@@ -8,6 +8,7 @@ Project {
         "blackbox/blackbox-android.qbs",
         "blackbox/blackbox-apple.qbs",
         "blackbox/blackbox-clangdb.qbs",
+        "blackbox/blackbox-examples.qbs",
         "blackbox/blackbox-java.qbs",
         "blackbox/blackbox-joblimits.qbs",
         "blackbox/blackbox-qt.qbs",
diff --git a/tests/auto/blackbox/blackbox-examples.pro b/tests/auto/blackbox/blackbox-examples.pro
new file mode 100644
index 000000000..70d5b6419
--- /dev/null
+++ b/tests/auto/blackbox/blackbox-examples.pro
@@ -0,0 +1,18 @@
+TARGET = tst_blackbox-examples
+
+HEADERS = tst_blackboxexamples.h tst_blackboxbase.h
+SOURCES = tst_blackboxexamples.cpp tst_blackboxbase.cpp
+OBJECTS_DIR = examples
+MOC_DIR = $${OBJECTS_DIR}-moc
+
+include(../auto.pri)
+
+DATA_DIRS = ../../../examples
+
+for(data_dir, DATA_DIRS) {
+    files = $$files($$PWD/$$data_dir/*, true)
+    win32:files ~= s|\\\\|/|g
+    for(file, files):!exists($$file/*):FILES += $$file
+}
+
+OTHER_FILES += $$FILES
diff --git a/tests/auto/blackbox/blackbox-examples.qbs b/tests/auto/blackbox/blackbox-examples.qbs
new file mode 100644
index 000000000..77d32636c
--- /dev/null
+++ b/tests/auto/blackbox/blackbox-examples.qbs
@@ -0,0 +1,21 @@
+import qbs.Utilities
+
+QbsAutotest {
+    testName: "blackbox-examples"
+    Depends { name: "qbs_app" }
+    Depends { name: "qbs-setup-toolchains" }
+    Group {
+        name: "testdata"
+        prefix: "../../../examples/"
+        files: ["**/*"]
+        fileTags: []
+    }
+    files: [
+        "../shared.h",
+        "tst_blackboxexamples.cpp",
+        "tst_blackboxexamples.h",
+        "tst_blackboxbase.cpp",
+        "tst_blackboxbase.h",
+    ]
+    cpp.defines: base.concat(["SRCDIR=" + Utilities.cStringQuote(path)])
+}
diff --git a/tests/auto/blackbox/tst_blackboxexamples.cpp b/tests/auto/blackbox/tst_blackboxexamples.cpp
new file mode 100644
index 000000000..1da56b311
--- /dev/null
+++ b/tests/auto/blackbox/tst_blackboxexamples.cpp
@@ -0,0 +1,103 @@
+/****************************************************************************
+**
+** Copyright (C) 2020 Ivan Komissarov (abbapoh@gmail.com)
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms and
+** conditions see http://www.qt.io/terms-conditions. For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#include "tst_blackboxexamples.h"
+
+#include <QtCore/qdir.h>
+#include <QtCore/qdiriterator.h>
+
+QStringList TestBlackboxExamples::collectExamples(const QString &dirPath)
+{
+    QStringList result;
+    QDir dir(dirPath);
+    const auto subDirs = dir.entryList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Name);
+    for (const auto &subDir : subDirs) {
+        const auto path = dir.filePath(subDir);
+        if (!QFileInfo::exists(path + "/" + subDir + ".qbs"))
+            continue;
+        result.append(QDir(testDataDir).relativeFilePath(path));
+    }
+    return result;
+}
+
+TestBlackboxExamples::TestBlackboxExamples()
+    : TestBlackboxBase(SRCDIR "/../../../examples/", "blackbox-examples")
+{
+}
+
+void TestBlackboxExamples::baremetal_data()
+{
+    QTest::addColumn<QString>("example");
+
+    QDir baremetal(testDataDir + "/baremetal/");
+    const auto subDirs = baremetal.entryList(QDir::Dirs | QDir::NoDotAndDotDot, QDir::Name);
+    for (const auto &subDir : subDirs) {
+        const auto examples = collectExamples(baremetal.filePath(subDir));
+        for (const auto &example: examples) {
+            const auto relativePath = baremetal.relativeFilePath(example);
+            QTest::newRow(relativePath.toUtf8().data()) << relativePath;
+        }
+    }
+}
+
+void TestBlackboxExamples::baremetal()
+{
+    QFETCH(QString, example);
+
+    QVERIFY(QDir::setCurrent(testDataDir + "/" + example));
+    QCOMPARE(runQbs(), 0);
+}
+
+void TestBlackboxExamples::examples_data()
+{
+    QTest::addColumn<QString>("example");
+
+    auto examples = collectExamples(testDataDir);
+    examples.append(collectExamples(testDataDir + "/protobuf"));
+    std::sort(examples.begin(), examples.end());
+
+    for (const auto &example: examples) {
+        if (example == u"baremetal")
+            continue;
+        QTest::newRow(example.toUtf8().data()) << example;
+    }
+}
+
+void TestBlackboxExamples::examples()
+{
+    QFETCH(QString, example);
+
+    QVERIFY(QDir::setCurrent(testDataDir + "/" + example));
+    QbsRunParameters params(
+            {QStringLiteral("-f"), QFileInfo(example).fileName() + QStringLiteral(".qbs")});
+    QCOMPARE(runQbs(params), 0);
+}
+
+QTEST_MAIN(TestBlackboxExamples)
diff --git a/tests/auto/blackbox/tst_blackboxexamples.h b/tests/auto/blackbox/tst_blackboxexamples.h
new file mode 100644
index 000000000..bea8be768
--- /dev/null
+++ b/tests/auto/blackbox/tst_blackboxexamples.h
@@ -0,0 +1,53 @@
+/****************************************************************************
+**
+** Copyright (C) 2020 Ivan Komissarov (abbapoh@gmail.com)
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms and
+** conditions see http://www.qt.io/terms-conditions. For further information
+** use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file.  Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, The Qt Company gives you certain additional
+** rights.  These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+****************************************************************************/
+
+#ifndef TST_BLACKBOXEXAMPLES_H
+#define TST_BLACKBOXEXAMPLES_H
+
+#include "tst_blackboxbase.h"
+
+class TestBlackboxExamples : public TestBlackboxBase
+{
+    Q_OBJECT
+
+private:
+    QStringList collectExamples(const QString &dirPath);
+
+public:
+    TestBlackboxExamples();
+
+private slots:
+    void baremetal_data();
+    void baremetal();
+    void examples_data();
+    void examples();
+};
+
+#endif // TST_BLACKBOXEXAMPLES_H
diff --git a/tests/auto/shared.h b/tests/auto/shared.h
index beda8fff3..8dced2062 100644
--- a/tests/auto/shared.h
+++ b/tests/auto/shared.h
@@ -335,6 +335,8 @@ inline QString testWorkDir(const QString &testName)
 inline bool copyDllExportHeader(const QString &srcDataDir, const QString &targetDataDir)
 {
     QFile sourceFile(srcDataDir + "/../../dllexport.h");
+    if (!sourceFile.exists())
+        return true;
     const QString targetPath = targetDataDir + "/dllexport.h";
     QFile::remove(targetPath);
     return sourceFile.copy(targetPath);