C++ scanner: Properly handle UTF-8 byte order mark

Includes in the first line of a source file starting with a byte order
mark would not get detected, resulting in failure to recompile after a
header file change.
Skip over the BOM before passing the data to the lexer.

Task-number: QBS-1348
Done-with: Ola Røer Thorsen <ola@silentwings.no>
Change-Id: I894a5b1667e49be5f8ce9b5cc66e4e796fd073f1
Reviewed-by: Ola Røer Thorsen <ola@silentwings.no>
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>

6 files changed, 35 insertions, 0 deletions

diff --git a/src/plugins/scanner/cpp/cppscanner.cpp b/src/plugins/scanner/cpp/cppscanner.cpp
index 7e5cbe146..1d7fb7c84 100644
--- a/src/plugins/scanner/cpp/cppscanner.cpp
+++ b/src/plugins/scanner/cpp/cppscanner.cpp
@@ -248,6 +248,16 @@ static void *openScanner(const unsigned short *filePath, const char *fileTags, i
         return nullptr;
 
     opaque->fileContent = reinterpret_cast<char *>(vmap);
+
+    // Check for UTF-8 Byte Order Mark (BOM). Skip if found.
+    if (mapl >= 3
+            && opaque->fileContent[0] == char(0xef)
+            && opaque->fileContent[1] == char(0xbb)
+            && opaque->fileContent[2] == char(0xbf)) {
+        opaque->fileContent += 3;
+        mapl -= 3;
+    }
+
     CPlusPlus::Lexer lex(opaque->fileContent, opaque->fileContent + mapl);
     scanCppFile(opaque.get(), lex, flags & ScanForFileTagsFlag, flags & ScanForDependenciesFlag);
     return opaque.release();
diff --git a/tests/auto/blackbox/testdata/bom-sources/bom-sources.qbs b/tests/auto/blackbox/testdata/bom-sources/bom-sources.qbs
new file mode 100644
index 000000000..8b5dbb238
--- /dev/null
+++ b/tests/auto/blackbox/testdata/bom-sources/bom-sources.qbs
@@ -0,0 +1,6 @@
+import qbs
+
+CppApplication {
+    name: "app"
+    files: ["main.cpp", "theheader.h"]
+}
diff --git a/tests/auto/blackbox/testdata/bom-sources/main.cpp b/tests/auto/blackbox/testdata/bom-sources/main.cpp
new file mode 100644
index 000000000..de10a65a2
--- /dev/null
+++ b/tests/auto/blackbox/testdata/bom-sources/main.cpp
@@ -0,0 +1,3 @@
+#include "theheader.h"
+
+int main() {}
diff --git a/tests/auto/blackbox/testdata/bom-sources/theheader.h b/tests/auto/blackbox/testdata/bom-sources/theheader.h
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/auto/blackbox/testdata/bom-sources/theheader.h
diff --git a/tests/auto/blackbox/tst_blackbox.cpp b/tests/auto/blackbox/tst_blackbox.cpp
index a833bd264..8ca35249a 100644
--- a/tests/auto/blackbox/tst_blackbox.cpp
+++ b/tests/auto/blackbox/tst_blackbox.cpp
@@ -5839,6 +5839,21 @@ void TestBlackbox::badInterpreter()
     QCOMPARE(runQbs(QbsRunParameters("run", QStringList() << "-p" << "script-ok")), 0);
 }
 
+void TestBlackbox::bomSources()
+{
+    QDir::setCurrent(testDataDir + "/bom-sources");
+    const bool success = runQbs() == 0;
+    if (!success)
+        QSKIP("Assuming compiler cannot deal with byte order mark");
+    QVERIFY2(m_qbsStdout.contains("compiling main.cpp"), m_qbsStdout.constData());
+    WAIT_FOR_NEW_TIMESTAMP();
+    QCOMPARE(runQbs(), 0);
+    QVERIFY2(!m_qbsStdout.contains("compiling main.cpp"), m_qbsStdout.constData());
+    touch("theheader.h");
+    QCOMPARE(runQbs(), 0);
+    QVERIFY2(m_qbsStdout.contains("compiling main.cpp"), m_qbsStdout.constData());
+}
+
 void TestBlackbox::buildDataOfDisabledProduct()
 {
     QDir::setCurrent(testDataDir + QLatin1String("/build-data-of-disabled-product"));
diff --git a/tests/auto/blackbox/tst_blackbox.h b/tests/auto/blackbox/tst_blackbox.h
index fb95e723a..e5adc762f 100644
--- a/tests/auto/blackbox/tst_blackbox.h
+++ b/tests/auto/blackbox/tst_blackbox.h
@@ -50,6 +50,7 @@ private slots:
     void autotestWithDependencies();
     void auxiliaryInputsFromDependencies();
     void badInterpreter();
+    void bomSources();
     void buildDataOfDisabledProduct();
     void buildDirectories();
     void buildEnvChange();