Add depfile support to qsb

This adds the new option '--depfile' to qsb tool. The tool now supports
generating of gcc depfiles based on the '#include' statements found in
the shader files. The generated depfile it then passed to the CMake
add_custom_command command that calls qsb to control changes in the
shader includes.

Change-Id: I6330fbaaef35cb294f1e3ce7a3054f836fe986db
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>

17 files changed, 321 insertions, 0 deletions

diff --git a/tests/auto/CMakeLists.txt b/tests/auto/CMakeLists.txt
index 8de2eb4..1471daa 100644
--- a/tests/auto/CMakeLists.txt
+++ b/tests/auto/CMakeLists.txt
@@ -8,3 +8,9 @@ endif()
 
 add_subdirectory(qshaderbaker)
 add_subdirectory(buildtimeqsb)
+
+_qt_internal_check_depfile_support(has_depfile_support)
+if(has_depfile_support AND NOT CMAKE_CROSSCOMPILING)
+    # The test is disabled for cross-compiling platforms since we cannot embed the test results.
+    add_subdirectory(qsbdepfiles)
+endif()
diff --git a/tests/auto/qsbdepfiles/CMakeLists.txt b/tests/auto/qsbdepfiles/CMakeLists.txt
new file mode 100644
index 0000000..5875504
--- /dev/null
+++ b/tests/auto/qsbdepfiles/CMakeLists.txt
@@ -0,0 +1,52 @@
+# Copyright (C) 2024 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+if(NOT QT_BUILD_STANDALONE_TESTS AND NOT QT_BUILDING_QT)
+    cmake_minimum_required(VERSION 3.16)
+    project(tst_qsbdepfiles LANGUAGES CXX)
+    find_package(Qt6BuildInternals REQUIRED COMPONENTS STANDALONE_TEST)
+endif()
+
+qt_internal_add_test(tst_qsbdepfiles
+    SOURCES
+    tst_qsbdepfiles.cpp
+)
+
+qt_add_shaders(tst_qsbdepfiles
+    "shaders"
+    PRECOMPILE
+    OPTIMIZED
+    PREFIX
+        "/"
+    FILES
+        "shaders/tst.frag"
+        "shaders/tstinclude.frag"
+        "shaders/tstincluderelative.frag"
+        "shaders/tstincludewhitespaces.frag"
+        "shaders/tstincludecomment.frag"
+)
+
+set(data_base_dir "${CMAKE_CURRENT_BINARY_DIR}/data")
+
+configure_file("data/tst.frag.qsb.d.in"
+    "${data_base_dir}/tst.frag.qsb.d" NEWLINE_STYLE LF)
+configure_file("data/tstinclude.frag.qsb.d.in"
+    "${data_base_dir}/tstinclude.frag.qsb.d" NEWLINE_STYLE LF)
+configure_file("data/tstincluderelative.frag.qsb.d.in"
+    "${data_base_dir}/tstincluderelative.frag.qsb.d" NEWLINE_STYLE LF)
+configure_file("data/tstincludewhitespaces.frag.qsb.d.in"
+    "${data_base_dir}/tstincludewhitespaces.frag.qsb.d" NEWLINE_STYLE LF)
+configure_file("data/tstincludecomment.frag.qsb.d.in"
+    "${data_base_dir}/tstincludecomment.frag.qsb.d" NEWLINE_STYLE LF)
+
+qt_add_resources(tst_qsbdepfiles "tst_qsbdepfiles_data"
+    PREFIX "/data"
+    BASE "${data_base_dir}"
+    FILES
+        "${data_base_dir}/tst.frag.qsb.d"
+        "${data_base_dir}/tstinclude.frag.qsb.d"
+        "${data_base_dir}/tstincluderelative.frag.qsb.d"
+        "${data_base_dir}/tstincludewhitespaces.frag.qsb.d"
+        "${data_base_dir}/tstincludecomment.frag.qsb.d"
+)
+
diff --git a/tests/auto/qsbdepfiles/data/tst.frag.qsb.d.in b/tests/auto/qsbdepfiles/data/tst.frag.qsb.d.in
new file mode 100644
index 0000000..309c6c4
--- /dev/null
+++ b/tests/auto/qsbdepfiles/data/tst.frag.qsb.d.in
@@ -0,0 +1,2 @@
+@CMAKE_CURRENT_BINARY_DIR@/.qsb/shaders/tst.frag.qsb: \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/tst.frag
diff --git a/tests/auto/qsbdepfiles/data/tstinclude.frag.qsb.d.in b/tests/auto/qsbdepfiles/data/tstinclude.frag.qsb.d.in
new file mode 100644
index 0000000..2276315
--- /dev/null
+++ b/tests/auto/qsbdepfiles/data/tstinclude.frag.qsb.d.in
@@ -0,0 +1,3 @@
+@CMAKE_CURRENT_BINARY_DIR@/.qsb/shaders/tstinclude.frag.qsb: \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/tstinclude.frag \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/common.glsl
diff --git a/tests/auto/qsbdepfiles/data/tstincludecomment.frag.qsb.d.in b/tests/auto/qsbdepfiles/data/tstincludecomment.frag.qsb.d.in
new file mode 100644
index 0000000..d28642f
--- /dev/null
+++ b/tests/auto/qsbdepfiles/data/tstincludecomment.frag.qsb.d.in
@@ -0,0 +1,4 @@
+@CMAKE_CURRENT_BINARY_DIR@/.qsb/shaders/tstincludecomment.frag.qsb: \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/tstincludecomment.frag \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/common.glsl \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/inner/common2.glsl
diff --git a/tests/auto/qsbdepfiles/data/tstincluderelative.frag.qsb.d.in b/tests/auto/qsbdepfiles/data/tstincluderelative.frag.qsb.d.in
new file mode 100644
index 0000000..f094642
--- /dev/null
+++ b/tests/auto/qsbdepfiles/data/tstincluderelative.frag.qsb.d.in
@@ -0,0 +1,4 @@
+@CMAKE_CURRENT_BINARY_DIR@/.qsb/shaders/tstincluderelative.frag.qsb: \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/tstincluderelative.frag \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/common.glsl \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/inner/common2.glsl
diff --git a/tests/auto/qsbdepfiles/data/tstincludewhitespaces.frag.qsb.d.in b/tests/auto/qsbdepfiles/data/tstincludewhitespaces.frag.qsb.d.in
new file mode 100644
index 0000000..1a84e3e
--- /dev/null
+++ b/tests/auto/qsbdepfiles/data/tstincludewhitespaces.frag.qsb.d.in
@@ -0,0 +1,3 @@
+@CMAKE_CURRENT_BINARY_DIR@/.qsb/shaders/tstincludewhitespaces.frag.qsb: \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/tstincludewhitespaces.frag \
+  @CMAKE_CURRENT_SOURCE_DIR@/shaders/common.glsl
diff --git a/tests/auto/qsbdepfiles/shaders/common.glsl b/tests/auto/qsbdepfiles/shaders/common.glsl
new file mode 100644
index 0000000..a471529
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/common.glsl
@@ -0,0 +1,8 @@
+#ifndef COMMON_GLSL
+#define COMMON_GLSL
+
+vec4 getColor()
+{
+    return vec4(1.0, 0.0, 0.0, 1.0);
+}
+#endif
diff --git a/tests/auto/qsbdepfiles/shaders/inner/common2.glsl b/tests/auto/qsbdepfiles/shaders/inner/common2.glsl
new file mode 100644
index 0000000..fd8c28d
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/inner/common2.glsl
@@ -0,0 +1,7 @@
+#ifndef COMMON2_GLSL
+#define COMMON2_GLSL
+vec4 getColor2()
+{
+    return vec4(1.0, 0.0, 0.0, 1.0);
+}
+#endif
diff --git a/tests/auto/qsbdepfiles/shaders/tst.frag b/tests/auto/qsbdepfiles/shaders/tst.frag
new file mode 100644
index 0000000..a8abe1b
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/tst.frag
@@ -0,0 +1,7 @@
+#version 440
+
+layout(location = 0) out vec4 fragColor;
+
+void main() {
+  fragColor = vec4(1.0, 0.0, 0.0, 1.0);
+}
diff --git a/tests/auto/qsbdepfiles/shaders/tstinclude.frag b/tests/auto/qsbdepfiles/shaders/tstinclude.frag
new file mode 100644
index 0000000..43052c4
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/tstinclude.frag
@@ -0,0 +1,11 @@
+#version 440
+#extension GL_GOOGLE_include_directive : enable
+
+#include "common.glsl"
+#include "common.glsl"
+
+layout(location = 0) out vec4 fragColor;
+
+void main() {
+  fragColor = getColor();
+}
diff --git a/tests/auto/qsbdepfiles/shaders/tstincludecomment.frag b/tests/auto/qsbdepfiles/shaders/tstincludecomment.frag
new file mode 100644
index 0000000..2e7520e
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/tstincludecomment.frag
@@ -0,0 +1,12 @@
+#version 440
+#extension GL_GOOGLE_include_directive : enable
+
+#include "common.glsl" // The followup comment
+#include "inner/common2.glsl" // The followup comment with "
+// #include "common.glsl" commented line
+
+layout(location = 0) out vec4 fragColor;
+
+void main() {
+  fragColor = getColor();
+}
diff --git a/tests/auto/qsbdepfiles/shaders/tstincluderelative.frag b/tests/auto/qsbdepfiles/shaders/tstincluderelative.frag
new file mode 100644
index 0000000..7278bdf
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/tstincluderelative.frag
@@ -0,0 +1,12 @@
+#version 440
+#extension GL_GOOGLE_include_directive : enable
+
+#include "common.glsl"
+#include "inner/common2.glsl"
+
+layout(location = 0) out vec4 fragColor;
+
+void main() {
+  fragColor = getColor();
+  fragColor = getColor2();
+}
diff --git a/tests/auto/qsbdepfiles/shaders/tstincludewhitespaces.frag b/tests/auto/qsbdepfiles/shaders/tstincludewhitespaces.frag
new file mode 100644
index 0000000..fe07fd2
--- /dev/null
+++ b/tests/auto/qsbdepfiles/shaders/tstincludewhitespaces.frag
@@ -0,0 +1,10 @@
+#version 440
+#extension GL_GOOGLE_include_directive : enable
+
+#  include    "common.glsl"
+
+layout(location = 0) out vec4 fragColor;
+
+void main() {
+  fragColor = vec4(1.0, 0.0, 0.0, 1.0);
+}
diff --git a/tests/auto/qsbdepfiles/tst_qsbdepfiles.cpp b/tests/auto/qsbdepfiles/tst_qsbdepfiles.cpp
new file mode 100644
index 0000000..af1f2bf
--- /dev/null
+++ b/tests/auto/qsbdepfiles/tst_qsbdepfiles.cpp
@@ -0,0 +1,48 @@
+// Copyright (C) 2024 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+#include <QtTest/QTest>
+#include <QtCore/qstring.h>
+
+using namespace Qt::StringLiterals;
+
+class tst_qsbdepfiles : public QObject
+{
+    Q_OBJECT
+private:
+    void init();
+private slots:
+    void Depfiles_data();
+    void Depfiles();
+};
+
+void tst_qsbdepfiles::Depfiles_data()
+{
+    QTest::addColumn<QLatin1String>("filename");
+
+    QTest::newRow("WithoutInclude") << "tst.frag.qsb.d"_L1;
+    QTest::newRow("WithInclude") << "tstinclude.frag.qsb.d"_L1;
+    QTest::newRow("WithWhitespaces") << "tstincludewhitespaces.frag.qsb.d"_L1;
+    QTest::newRow("WithRelativeInclude") << "tstincluderelative.frag.qsb.d"_L1;
+    QTest::newRow("WithComment") << "tstincludecomment.frag.qsb.d"_L1;
+}
+
+void tst_qsbdepfiles::Depfiles()
+{
+    QFETCH(const QLatin1String, filename);
+    QString resultFilePath = QFINDTESTDATA(QString(".qsb/shaders/%1").arg(filename));
+    QString expectedFilePath = QString(":/data/%1").arg(filename);
+
+    QFile resultFile(resultFilePath);
+    QFile expectedFile(expectedFilePath);
+
+    QVERIFY(resultFile.open(QFile::ReadOnly));
+    QVERIFY(expectedFile.open(QFile::ReadOnly));
+
+    // We chop the one byte from expectedFile because of CMake issue.
+    // See https://gitlab.kitware.com/cmake/cmake/-/issues/25164
+    QVERIFY(resultFile.readAll() == expectedFile.readAll().chopped(1));
+}
+
+QTEST_MAIN(tst_qsbdepfiles)
+
+#include "tst_qsbdepfiles.moc"
diff --git a/tools/qsb/Qt6ShaderToolsMacros.cmake b/tools/qsb/Qt6ShaderToolsMacros.cmake
index d9122cf..e939528 100644
--- a/tools/qsb/Qt6ShaderToolsMacros.cmake
+++ b/tools/qsb/Qt6ShaderToolsMacros.cmake
@@ -75,6 +75,8 @@ function(_qt_internal_add_shaders_impl target resourcename)
         ${ARGN}
     )
 
+    _qt_internal_check_depfile_support(has_depfile_support)
+
     math(EXPR file_index "0")
     foreach(file_and_replacements IN LISTS arg_FILES)
         string(REPLACE "@" ";" file_and_replacement_list "${file_and_replacements}")
@@ -210,6 +212,14 @@ function(_qt_internal_add_shaders_impl target resourcename)
         list(APPEND qsb_args "-o")
         list(APPEND qsb_args "${qsb_result}")
 
+        if(has_depfile_support)
+            set(depfile "${qsb_result}.d")
+            list(APPEND qsb_args "--depfile" "${depfile}")
+            set(depfile_extra_args DEPFILE "${depfile}")
+        else()
+            set(depfile_extra_args "")
+        endif()
+
         list(APPEND qsb_args "${file_absolute}")
 
         if (qsb_replace_args)
@@ -227,6 +237,7 @@ function(_qt_internal_add_shaders_impl target resourcename)
                 DEPENDS
                     "${file_absolute}"
                     ${QT_CMAKE_EXPORT_NAMESPACE}::qsb
+                ${depfile_extra_args}
                 VERBATIM
             )
         else()
@@ -238,6 +249,7 @@ function(_qt_internal_add_shaders_impl target resourcename)
                 DEPENDS
                     "${file_absolute}"
                     ${QT_CMAKE_EXPORT_NAMESPACE}::qsb
+                ${depfile_extra_args}
                 VERBATIM
             )
         endif()
@@ -264,6 +276,15 @@ function(_qt_internal_add_shaders_impl target resourcename)
             list(APPEND qsb_multiview2_args "-o")
             list(APPEND qsb_multiview2_args "${qsb_multiview2_result}")
             list(APPEND qsb_multiview2_args "${file_absolute}")
+
+            if(has_depfile_support)
+                set(depfile "${qsb_multiview2_result}.d")
+                list(APPEND qsb_multiview2_args "--depfile" "${depfile}")
+                set(depfile_extra_args DEPFILE "${depfile}")
+            else()
+                set(depfile_extra_args "")
+            endif()
+
             add_custom_command(
                 OUTPUT
                     ${qsb_multiview2_result}
@@ -272,6 +293,7 @@ function(_qt_internal_add_shaders_impl target resourcename)
                 DEPENDS
                     "${file_absolute}"
                     ${QT_CMAKE_EXPORT_NAMESPACE}::qsb
+                ${depfile_extra_args}
                 VERBATIM
             )
             list(APPEND qsb_files "${qsb_multiview2_result}")
diff --git a/tools/qsb/qsb.cpp b/tools/qsb/qsb.cpp
index 3700a07..87a57a4 100644
--- a/tools/qsb/qsb.cpp
+++ b/tools/qsb/qsb.cpp
@@ -6,6 +6,7 @@
 #include <QtCore/qtextstream.h>
 #include <QtCore/qfile.h>
 #include <QtCore/qdir.h>
+#include <QtCore/qset.h>
 #include <QtCore/qtemporarydir.h>
 #include <QtCore/qdebug.h>
 #include <QtCore/qlibraryinfo.h>
@@ -24,6 +25,8 @@
 // invocations must be guarded with !silent.
 static bool silent = false;
 
+using namespace Qt::StringLiterals;
+
 enum class FileType
 {
     Binary,
@@ -460,6 +463,91 @@ static void replaceShaderContents(QShader *shaderPack,
     }
 }
 
+static bool generateDepfile(QFile &depfile, const QString &inputFilename,
+                            const QString &outputFilename)
+{
+    constexpr QByteArrayView includeKeyword("include");
+    // Assume that the minimalistic include statment should look as following: #include "x"
+    constexpr qsizetype minIncludeStatementSize = includeKeyword.size() + 5;
+
+    QFile inputFile(inputFilename);
+    if (!inputFile.open(QFile::ReadOnly)) {
+        printError("Unable to open input file: '%s'", qPrintable(inputFilename));
+        return false;
+    }
+    depfile.write(outputFilename.toUtf8());
+    depfile.write(": \\\n  "_ba);
+    depfile.write(inputFilename.toUtf8());
+    enum { ParseHash, ParseInclude, ParseFilename } parserState = ParseHash;
+
+    QSet<QString> knownDeps;
+    QByteArray outputBuffer;
+    while (!inputFile.atEnd()) {
+        QByteArray line = inputFile.readLine();
+        if (line.size() < minIncludeStatementSize)
+            continue;
+
+        parserState = ParseHash;
+        for (auto it = line.constBegin(); it < line.constEnd(); ++it) {
+            const auto c = *it;
+            if (c == '\t' || c == ' ')
+                continue;
+
+            if (parserState == ParseHash) {
+                // Looking for #
+                if (c == '#')
+                    parserState = ParseInclude;
+                else
+                    break;
+            } else if (parserState == ParseInclude) {
+                // Looking for 'include'
+                if (includeKeyword == QByteArrayView(it, includeKeyword.size()))
+                    parserState = ParseFilename;
+                else
+                    break;
+                it += includeKeyword.size();
+            } else if (parserState == ParseFilename) {
+                // Looking for wrapping quotes
+                QString includeString = QString::fromUtf8(QByteArrayView(it, line.constEnd()));
+                QChar quoteCounterpart;
+                if (includeString.front() == '<') {
+                    quoteCounterpart = '>';
+                } else if (includeString.front() == '"') {
+                    quoteCounterpart = '"';
+                } else {
+                    qWarning("Unrecognised include statement: '%s'", qPrintable(includeString));
+                    break;
+                }
+
+                const auto filenameBegin = 1;
+                const auto filenameEnd = includeString.indexOf(quoteCounterpart, filenameBegin);
+                if (filenameEnd > filenameBegin) {
+                    QString filename =
+                            includeString.sliced(filenameBegin, filenameEnd - filenameBegin);
+                    QFileInfo info(QFileInfo(inputFilename).absolutePath() + '/' + filename);
+
+                    if (info.exists()) {
+                        QString filePath = info.absoluteFilePath();
+                        if (!knownDeps.contains(filePath)) {
+                            outputBuffer.append(" \\\n  "_ba + filePath.toUtf8());
+                            knownDeps.insert(filePath);
+                        }
+                    } else {
+                        qWarning("File '%s' included in '%s' doesn't exist. Skip adding "
+                                 "dependency.",
+                                 qPrintable(filename), qPrintable(inputFilename));
+                    }
+                }
+                break;
+            }
+        }
+    }
+
+    if (!outputBuffer.isEmpty())
+        depfile.write(outputBuffer);
+    return true;
+}
+
 int main(int argc, char **argv)
 {
     QCoreApplication app(argc, argv);
@@ -571,6 +659,12 @@ int main(int argc, char **argv)
     QCommandLineOption silentOption({ "s", "silent" }, QObject::tr("Enables silent mode. Only fatal errors will be printed."));
     cmdLineParser.addOption(silentOption);
 
+    QCommandLineOption depfileOption("depfile",
+                                     QObject::tr("Enables generating the depfile for the input "
+                                                 "shaders, using the #include statements."),
+                                     QObject::tr("depfile"));
+    cmdLineParser.addOption(depfileOption);
+
     cmdLineParser.process(app);
 
     if (cmdLineParser.positionalArguments().isEmpty()) {
@@ -581,6 +675,18 @@ int main(int argc, char **argv)
     silent = cmdLineParser.isSet(silentOption);
 
     QShaderBaker baker;
+
+    QFile depfile;
+    if (const QString depfilePath = cmdLineParser.value(depfileOption); !depfilePath.isEmpty()) {
+        depfile.setFileName(depfilePath);
+        if (!depfile.open(QFile::WriteOnly | QFile::Truncate)) {
+            printError("Unable to create DEPFILE: '%s'", qPrintable(depfilePath));
+            return 1;
+        }
+    }
+
+    const bool depfileRequired = depfile.isOpen();
+
     for (const QString &fn : cmdLineParser.positionalArguments()) {
         auto qsbVersion = QShader::SerializedFormatVersion::Latest;
         if (cmdLineParser.isSet(qsbVersionOption)) {
@@ -595,6 +701,7 @@ int main(int argc, char **argv)
                 return 1;
             }
         }
+
         if (cmdLineParser.isSet(dumpOption)
                 || cmdLineParser.isSet(extractOption)
                 || cmdLineParser.isSet(replaceOption)
@@ -629,6 +736,9 @@ int main(int argc, char **argv)
             continue;
         }
 
+        if (depfileRequired && !generateDepfile(depfile, fn, cmdLineParser.value(outputOption)))
+            return 1;
+
         baker.setSourceFileName(fn);
 
         baker.setPerTargetCompilation(cmdLineParser.isSet(perTargetCompileOption));