CMake: Tell shiboken to process Qt headers inside system include dirs

When building Qt For Python using yocto, the Qt headers are in a
sysroot which libclang considers as system headers. Shiboken skips
processing system headers.

To ensure Qt headers are still processed, introduce a new
--force-process-system-include-paths option to shiboken and two new
CMake variables: PYSIDE_TREAT_QT_INCLUDE_DIRS_AS_NON_SYSTEM and
SHIBOKEN_FORCE_PROCESS_SYSTEM_INCLUDE_PATHS.

When PYSIDE_TREAT_QT_INCLUDE_DIRS_AS_NON_SYSTEM is set to true, the
build system will pass the Qt include dirs to
--force-process-system-include-paths to ensure the Qt headers are
processed and their types extracted.

Similarly SHIBOKEN_FORCE_PROCESS_SYSTEM_INCLUDE_PATHS can be passed a
list of extra dir paths when creating non-Qt related bindings.

Sysroots usually contain headers other than Qt headers, so there's a
chance that a non-Qt-related system header fails to be be parsed and
will fail the shiboken execution.
To avoid breaking setups that previously worked because of the issue
described above, the new options are opt-in rather than opt-out.

In case one such an issue is encountered, the solution would be to
copy / move the Qt headers into a separate location and specify the
new location as both an include path and a force process system include
path (in case the copied headers are still somewhere under the
sysroot).

Task-number: PYSIDE-1958
Change-Id: I1733478e9c6057f84de7864940c6150b378749cf
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>
(cherry picked from commit 7cc5c139482d735c49002649d26bb524c92cc86b)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

8 files changed, 71 insertions, 17 deletions

diff --git a/sources/pyside6/cmake/Macros/PySideModules.cmake b/sources/pyside6/cmake/Macros/PySideModules.cmake
index 463c60945..1288abd40 100644
--- a/sources/pyside6/cmake/Macros/PySideModules.cmake
+++ b/sources/pyside6/cmake/Macros/PySideModules.cmake
@@ -112,6 +112,39 @@ macro(create_pyside_module)
     # Transform the path separators into something shiboken understands.
     make_path(shiboken_include_dirs ${shiboken_include_dir_list})
 
+    set(force_process_system_include_paths_list "")
+    # When building against system Qt (as it happens with yocto / Boot2Qt), the Qt headers are
+    # considered system headers by clang_Location_isInSystemHeader, and thus shiboken will not
+    # process them.
+    #
+    # We do want to process them.
+    #
+    # Tell shiboken to consider them as special typesystem system include paths, which ensures
+    # the types are processed and extracted.
+    #
+    # This option is opt-in because it might cause problems if there are other system headers
+    # installed in the same location as the Qt ones, resulting in processing more non-Qt system
+    # types that might not be supported by shiboken.
+    if(PYSIDE_TREAT_QT_INCLUDE_DIRS_AS_NON_SYSTEM)
+        list(APPEND force_process_system_include_paths_list
+            ${qt_platform_includes}
+            ${qt_core_includes})
+    endif()
+
+    # Allow passing extra non system inlcude dirs.
+    if(SHIBOKEN_FORCE_PROCESS_SYSTEM_INCLUDE_PATHS)
+        list(APPEND force_process_system_include_paths_list
+            ${SHIBOKEN_FORCE_PROCESS_SYSTEM_INCLUDE_PATHS})
+    endif()
+
+    # Transform the path separators into something shiboken understands.
+    make_path(force_process_system_include_paths ${force_process_system_include_paths_list})
+
+    if(force_process_system_include_paths)
+        set(force_process_system_include_paths
+            "--force-process-system-include-paths=${force_process_system_include_paths}")
+    endif()
+
     get_filename_component(pyside_binary_dir ${CMAKE_CURRENT_BINARY_DIR} DIRECTORY)
 
     # Install module glue files.
@@ -136,6 +169,7 @@ macro(create_pyside_module)
         $<TARGET_FILE:Shiboken6::shiboken6>
         ${GENERATOR_EXTRA_FLAGS}
         "--include-paths=${shiboken_include_dirs}"
+        "${force_process_system_include_paths}"
         "--typesystem-paths=${pyside_binary_dir}${PATH_SEP}${pyside6_SOURCE_DIR}${PATH_SEP}${${module_TYPESYSTEM_PATH}}"
         --output-directory=${CMAKE_CURRENT_BINARY_DIR}
         --license-file=${CMAKE_CURRENT_SOURCE_DIR}/../licensecomment.txt
diff --git a/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp b/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp
index 8872701d7..53be95ca8 100644
--- a/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp
+++ b/sources/shiboken6/ApiExtractor/abstractmetabuilder.cpp
@@ -449,7 +449,7 @@ FileModelItem AbstractMetaBuilderPrivate::buildDom(QByteArrayList arguments,
                                                    unsigned clangFlags)
 {
     clang::Builder builder;
-    builder.setSystemIncludes(TypeDatabase::instance()->systemIncludes());
+    builder.setForceProcessSystemIncludes(TypeDatabase::instance()->forceProcessSystemIncludes());
     if (addCompilerSupportArguments) {
         if (level == LanguageLevel::Default)
             level = clang::emulatedCompilerLanguageLevel();
diff --git a/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.cpp b/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.cpp
index cb965e8db..bb072274c 100644
--- a/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.cpp
+++ b/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.cpp
@@ -216,8 +216,8 @@ public:
     ArgumentModelItem m_currentArgument;
     VariableModelItem m_currentField;
     TemplateTypeAliasModelItem m_currentTemplateTypeAlias;
-    QStringList m_systemIncludes; // files, like "memory"
-    QStringList m_systemIncludePaths; // paths, like "/usr/include/Qt/"
+    QStringList m_forceProcessSystemIncludes; // files, like "memory"
+    QStringList m_forceProcessSystemIncludePaths; // paths, like "/usr/include/Qt/"
     QString m_usingTypeRef; // Base classes in "using Base::member;"
     bool m_withinUsingDeclaration = false;
 
@@ -850,10 +850,16 @@ BuilderPrivate::SpecialSystemHeader
         break;
     }
 
-    if (m_systemIncludes.contains(baseName))
+    // When building against system Qt (as it happens with yocto / Boot2Qt), the Qt headers are
+    // considered system headers by clang_Location_isInSystemHeader, and shiboken will not
+    // process them. We need to explicitly process them by checking against the list of
+    // include paths that were passed to shiboken's --force-process-system-include-paths option
+    // or specified via the <system-include> xml tag.
+    if (m_forceProcessSystemIncludes.contains(baseName))
         return SpecialSystemHeader::WhiteListed;
 
-    if (std::any_of(m_systemIncludePaths.cbegin(), m_systemIncludePaths.cend(),
+    if (std::any_of(m_forceProcessSystemIncludePaths.cbegin(),
+                    m_forceProcessSystemIncludePaths.cend(),
                     [fileName](const QString &p) { return fileName.startsWith(p); })) {
         return SpecialSystemHeader::WhiteListedPath;
     }
@@ -866,13 +872,13 @@ bool Builder::visitLocation(const QString &fileName, LocationType locationType)
     return locationType != LocationType::System || d->visitHeader(fileName);
 }
 
-void Builder::setSystemIncludes(const QStringList &systemIncludes)
+void Builder::setForceProcessSystemIncludes(const QStringList &systemIncludes)
 {
     for (const auto &i : systemIncludes) {
         if (i.endsWith(u'/'))
-            d->m_systemIncludePaths.append(i);
+            d->m_forceProcessSystemIncludePaths.append(i);
         else
-            d->m_systemIncludes.append(i);
+            d->m_forceProcessSystemIncludes.append(i);
     }
 }
 
diff --git a/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.h b/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.h
index 035413547..b2ec6d304 100644
--- a/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.h
+++ b/sources/shiboken6/ApiExtractor/clangparser/clangbuilder.h
@@ -19,7 +19,7 @@ public:
     Builder();
     ~Builder();
 
-    void setSystemIncludes(const QStringList &systemIncludes);
+    void setForceProcessSystemIncludes(const QStringList &systemIncludes);
 
     bool visitLocation(const QString &fileName, LocationType locationType) const override;
 
diff --git a/sources/shiboken6/ApiExtractor/typedatabase.cpp b/sources/shiboken6/ApiExtractor/typedatabase.cpp
index 4473b1051..41bd0b0be 100644
--- a/sources/shiboken6/ApiExtractor/typedatabase.cpp
+++ b/sources/shiboken6/ApiExtractor/typedatabase.cpp
@@ -122,6 +122,9 @@ QList<OptionDescription> TypeDatabase::options()
         {u"-T<path>"_s, {} },
         {u"typesystem-paths="_s + OptionsParser::pathSyntax(),
          u"Paths used when searching for typesystems"_s},
+        {u"force-process-system-include-paths="_s + OptionsParser::pathSyntax(),
+         u"Include paths that are considered as system headers by the C++ parser, but should still "
+         "be processed to extract types (e.g. Qt include paths in a yocto sysroot)"_s},
         {u"keywords=keyword1[,keyword2,...]"_s,
          u"A comma-separated list of keywords for conditional typesystem parsing"_s},
     };
@@ -130,7 +133,7 @@ QList<OptionDescription> TypeDatabase::options()
 struct TypeDatabaseOptions
 {
     QStringList m_dropTypeEntries;
-    QStringList m_systemIncludes;
+    QStringList m_forceProcessSystemIncludes;
     QStringList m_typesystemKeywords;
     QStringList m_typesystemPaths;
     bool m_suppressWarnings = true;
@@ -202,6 +205,11 @@ bool TypeDatabaseOptionsParser::handleOption(const QString &key, const QString &
         return true;
     }
 
+    if (key == u"force-process-system-include-paths") {
+        m_options->m_forceProcessSystemIncludes += value.split(QDir::listSeparator());
+        return true;
+    }
+
     if (source == OptionSource::ProjectFile) {
         if (key == u"typesystem-path") {
             m_options->m_typesystemPaths += value;
@@ -447,14 +455,14 @@ IncludeList TypeDatabase::extraIncludes(const QString& className) const
     return typeEntry ? typeEntry->extraIncludes() :  IncludeList();
 }
 
-const QStringList &TypeDatabase::systemIncludes() const
+const QStringList &TypeDatabase::forceProcessSystemIncludes() const
 {
-    return d->m_systemIncludes;
+    return d->m_forceProcessSystemIncludes;
 }
 
-void TypeDatabase::addSystemInclude(const QString &name)
+void TypeDatabase::addForceProcessSystemInclude(const QString &name)
 {
-    d->m_systemIncludes.append(name);
+    d->m_forceProcessSystemIncludes.append(name);
 }
 
 // Add a lookup for the short name excluding inline namespaces
diff --git a/sources/shiboken6/ApiExtractor/typedatabase.h b/sources/shiboken6/ApiExtractor/typedatabase.h
index a0f490d69..d5adca324 100644
--- a/sources/shiboken6/ApiExtractor/typedatabase.h
+++ b/sources/shiboken6/ApiExtractor/typedatabase.h
@@ -89,8 +89,8 @@ public:
 
     IncludeList extraIncludes(const QString &className) const;
 
-    const QStringList &systemIncludes() const;
-    void addSystemInclude(const QString &name);
+    const QStringList &forceProcessSystemIncludes() const;
+    void addForceProcessSystemInclude(const QString &name);
 
     void addInlineNamespaceLookups(const NamespaceTypeEntryCPtr &n);
 
diff --git a/sources/shiboken6/ApiExtractor/typesystemparser.cpp b/sources/shiboken6/ApiExtractor/typesystemparser.cpp
index d9f550c7c..3615710a9 100644
--- a/sources/shiboken6/ApiExtractor/typesystemparser.cpp
+++ b/sources/shiboken6/ApiExtractor/typesystemparser.cpp
@@ -3097,7 +3097,7 @@ bool TypeSystemParser::parseSystemInclude(const ConditionalStreamReader &,
         m_error = msgMissingAttribute(fileNameAttribute());
         return false;
     }
-    TypeDatabase::instance()->addSystemInclude(attributes->takeAt(index).value().toString());
+    TypeDatabase::instance()->addForceProcessSystemInclude(attributes->takeAt(index).value().toString());
     return true;
 }
 
diff --git a/sources/shiboken6/doc/shibokengenerator.rst b/sources/shiboken6/doc/shibokengenerator.rst
index 09a812eea..dde5fca62 100644
--- a/sources/shiboken6/doc/shibokengenerator.rst
+++ b/sources/shiboken6/doc/shibokengenerator.rst
@@ -226,6 +226,12 @@ Options
 ``-F<path>, --framework-include-paths=<path>[:<path>:...]``
     Framework include paths used by the C++ parser
 
+.. _force-process-system-include-paths:
+
+``--force-process-system-include-paths=<path>[:<path>:...]``
+    Include paths that are considered as system headers by the C++ parser,
+    but should still be processed to extract types
+
 .. _language-level:
 
 ``--language-level=, -std=<level>``