Merge remote-tracking branch 'origin/5.9' into dev

Change-Id: I622426aa55762e32f40995b72d49a3f966103b36

11 files changed, 179 insertions, 67 deletions

diff --git a/examples/scriptableapplication/CMakeLists.txt b/examples/scriptableapplication/CMakeLists.txt
index 40e29346a..6641d77c6 100644
--- a/examples/scriptableapplication/CMakeLists.txt
+++ b/examples/scriptableapplication/CMakeLists.txt
@@ -58,6 +58,15 @@ foreach(INCLUDE_DIR ${QT_INCLUDE_DIRS})
     list(APPEND INCLUDES "-I${INCLUDE_DIR}")
 endforeach()
 
+# On macOS, check if Qt is a framework build. This affects how include paths should be handled.
+get_target_property(QtCore_is_framework Qt5::Core FRAMEWORK)
+if (QtCore_is_framework)
+    get_target_property(qt_core_library_location Qt5::Core LOCATION)
+    get_filename_component(qt_core_library_location_dir "${qt_core_library_location}" DIRECTORY)
+    get_filename_component(lib_dir "${qt_core_library_location_dir}/../" ABSOLUTE)
+    list(APPEND INCLUDES "--framework-include-paths=${lib_dir}")
+endif()
+
 # Set up the options to pass to shiboken.
 set(WRAPPED_HEADER ${CMAKE_SOURCE_DIR}/wrappedclasses.h)
 set(TYPESYSTEM_FILE ${CMAKE_SOURCE_DIR}/scriptableapplication.xml)
diff --git a/examples/scriptableapplication/scriptableapplication.pro b/examples/scriptableapplication/scriptableapplication.pro
index 9f888f6a0..bcb05c115 100644
--- a/examples/scriptableapplication/scriptableapplication.pro
+++ b/examples/scriptableapplication/scriptableapplication.pro
@@ -12,6 +12,9 @@ TYPESYSTEM_FILE = scriptableapplication.xml
 QT_INCLUDEPATHS = -I$$[QT_INSTALL_HEADERS] -I$$[QT_INSTALL_HEADERS]/QtCore \
     -I$$[QT_INSTALL_HEADERS]/QtGui -I$$[QT_INSTALL_HEADERS]/QtWidgets
 
+# On macOS, check if Qt is a framework build. This affects how include paths should be handled.
+qtConfig(framework): QT_INCLUDEPATHS += --framework-include-paths=$$[QT_INSTALL_LIBS]
+
 SHIBOKEN_OPTIONS = --generator-set=shiboken --enable-parent-ctor-heuristic \
     --enable-pyside-extensions --enable-return-value-heuristic --use-isnull-as-nb_nonzero \
     $$QT_INCLUDEPATHS -I$$PWD -T$$PWD -T$$PYSIDE2/typesystems --output-directory=$$OUT_PWD
diff --git a/setup.py b/setup.py
index ccce48e9c..559e9286f 100644
--- a/setup.py
+++ b/setup.py
@@ -161,9 +161,7 @@ __version__ = get_package_version()
 containedModules = ['shiboken2', 'pyside2', 'pyside2-tools']
 
 # Git submodules: ["submodule_name", "location_relative_to_sources_folder"]
-submodules = [["pyside2-tools"],
-              ["pyside2-examples"],
-              ["wiki", ".."]]
+submodules = [["pyside2-tools"]]
 
 pyside_package_dir_name = "pyside_package"
 
@@ -223,14 +221,6 @@ from utils import find_files_using_glob
 
 from textwrap import dedent
 
-# guess a close folder name for extensions
-def get_extension_folder(ext):
-    candidates = list(containedModules)
-    for gitModules in submodules:
-        candidates.append(gitModules[0])
-    folder = difflib.get_close_matches(ext, candidates)[0]
-    return folder
-
 # make sure that setup.py is run with an allowed python version
 def check_allowed_python_version():
     import re
@@ -368,12 +358,6 @@ if os.path.dirname(this_file):
 def is_debug_python():
     return getattr(sys, "gettotalrefcount", None) is not None
 
-if OPTION_NOEXAMPLES:
-    # Remove pyside2-examples from submodules so they will not be included.
-    for idx, item in enumerate(submodules):
-        if item[0].startswith('pyside2-examples'):
-            del submodules[idx]
-
 # Return a prefix suitable for the _install/_build directory
 def prefix():
     virtualEnvName = os.environ.get('VIRTUAL_ENV', None)
@@ -427,7 +411,12 @@ def get_qt_version():
     qt_version = qtinfo.version
 
     if not qt_version:
-        log.error("Failed to query the Qt version with qmake %s" % self.qtinfo.qmake_command)
+        log.error("Failed to query the Qt version with qmake {0}".format(self.qtinfo.qmake_command))
+        sys.exit(1)
+
+    if LooseVersion(qtinfo.version) < LooseVersion("5.7"):
+        m = "Incompatible Qt version detected: {0}. A Qt version >= 5.7 is required."
+        log.error(m.format(qt_version))
         sys.exit(1)
 
     return qt_version
@@ -991,7 +980,6 @@ class pyside_build(_build):
 
     def build_extension(self, extension):
         # calculate the subrepos folder name
-        folder = get_extension_folder(extension)
 
         log.info("Building module %s..." % extension)
 
@@ -1019,7 +1007,7 @@ class pyside_build(_build):
             os.makedirs(module_build_dir)
         os.chdir(module_build_dir)
 
-        module_src_dir = os.path.join(self.sources_dir, folder)
+        module_src_dir = os.path.join(self.sources_dir, extension)
 
         # Build module
         cmake_cmd = [
diff --git a/sources/pyside2/CMakeLists.txt b/sources/pyside2/CMakeLists.txt
index 4db611f0d..41c62c67a 100644
--- a/sources/pyside2/CMakeLists.txt
+++ b/sources/pyside2/CMakeLists.txt
@@ -96,7 +96,7 @@ message(STATUS "PYTHON_CONFIG_SUFFIX:         ${PYTHON_CONFIG_SUFFIX}")
 message(STATUS "PYTHON_SHARED_LIBRARY_SUFFIX: ${PYTHON_SHARED_LIBRARY_SUFFIX}")
 
 find_package(Shiboken2 2.0.0 REQUIRED)
-find_package(Qt5 5.6.0 REQUIRED COMPONENTS Core)
+find_package(Qt5 5.7 REQUIRED COMPONENTS Core)
 add_definitions(${Qt5Core_DEFINITIONS})
 
 find_file(GL_H "gl.h" PATH_SUFFIXES "GL")
diff --git a/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml b/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml
index 11111d96c..554111e35 100644
--- a/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml
+++ b/sources/pyside2/PySide2/QtCore/typesystem_core_common.xml
@@ -2374,9 +2374,6 @@
     <enum-type name="UrlFormattingOption" />
     <enum-type name="UserInputResolutionOption" flags="UserInputResolutionOptions"/>
     <enum-type name="ParsingMode"/>
-    <modify-documentation xpath='description/para[3]'>
-        &lt;para>URLs can be represented in two forms: encoded or unencoded. The unencoded representation is suitable for showing to users, but the encoded representation is typically what you would send to a web server. For example, the unencoded URL &quot;http://bühler.example.com&quot; would be sent to the server as &quot;http://xn--bhler-kva.example.com/List%20of%20applicants.xml&quot;.&lt;/para>
-    </modify-documentation>
     <extra-includes>
       <include file-name="QStringList" location="global"/>
     </extra-includes>
@@ -3989,9 +3986,9 @@
     <modify-documentation xpath='description/code'>
         &lt;code>machine = QStateMachine()
 
-s1 = new QState()
-s11 = new QState(s1)
-s12 = new QState(s1)
+s1 = QState()
+s11 = QState(s1)
+s12 = QState(s1)
 
 s1h = QHistoryState(s1)
 s1h.setDefaultState(s11)
diff --git a/sources/shiboken2/ApiExtractor/CMakeLists.txt b/sources/shiboken2/ApiExtractor/CMakeLists.txt
index f2af51c02..4355e32ef 100644
--- a/sources/shiboken2/ApiExtractor/CMakeLists.txt
+++ b/sources/shiboken2/ApiExtractor/CMakeLists.txt
@@ -5,11 +5,13 @@ find_package(LibXslt 1.1.19)
 
 option(DISABLE_DOCSTRINGS "Disable documentation extraction." FALSE)
 
+set (USE_LIBXSLT 0)
 if (NOT DISABLE_DOCSTRINGS)
-    if (NOT LIBXSLT_FOUND OR NOT LIBXML2_FOUND)
-        set(DISABLE_DOCSTRINGS TRUE CACHE BOOL "Disable doc strings" PARENT_SCOPE)
-        set(DISABLE_DOCSTRINGS TRUE)
-        message(WARNING "libxslt and/or libxml not found, disabling support for doc strings!")
+    if (LIBXSLT_FOUND AND LIBXML2_FOUND)
+        add_definitions(-DHAVE_LIBXSLT)
+        set (USE_LIBXSLT 1)
+    else()
+        message(WARNING "libxslt and/or libxml not found, falling back to QtXmlPatterns (QTBUG-66925)")
     endif()
 endif()
 
@@ -65,8 +67,12 @@ if (NOT DISABLE_DOCSTRINGS)
         doxygenparser.cpp
         qtdocparser.cpp
     )
-    set(APIEXTRACTOR_EXTRA_INCLUDES ${APIEXTRACTOR_EXTRA_INCLUDES} ${LIBXSLT_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR})
-    set(APIEXTRACTOR_EXTRA_LIBRARIES ${APIEXTRACTOR_EXTRA_LIBRARIES} ${LIBXSLT_LIBRARIES} ${LIBXML2_LIBRARIES})
+    set(APIEXTRACTOR_EXTRA_INCLUDES ${APIEXTRACTOR_EXTRA_INCLUDES})
+    set(APIEXTRACTOR_EXTRA_LIBRARIES ${APIEXTRACTOR_EXTRA_LIBRARIES})
+    if (USE_LIBXSLT)
+        list(APPEND APIEXTRACTOR_EXTRA_INCLUDES ${LIBXSLT_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR})
+        list(APPEND APIEXTRACTOR_EXTRA_LIBRARIES ${LIBXSLT_LIBRARIES} ${LIBXML2_LIBRARIES})
+    endif()
 endif()
 
 set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}" CACHE PATH "The subdirectory relative to the install prefix where libraries will be installed (default is /lib${LIB_SUFFIX})" FORCE)
diff --git a/sources/shiboken2/ApiExtractor/docparser.cpp b/sources/shiboken2/ApiExtractor/docparser.cpp
index 1d1783f05..d24e09acd 100644
--- a/sources/shiboken2/ApiExtractor/docparser.cpp
+++ b/sources/shiboken2/ApiExtractor/docparser.cpp
@@ -27,6 +27,7 @@
 ****************************************************************************/
 #include "docparser.h"
 #include "abstractmetalang.h"
+#include "reporthandler.h"
 #include "typesystem.h"
 #include <QtCore/QDebug>
 #include <QtCore/QDir>
@@ -35,12 +36,18 @@
 #include <QBuffer>
 
 #include <cstdlib>
-#include <libxslt/xsltutils.h>
-#include <libxslt/transform.h>
+#ifdef HAVE_LIBXSLT
+#  include <libxslt/xsltutils.h>
+#  include <libxslt/transform.h>
+#endif
+
+#include <algorithm>
 
 DocParser::DocParser()
 {
+#ifdef HAVE_LIBXSLT
     xmlSubstituteEntitiesDefault(1);
+#endif
 }
 
 DocParser::~DocParser()
@@ -142,6 +149,7 @@ QString DocParser::msgCannotFindDocumentation(const QString &fileName,
                                       query);
 }
 
+#ifdef HAVE_LIBXSLT
 namespace
 {
 
@@ -170,23 +178,57 @@ struct XslResources
 };
 
 } // namespace
+#endif // HAVE_LIBXSLT
 
-QString DocParser::applyDocModifications(const DocModificationList& mods, const QString& xml) const
+static inline bool isXpathDocModification(const DocModification &mod)
 {
-    if (mods.isEmpty() || xml.isEmpty())
-        return xml;
+    return mod.mode() == TypeSystem::DocModificationXPathReplace;
+}
 
-    bool hasXPathBasedModification = false;
+QString msgXpathDocModificationError(const DocModificationList& mods,
+                                     const QString &what)
+{
+    QString result;
+    QTextStream str(&result);
+    str << "Error when applying modifications (";
     for (const DocModification &mod : mods) {
-        if (mod.mode() == TypeSystem::DocModificationXPathReplace) {
-            hasXPathBasedModification = true;
-            break;
+        if (isXpathDocModification(mod)) {
+            str << '"' << mod.xpath() << "\" -> \"";
+            const QString simplified = mod.code().simplified();
+            if (simplified.size() > 20)
+                str << simplified.leftRef(20) << "...";
+            else
+                str << simplified;
+            str << '"';
         }
     }
+    str << "): " << what;
+    return result;
+}
 
-    if (!hasXPathBasedModification)
+QString DocParser::applyDocModifications(const DocModificationList& mods, const QString& xml) const
+{
+    if (mods.isEmpty() || xml.isEmpty()
+        || !std::any_of(mods.cbegin(), mods.cend(), isXpathDocModification)) {
         return xml;
+    }
+#ifdef HAVE_LIBXSLT
+    const QString result = applyDocModificationsLibXsl(mods, xml);
+#else
+    const QString result = applyDocModificationsQt(mods, xml);
+#endif
+    if (result == xml) {
+        const QString message = QLatin1String("Query did not result in any modifications to \"")
+            + xml + QLatin1Char('"');
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgXpathDocModificationError(mods, message)));
+    }
+    return result;
+}
 
+QString DocParser::applyDocModificationsLibXsl(const DocModificationList& mods, const QString& xml) const
+{
+#ifdef HAVE_LIBXSLT
     QString xsl = QLatin1String("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
                                 "<xsl:transform version=\"1.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"
                                 "<xsl:template match=\"/\">\n"
@@ -200,7 +242,7 @@ QString DocParser::applyDocModifications(const DocModificationList& mods, const
                                 "</xsl:template>\n"
                                );
     for (const DocModification &mod : mods) {
-        if (mod.mode() == TypeSystem::DocModificationXPathReplace) {
+        if (isXpathDocModification(mod)) {
             QString xpath = mod.xpath();
             xpath.replace(QLatin1Char('"'), QLatin1String("&quot;"));
             xsl += QLatin1String("<xsl:template match=\"")
@@ -240,8 +282,54 @@ QString DocParser::applyDocModifications(const DocModificationList& mods, const
     } else {
         result = xml;
     }
-
-    Q_ASSERT(result != xml);
     return result.trimmed();
+#else // HAVE_LIBXSLT
+    Q_UNUSED(mods)
+    return xml;
+#endif // !HAVE_LIBXSLT
 }
 
+QString DocParser::applyDocModificationsQt(const DocModificationList& mods, const QString& xml) const
+{
+    const char xslPrefix[] =
+R"(<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0">
+    <xsl:template match="/">
+        <xsl:apply-templates/>\n"
+    </xsl:template>
+    <xsl:template match="*">
+        <xsl:copy>
+            <xsl:copy-of select="@*"/>
+        <xsl:apply-templates/>
+    </xsl:copy>
+    </xsl:template>
+)";
+
+    QString xsl = QLatin1String(xslPrefix);
+    for (const DocModification &mod : mods) {
+        if (isXpathDocModification(mod)) {
+            QString xpath = mod.xpath();
+            xpath.replace(QLatin1Char('"'), QLatin1String("&quot;"));
+            xsl += QLatin1String("<xsl:template match=\"")
+                   + xpath + QLatin1String("\">")
+                   + mod.code() + QLatin1String("</xsl:template>\n");
+        }
+    }
+    xsl += QLatin1String("</xsl:stylesheet>");
+
+    QXmlQuery query(QXmlQuery::XSLT20);
+    query.setFocus(xml);
+    query.setQuery(xsl);
+    if (!query.isValid()) {
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgXpathDocModificationError(mods, QLatin1String("Invalid query."))));
+        return xml;
+    }
+    QString result;
+    if (!query.evaluateTo(&result)) {
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgXpathDocModificationError(mods, QLatin1String("evaluate() failed."))));
+        return xml;
+    }
+    return result.trimmed();
+}
diff --git a/sources/shiboken2/ApiExtractor/docparser.h b/sources/shiboken2/ApiExtractor/docparser.h
index 1770815b0..1da58fc11 100644
--- a/sources/shiboken2/ApiExtractor/docparser.h
+++ b/sources/shiboken2/ApiExtractor/docparser.h
@@ -143,6 +143,8 @@ private:
 
     QString execXQuery(QXmlQuery& xquery, const QString& query) const;
     QString applyDocModifications(const DocModificationList& mods, const QString& xml) const;
+    QString applyDocModificationsLibXsl(const DocModificationList& mods, const QString& xml) const;
+    QString applyDocModificationsQt(const DocModificationList& mods, const QString& xml) const;
 };
 
 #endif // DOCPARSER_H
diff --git a/sources/shiboken2/ApiExtractor/tests/a.xml b/sources/shiboken2/ApiExtractor/tests/a.xml
index 1c6d62a17..3c09d3800 100644
--- a/sources/shiboken2/ApiExtractor/tests/a.xml
+++ b/sources/shiboken2/ApiExtractor/tests/a.xml
@@ -1,9 +1,10 @@
 <?xml version="1.0" ?>
-
+<!-- Sample for testModifyDocumentation -->
 <WebXML>
     <document>
         <class name="A">
             <description>oi
+                <brief>Brief description</brief>
                 <para>Paragraph number 1</para>
                 <para>Paragraph number 2</para>
                 <para>Paragraph number 3</para>
diff --git a/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp b/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp
index d56186479..cc95186ef 100644
--- a/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp
+++ b/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp
@@ -38,35 +38,53 @@
 void TestModifyDocumentation::testModifyDocumentation()
 {
     const char* cppCode ="struct B { void b(); }; class A {};\n";
-    const char* xmlCode = "<typesystem package=\"Foo\">\n\
-    <value-type name='B'>\n\
-        <modify-function signature='b()' remove='all'/>\n\
-    </value-type>\n\
-    <value-type name='A'>\n\
-    <modify-documentation xpath='description/para[3]'>\n\
-    &lt;para>Some changed contents here&lt;/para>\n\
-    </modify-documentation>\n\
-    </value-type>\n\
-    </typesystem>\n";
+    const char xmlCode[] =
+R"(<typesystem package="Foo">
+    <value-type name='B'>
+        <modify-function signature='b()' remove='all'/>
+    </value-type>
+    <value-type name='A'>
+    <modify-documentation xpath='description/brief'>&lt;brief>Modified Brief&lt;/brief></modify-documentation>
+    <modify-documentation xpath='description/para[3]'>&lt;para>Some changed contents here&lt;/para></modify-documentation>
+    </value-type>
+</typesystem>
+)";
     QScopedPointer<AbstractMetaBuilder> builder(TestUtil::parse(cppCode, xmlCode));
     QVERIFY(!builder.isNull());
     AbstractMetaClass *classA = AbstractMetaClass::findClass(builder->classes(), QLatin1String("A"));
     QVERIFY(classA);
     DocModificationList docMods = classA->typeEntry()->docModifications();
-    QCOMPARE(docMods.count(), 1);
-    QCOMPARE(docMods[0].code().trimmed(), QLatin1String("<para>Some changed contents here</para>"));
+    QCOMPARE(docMods.count(), 2);
+    QCOMPARE(docMods[0].code().trimmed(), QLatin1String("<brief>Modified Brief</brief>"));
     QCOMPARE(docMods[0].signature(), QString());
+    QCOMPARE(docMods[1].code().trimmed(), QLatin1String("<para>Some changed contents here</para>"));
+    QCOMPARE(docMods[1].signature(), QString());
     QtDocParser docParser;
     docParser.setDocumentationDataDirectory(QDir::currentPath());
     docParser.fillDocumentation(classA);
 
-    QVERIFY(!classA->documentation().value().trimmed().isEmpty());
-    QCOMPARE(classA->documentation().value(), QLatin1String("<?xml version=\"1.0\"?>\n\
-<description>oi\n\
-                <para>Paragraph number 1</para>\n\
-    <para>Paragraph number 2</para>\n\
-    <para>Some changed contents here</para>\n\
-</description>"));
+    const QString actualDocSimplified = classA->documentation().value().simplified();
+    QVERIFY(!actualDocSimplified.isEmpty());
+
+const char expectedDoc[] =
+R"(<?xml version="1.0"?>
+<description>oi
+<brief>Modified Brief</brief>
+<para>Paragraph number 1</para>
+<para>Paragraph number 2</para>
+<para>Some changed contents here</para>
+</description>
+)";
+    const QString expectedDocSimplified = QString::fromLatin1(expectedDoc).simplified();
+    // Check whether the first modification worked.
+    QVERIFY(actualDocSimplified.contains(QLatin1String("Modified Brief")));
+
+#ifndef HAVE_LIBXSLT
+    // QtXmlPatterns is unable to handle para[3] in style sheets,
+    // this only works in its XPath search.
+    QEXPECT_FAIL("", "QtXmlPatterns cannot handle para[3] (QTBUG-66925)", Abort);
+#endif
+    QCOMPARE(actualDocSimplified, expectedDocSimplified);
 }
 
 // We expand QTEST_MAIN macro but using QCoreApplication instead of QApplication
diff --git a/sources/shiboken2/CMakeLists.txt b/sources/shiboken2/CMakeLists.txt
index ccabc72e3..d9e110d6d 100644
--- a/sources/shiboken2/CMakeLists.txt
+++ b/sources/shiboken2/CMakeLists.txt
@@ -7,7 +7,7 @@ cmake_policy(VERSION 3.1)
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/Modules/
                       ${CMAKE_MODULE_PATH})
 
-find_package(Qt5 REQUIRED COMPONENTS Core Xml XmlPatterns)
+find_package(Qt5 5.7 REQUIRED COMPONENTS Core Xml XmlPatterns)
 
 add_definitions(${Qt5Core_DEFINITIONS})