shiboken/qtdocgenerator: Fall back to QtXmlPatterns when libxml/libxslt cannot be found

In PySide2, there is one XSL-based documentation modification which QXmlQuery
can handle.

However, due to QTBUG-66925, it is not a full replacement, since the
issue (XPATH specifying the number of the element to match) might be
useful for documentation modification.

Add QtXmlPatterns as fallback and modify the test accordingly.

Task-number: PYSIDE-363
Change-Id: I969fbe210725bb748d76028c6f542bae6b471a76
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>

5 files changed, 154 insertions, 39 deletions

diff --git a/sources/shiboken2/ApiExtractor/CMakeLists.txt b/sources/shiboken2/ApiExtractor/CMakeLists.txt
index f2af51c02..4355e32ef 100644
--- a/sources/shiboken2/ApiExtractor/CMakeLists.txt
+++ b/sources/shiboken2/ApiExtractor/CMakeLists.txt
@@ -5,11 +5,13 @@ find_package(LibXslt 1.1.19)
 
 option(DISABLE_DOCSTRINGS "Disable documentation extraction." FALSE)
 
+set (USE_LIBXSLT 0)
 if (NOT DISABLE_DOCSTRINGS)
-    if (NOT LIBXSLT_FOUND OR NOT LIBXML2_FOUND)
-        set(DISABLE_DOCSTRINGS TRUE CACHE BOOL "Disable doc strings" PARENT_SCOPE)
-        set(DISABLE_DOCSTRINGS TRUE)
-        message(WARNING "libxslt and/or libxml not found, disabling support for doc strings!")
+    if (LIBXSLT_FOUND AND LIBXML2_FOUND)
+        add_definitions(-DHAVE_LIBXSLT)
+        set (USE_LIBXSLT 1)
+    else()
+        message(WARNING "libxslt and/or libxml not found, falling back to QtXmlPatterns (QTBUG-66925)")
     endif()
 endif()
 
@@ -65,8 +67,12 @@ if (NOT DISABLE_DOCSTRINGS)
         doxygenparser.cpp
         qtdocparser.cpp
     )
-    set(APIEXTRACTOR_EXTRA_INCLUDES ${APIEXTRACTOR_EXTRA_INCLUDES} ${LIBXSLT_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR})
-    set(APIEXTRACTOR_EXTRA_LIBRARIES ${APIEXTRACTOR_EXTRA_LIBRARIES} ${LIBXSLT_LIBRARIES} ${LIBXML2_LIBRARIES})
+    set(APIEXTRACTOR_EXTRA_INCLUDES ${APIEXTRACTOR_EXTRA_INCLUDES})
+    set(APIEXTRACTOR_EXTRA_LIBRARIES ${APIEXTRACTOR_EXTRA_LIBRARIES})
+    if (USE_LIBXSLT)
+        list(APPEND APIEXTRACTOR_EXTRA_INCLUDES ${LIBXSLT_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR})
+        list(APPEND APIEXTRACTOR_EXTRA_LIBRARIES ${LIBXSLT_LIBRARIES} ${LIBXML2_LIBRARIES})
+    endif()
 endif()
 
 set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}" CACHE PATH "The subdirectory relative to the install prefix where libraries will be installed (default is /lib${LIB_SUFFIX})" FORCE)
diff --git a/sources/shiboken2/ApiExtractor/docparser.cpp b/sources/shiboken2/ApiExtractor/docparser.cpp
index 1d1783f05..d24e09acd 100644
--- a/sources/shiboken2/ApiExtractor/docparser.cpp
+++ b/sources/shiboken2/ApiExtractor/docparser.cpp
@@ -27,6 +27,7 @@
 ****************************************************************************/
 #include "docparser.h"
 #include "abstractmetalang.h"
+#include "reporthandler.h"
 #include "typesystem.h"
 #include <QtCore/QDebug>
 #include <QtCore/QDir>
@@ -35,12 +36,18 @@
 #include <QBuffer>
 
 #include <cstdlib>
-#include <libxslt/xsltutils.h>
-#include <libxslt/transform.h>
+#ifdef HAVE_LIBXSLT
+#  include <libxslt/xsltutils.h>
+#  include <libxslt/transform.h>
+#endif
+
+#include <algorithm>
 
 DocParser::DocParser()
 {
+#ifdef HAVE_LIBXSLT
     xmlSubstituteEntitiesDefault(1);
+#endif
 }
 
 DocParser::~DocParser()
@@ -142,6 +149,7 @@ QString DocParser::msgCannotFindDocumentation(const QString &fileName,
                                       query);
 }
 
+#ifdef HAVE_LIBXSLT
 namespace
 {
 
@@ -170,23 +178,57 @@ struct XslResources
 };
 
 } // namespace
+#endif // HAVE_LIBXSLT
 
-QString DocParser::applyDocModifications(const DocModificationList& mods, const QString& xml) const
+static inline bool isXpathDocModification(const DocModification &mod)
 {
-    if (mods.isEmpty() || xml.isEmpty())
-        return xml;
+    return mod.mode() == TypeSystem::DocModificationXPathReplace;
+}
 
-    bool hasXPathBasedModification = false;
+QString msgXpathDocModificationError(const DocModificationList& mods,
+                                     const QString &what)
+{
+    QString result;
+    QTextStream str(&result);
+    str << "Error when applying modifications (";
     for (const DocModification &mod : mods) {
-        if (mod.mode() == TypeSystem::DocModificationXPathReplace) {
-            hasXPathBasedModification = true;
-            break;
+        if (isXpathDocModification(mod)) {
+            str << '"' << mod.xpath() << "\" -> \"";
+            const QString simplified = mod.code().simplified();
+            if (simplified.size() > 20)
+                str << simplified.leftRef(20) << "...";
+            else
+                str << simplified;
+            str << '"';
         }
     }
+    str << "): " << what;
+    return result;
+}
 
-    if (!hasXPathBasedModification)
+QString DocParser::applyDocModifications(const DocModificationList& mods, const QString& xml) const
+{
+    if (mods.isEmpty() || xml.isEmpty()
+        || !std::any_of(mods.cbegin(), mods.cend(), isXpathDocModification)) {
         return xml;
+    }
+#ifdef HAVE_LIBXSLT
+    const QString result = applyDocModificationsLibXsl(mods, xml);
+#else
+    const QString result = applyDocModificationsQt(mods, xml);
+#endif
+    if (result == xml) {
+        const QString message = QLatin1String("Query did not result in any modifications to \"")
+            + xml + QLatin1Char('"');
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgXpathDocModificationError(mods, message)));
+    }
+    return result;
+}
 
+QString DocParser::applyDocModificationsLibXsl(const DocModificationList& mods, const QString& xml) const
+{
+#ifdef HAVE_LIBXSLT
     QString xsl = QLatin1String("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
                                 "<xsl:transform version=\"1.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\">\n"
                                 "<xsl:template match=\"/\">\n"
@@ -200,7 +242,7 @@ QString DocParser::applyDocModifications(const DocModificationList& mods, const
                                 "</xsl:template>\n"
                                );
     for (const DocModification &mod : mods) {
-        if (mod.mode() == TypeSystem::DocModificationXPathReplace) {
+        if (isXpathDocModification(mod)) {
             QString xpath = mod.xpath();
             xpath.replace(QLatin1Char('"'), QLatin1String("&quot;"));
             xsl += QLatin1String("<xsl:template match=\"")
@@ -240,8 +282,54 @@ QString DocParser::applyDocModifications(const DocModificationList& mods, const
     } else {
         result = xml;
     }
-
-    Q_ASSERT(result != xml);
     return result.trimmed();
+#else // HAVE_LIBXSLT
+    Q_UNUSED(mods)
+    return xml;
+#endif // !HAVE_LIBXSLT
 }
 
+QString DocParser::applyDocModificationsQt(const DocModificationList& mods, const QString& xml) const
+{
+    const char xslPrefix[] =
+R"(<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0">
+    <xsl:template match="/">
+        <xsl:apply-templates/>\n"
+    </xsl:template>
+    <xsl:template match="*">
+        <xsl:copy>
+            <xsl:copy-of select="@*"/>
+        <xsl:apply-templates/>
+    </xsl:copy>
+    </xsl:template>
+)";
+
+    QString xsl = QLatin1String(xslPrefix);
+    for (const DocModification &mod : mods) {
+        if (isXpathDocModification(mod)) {
+            QString xpath = mod.xpath();
+            xpath.replace(QLatin1Char('"'), QLatin1String("&quot;"));
+            xsl += QLatin1String("<xsl:template match=\"")
+                   + xpath + QLatin1String("\">")
+                   + mod.code() + QLatin1String("</xsl:template>\n");
+        }
+    }
+    xsl += QLatin1String("</xsl:stylesheet>");
+
+    QXmlQuery query(QXmlQuery::XSLT20);
+    query.setFocus(xml);
+    query.setQuery(xsl);
+    if (!query.isValid()) {
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgXpathDocModificationError(mods, QLatin1String("Invalid query."))));
+        return xml;
+    }
+    QString result;
+    if (!query.evaluateTo(&result)) {
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgXpathDocModificationError(mods, QLatin1String("evaluate() failed."))));
+        return xml;
+    }
+    return result.trimmed();
+}
diff --git a/sources/shiboken2/ApiExtractor/docparser.h b/sources/shiboken2/ApiExtractor/docparser.h
index 1770815b0..1da58fc11 100644
--- a/sources/shiboken2/ApiExtractor/docparser.h
+++ b/sources/shiboken2/ApiExtractor/docparser.h
@@ -143,6 +143,8 @@ private:
 
     QString execXQuery(QXmlQuery& xquery, const QString& query) const;
     QString applyDocModifications(const DocModificationList& mods, const QString& xml) const;
+    QString applyDocModificationsLibXsl(const DocModificationList& mods, const QString& xml) const;
+    QString applyDocModificationsQt(const DocModificationList& mods, const QString& xml) const;
 };
 
 #endif // DOCPARSER_H
diff --git a/sources/shiboken2/ApiExtractor/tests/a.xml b/sources/shiboken2/ApiExtractor/tests/a.xml
index 1c6d62a17..3c09d3800 100644
--- a/sources/shiboken2/ApiExtractor/tests/a.xml
+++ b/sources/shiboken2/ApiExtractor/tests/a.xml
@@ -1,9 +1,10 @@
 <?xml version="1.0" ?>
-
+<!-- Sample for testModifyDocumentation -->
 <WebXML>
     <document>
         <class name="A">
             <description>oi
+                <brief>Brief description</brief>
                 <para>Paragraph number 1</para>
                 <para>Paragraph number 2</para>
                 <para>Paragraph number 3</para>
diff --git a/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp b/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp
index d56186479..cc95186ef 100644
--- a/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp
+++ b/sources/shiboken2/ApiExtractor/tests/testmodifydocumentation.cpp
@@ -38,35 +38,53 @@
 void TestModifyDocumentation::testModifyDocumentation()
 {
     const char* cppCode ="struct B { void b(); }; class A {};\n";
-    const char* xmlCode = "<typesystem package=\"Foo\">\n\
-    <value-type name='B'>\n\
-        <modify-function signature='b()' remove='all'/>\n\
-    </value-type>\n\
-    <value-type name='A'>\n\
-    <modify-documentation xpath='description/para[3]'>\n\
-    &lt;para>Some changed contents here&lt;/para>\n\
-    </modify-documentation>\n\
-    </value-type>\n\
-    </typesystem>\n";
+    const char xmlCode[] =
+R"(<typesystem package="Foo">
+    <value-type name='B'>
+        <modify-function signature='b()' remove='all'/>
+    </value-type>
+    <value-type name='A'>
+    <modify-documentation xpath='description/brief'>&lt;brief>Modified Brief&lt;/brief></modify-documentation>
+    <modify-documentation xpath='description/para[3]'>&lt;para>Some changed contents here&lt;/para></modify-documentation>
+    </value-type>
+</typesystem>
+)";
     QScopedPointer<AbstractMetaBuilder> builder(TestUtil::parse(cppCode, xmlCode));
     QVERIFY(!builder.isNull());
     AbstractMetaClass *classA = AbstractMetaClass::findClass(builder->classes(), QLatin1String("A"));
     QVERIFY(classA);
     DocModificationList docMods = classA->typeEntry()->docModifications();
-    QCOMPARE(docMods.count(), 1);
-    QCOMPARE(docMods[0].code().trimmed(), QLatin1String("<para>Some changed contents here</para>"));
+    QCOMPARE(docMods.count(), 2);
+    QCOMPARE(docMods[0].code().trimmed(), QLatin1String("<brief>Modified Brief</brief>"));
     QCOMPARE(docMods[0].signature(), QString());
+    QCOMPARE(docMods[1].code().trimmed(), QLatin1String("<para>Some changed contents here</para>"));
+    QCOMPARE(docMods[1].signature(), QString());
     QtDocParser docParser;
     docParser.setDocumentationDataDirectory(QDir::currentPath());
     docParser.fillDocumentation(classA);
 
-    QVERIFY(!classA->documentation().value().trimmed().isEmpty());
-    QCOMPARE(classA->documentation().value(), QLatin1String("<?xml version=\"1.0\"?>\n\
-<description>oi\n\
-                <para>Paragraph number 1</para>\n\
-    <para>Paragraph number 2</para>\n\
-    <para>Some changed contents here</para>\n\
-</description>"));
+    const QString actualDocSimplified = classA->documentation().value().simplified();
+    QVERIFY(!actualDocSimplified.isEmpty());
+
+const char expectedDoc[] =
+R"(<?xml version="1.0"?>
+<description>oi
+<brief>Modified Brief</brief>
+<para>Paragraph number 1</para>
+<para>Paragraph number 2</para>
+<para>Some changed contents here</para>
+</description>
+)";
+    const QString expectedDocSimplified = QString::fromLatin1(expectedDoc).simplified();
+    // Check whether the first modification worked.
+    QVERIFY(actualDocSimplified.contains(QLatin1String("Modified Brief")));
+
+#ifndef HAVE_LIBXSLT
+    // QtXmlPatterns is unable to handle para[3] in style sheets,
+    // this only works in its XPath search.
+    QEXPECT_FAIL("", "QtXmlPatterns cannot handle para[3] (QTBUG-66925)", Abort);
+#endif
+    QCOMPARE(actualDocSimplified, expectedDocSimplified);
 }
 
 // We expand QTEST_MAIN macro but using QCoreApplication instead of QApplication