shiboken6: Add a processing instruction for defining entities to typesystem parsing

Add a caching proxy entity resolver to ConditionalStreamReader
and a processing instruction for defining entities.

Remove caching from the entity resolver of the type system parser.

Task-number: PYSIDE-1646
Change-Id: Ibdccd6b57bf19586f3e1ef314a5e65daf2b4f566
Reviewed-by: Christian Tismer <tismer@stackless.com>

6 files changed, 170 insertions, 16 deletions

diff --git a/sources/shiboken6/ApiExtractor/conditionalstreamreader.cpp b/sources/shiboken6/ApiExtractor/conditionalstreamreader.cpp
index c3c312eeb..20209769a 100644
--- a/sources/shiboken6/ApiExtractor/conditionalstreamreader.cpp
+++ b/sources/shiboken6/ApiExtractor/conditionalstreamreader.cpp
@@ -29,10 +29,96 @@
 #include "conditionalstreamreader.h"
 
 #include <QtCore/QDebug>
+#include <QtCore/QHash>
+
+// ProxyEntityResolver proxies a QXmlStreamEntityResolver set by the user
+// on ConditionalStreamReader and stores entity definitions from the
+// <?entity name value?> processing instruction in a cache
+// (which is also used for the proxied resolver).
+class ProxyEntityResolver : public QXmlStreamEntityResolver
+{
+public:
+    QString resolveEntity(const QString& publicId,
+                          const QString& systemId) override;
+    QString resolveUndeclaredEntity(const QString &name) override;
+
+    QXmlStreamEntityResolver *source() const { return m_source; }
+    void setSource(QXmlStreamEntityResolver *s) { m_source = s; }
+
+    void defineEntity(const QString &name, const QString &value)
+    {
+        m_undeclaredEntityCache.insert(name, value);
+    }
+
+private:
+    QHash<QString, QString> m_undeclaredEntityCache;
+    QXmlStreamEntityResolver *m_source = nullptr;
+};
+
+QString ProxyEntityResolver::resolveEntity(const QString &publicId, const QString &systemId)
+{
+    QString result;
+    if (m_source != nullptr)
+        result = m_source->resolveEntity(publicId, systemId);
+    if (result.isEmpty())
+        result = QXmlStreamEntityResolver::resolveEntity(publicId, systemId);
+    return result;
+}
+
+QString ProxyEntityResolver::resolveUndeclaredEntity(const QString &name)
+{
+    const auto it = m_undeclaredEntityCache.constFind(name);
+    if (it != m_undeclaredEntityCache.constEnd())
+        return it.value();
+    if (m_source == nullptr)
+        return {};
+    const QString result = m_source->resolveUndeclaredEntity(name);
+   if (!result.isEmpty())
+        defineEntity(name, result);
+   return result;
+}
+
+ConditionalStreamReader::ConditionalStreamReader(QIODevice *iod) :
+    m_reader(iod)
+{
+    init();
+}
+
+ConditionalStreamReader::ConditionalStreamReader(const QString &s) :
+    m_reader(s)
+{
+    init();
+}
+
+void ConditionalStreamReader::init()
+{
+    m_proxyEntityResolver = new ProxyEntityResolver;
+    m_reader.setEntityResolver(m_proxyEntityResolver);
+}
+
+ConditionalStreamReader::~ConditionalStreamReader()
+{
+    m_reader.setEntityResolver(nullptr);
+    delete m_proxyEntityResolver;
+}
+
+void ConditionalStreamReader::setEntityResolver(QXmlStreamEntityResolver *resolver)
+{
+    m_proxyEntityResolver->setSource(resolver);
+}
+
+QXmlStreamEntityResolver *ConditionalStreamReader::entityResolver() const
+{
+    return m_proxyEntityResolver->source();
+}
 
 QXmlStreamReader::TokenType ConditionalStreamReader::readNext()
 {
     auto exToken = readNextInternal();
+
+    if (exToken.second == PiTokens::EntityDefinition)
+        return readEntityDefinitonPi() ? exToken.first : QXmlStreamReader::Invalid;
+
     if (exToken.second != PiTokens::If || conditionMatches())
         return exToken.first;
 
@@ -54,6 +140,25 @@ QXmlStreamReader::TokenType ConditionalStreamReader::readNext()
     return exToken.first;
 }
 
+void ConditionalStreamReader::defineEntity(const QString &name, const QString &value)
+{
+    m_proxyEntityResolver->defineEntity(name, value);
+}
+
+// Read an entity definition: "<?entity name value?>:
+bool ConditionalStreamReader::readEntityDefinitonPi()
+{
+    const auto data = m_reader.processingInstructionData();
+    const auto separator = data.indexOf(u' ');
+    if (separator <= 0 || separator == data.size() - 1) {
+        m_reader.raiseError(u"Malformed entity definition: "_qs + data.toString());
+        return false;
+    }
+    defineEntity(data.left(separator).toString(),
+                 data.right(data.size() - separator - 1).toString());
+    return true;
+}
+
 bool ConditionalStreamReader::conditionMatches() const
 {
     const auto keywords = m_reader.processingInstructionData().split(u' ', Qt::SkipEmptyParts);
@@ -106,6 +211,8 @@ ConditionalStreamReader::ExtendedToken ConditionalStreamReader::readNextInternal
             piToken = PiTokens::If;
         else if (target == u"endif")
             piToken = PiTokens::Endif;
+        else if (target == u"entity")
+            piToken = PiTokens::EntityDefinition;
     }
     return {token, piToken};
 }
@@ -124,3 +231,4 @@ QDebug operator<<(QDebug dbg, const QXmlStreamAttributes &attrs)
     dbg << ')';
     return dbg;
 }
+
diff --git a/sources/shiboken6/ApiExtractor/conditionalstreamreader.h b/sources/shiboken6/ApiExtractor/conditionalstreamreader.h
index 691c5e21d..db86ac1e6 100644
--- a/sources/shiboken6/ApiExtractor/conditionalstreamreader.h
+++ b/sources/shiboken6/ApiExtractor/conditionalstreamreader.h
@@ -35,6 +35,8 @@
 
 QT_FORWARD_DECLARE_CLASS(QDebug)
 
+class ProxyEntityResolver;
+
 /// ConditionalStreamReader encapsulates QXmlStreamReader, offering the same
 /// API (except readNextStartElement() and similar conveniences) and internally
 /// uses Processing Instructions like:
@@ -43,17 +45,23 @@ QT_FORWARD_DECLARE_CLASS(QDebug)
 /// containing for example the OS.
 /// It should be possible to use it as a drop-in replacement for
 /// QXmlStreamReader for any parsing code based on readNext().
+/// It also allows for specifying entities using a Processing Instruction:
+/// <?entity name value?>
+/// which can be used in conjunction with conditional processing.
 class ConditionalStreamReader
 {
 public:
     using TokenType = QXmlStreamReader::TokenType;
-    explicit ConditionalStreamReader(QIODevice *iod) : m_reader(iod) { }
-    explicit ConditionalStreamReader(const QString &s) : m_reader(s) { }
+    explicit ConditionalStreamReader(QIODevice *iod);
+    explicit ConditionalStreamReader(const QString &s);
+    ~ConditionalStreamReader();
 
     QIODevice *device() const { return m_reader.device(); }
 
-    void setEntityResolver(QXmlStreamEntityResolver *resolver) { m_reader.setEntityResolver(resolver); }
-    QXmlStreamEntityResolver *entityResolver() const { return m_reader.entityResolver(); }
+    // Note: Caching of entity values is done internally by
+    // ConditionalStreamReader.
+    void setEntityResolver(QXmlStreamEntityResolver *resolver);
+    QXmlStreamEntityResolver *entityResolver() const;
 
     bool atEnd() const { return m_reader.atEnd(); }
     TokenType readNext();
@@ -78,20 +86,25 @@ public:
 
     bool hasError() const { return m_reader.hasError(); }
 
+    // Additional functions (not delegating to QXmlStreamReader)
+    void defineEntity(const QString &name, const QString &value);
+
     const QStringList &conditions() const { return m_conditions; }
     void setConditions(const QStringList &newConditions);
 
     static QStringList platformConditions();
 
 private:
-    enum class PiTokens { None, If, Endif };
+    enum class PiTokens { None, If, Endif, EntityDefinition };
 
     using ExtendedToken = std::pair<TokenType, PiTokens>;
     ExtendedToken readNextInternal();
-
+    void init();
     bool conditionMatches() const;
+    bool readEntityDefinitonPi();
 
     QXmlStreamReader m_reader;
+    ProxyEntityResolver *m_proxyEntityResolver = nullptr;
     QStringList m_conditions = ConditionalStreamReader::platformConditions();
 };
 
diff --git a/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.cpp b/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.cpp
index d59c365dd..303532b65 100644
--- a/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.cpp
+++ b/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.cpp
@@ -222,4 +222,27 @@ void TestDropTypeEntries::testConditionalParsing()
     QCOMPARE(actualTags, expectedTags);
 }
 
+void TestDropTypeEntries::testEntityParsing()
+{
+    const QString xml = QStringLiteral(R"(<?xml version="1.0" encoding="UTF-8"?>
+<root>
+    <?entity testentity word1 word2?>
+    <text>bla &testentity;</text>
+</root>)");
+
+    QString actual;
+    ConditionalStreamReader reader(xml);
+    while (!reader.atEnd()) {
+        auto t = reader.readNext();
+        switch (t) {
+        case QXmlStreamReader::Characters:
+            actual.append(reader.text());
+        default:
+            break;
+        }
+    }
+    QVERIFY2(!reader.hasError(), qPrintable(reader.errorString()));
+    QCOMPARE(actual.trimmed(), u"bla word1 word2");
+}
+
 QTEST_APPLESS_MAIN(TestDropTypeEntries)
diff --git a/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.h b/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.h
index 7746234ba..d163b594d 100644
--- a/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.h
+++ b/sources/shiboken6/ApiExtractor/tests/testdroptypeentries.h
@@ -41,6 +41,7 @@ class TestDropTypeEntries : public QObject
         void testDontDropEntryWithChildTags();
         void testConditionalParsing_data();
         void testConditionalParsing();
+        void testEntityParsing();
 };
 
 #endif
diff --git a/sources/shiboken6/ApiExtractor/typesystemparser.cpp b/sources/shiboken6/ApiExtractor/typesystemparser.cpp
index 3f446c273..4b95d2186 100644
--- a/sources/shiboken6/ApiExtractor/typesystemparser.cpp
+++ b/sources/shiboken6/ApiExtractor/typesystemparser.cpp
@@ -496,7 +496,6 @@ private:
     QString readFile(const QString &entityName, QString *errorMessage) const;
 
     const QString m_currentPath;
-    QHash<QString, QString> m_cache;
 };
 
 QString TypeSystemEntityResolver::readFile(const QString &entityName, QString *errorMessage) const
@@ -531,16 +530,13 @@ QString TypeSystemEntityResolver::readFile(const QString &entityName, QString *e
 
 QString TypeSystemEntityResolver::resolveUndeclaredEntity(const QString &name)
 {
-    auto it = m_cache.find(name);
-    if (it == m_cache.end()) {
-        QString errorMessage;
-        it = m_cache.insert(name, readFile(name, &errorMessage));
-        if (it.value().isEmpty()) { // The parser will fail and display the line number.
-            qCWarning(lcShiboken, "%s",
-                      qPrintable(msgCannotResolveEntity(name, errorMessage)));
-        }
+    QString errorMessage;
+    const QString result = readFile(name, &errorMessage);
+    if (result.isEmpty()) { // The parser will fail and display the line number.
+        qCWarning(lcShiboken, "%s",
+                  qPrintable(msgCannotResolveEntity(name, errorMessage)));
     }
-    return it.value();
+    return result;
 }
 
 TypeSystemParser::TypeSystemParser(TypeDatabase *database, bool generate) :
diff --git a/sources/shiboken6/doc/typesystem_specifying_types.rst b/sources/shiboken6/doc/typesystem_specifying_types.rst
index bec7087a0..3b2e979b3 100644
--- a/sources/shiboken6/doc/typesystem_specifying_types.rst
+++ b/sources/shiboken6/doc/typesystem_specifying_types.rst
@@ -634,6 +634,19 @@ Conditional Processing
 
 .. _private_types:
 
+Defining Entities
+^^^^^^^^^^^^^^^^^
+
+It is possible to define entities using a simple processing instruction:
+
+     .. code-block:: xml
+
+         <?entity name value?>
+         <text>&name;</text>
+
+This allows for defining function signatures depending on platform
+in conjunction with :ref:`conditional_processing`.
+
 Private Types
 ^^^^^^^^^^^^^