Network: Use system publicsuffix database copy when available

[ChangeLog][Network][QNetworkCookieJar] It is possible to use
system's copy of publicsuffix database when it is available.
This behavior is enabled by default on Linux and can be
controlled using new command line switches -system-publicsuffix,
-qt-publicsuffix, -no-publicsuffix, and -publicsuffix=all.

Fixes: QTBUG-95889
Change-Id: I911e1a13c1422cdc35851953309fff064e7c5f26
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

2 files changed, 155 insertions, 6 deletions

diff --git a/src/network/kernel/qtldurl.cpp b/src/network/kernel/qtldurl.cpp
index 046a416914..7d492b23cb 100644
--- a/src/network/kernel/qtldurl.cpp
+++ b/src/network/kernel/qtldurl.cpp
@@ -44,10 +44,19 @@
 #if QT_CONFIG(topleveldomain)
 
 #include "qurl.h"
-#include "private/qtldurl_p.h"
+#include "QtCore/qfile.h"
+#include "QtCore/qfileinfo.h"
+#include "QtCore/qloggingcategory.h"
+#include "QtCore/qstandardpaths.h"
 #include "QtCore/qstring.h"
 
-#include "qurltlds_p.h"
+#if !QT_CONFIG(publicsuffix_qt) && !QT_CONFIG(publicsuffix_system)
+#   error Enable at least one feature: publicsuffix-qt, publicsuffix-system
+#endif
+
+#if QT_CONFIG(publicsuffix_qt)
+#   include "qurltlds_p.h"
+#endif
 
 // Defined in src/3rdparty/libpsl/src/lookup_string_in_fixed_set.c
 extern "C" int LookupStringInFixedSet(const unsigned char *graph, std::size_t length,
@@ -55,15 +64,146 @@ extern "C" int LookupStringInFixedSet(const unsigned char *graph, std::size_t le
 
 QT_BEGIN_NAMESPACE
 
+Q_LOGGING_CATEGORY(lcTld, "qt.network.tld")
+
 static constexpr int PSL_NOT_FOUND = -1;
 static constexpr int PSL_FLAG_EXCEPTION = 1 << 0;
 static constexpr int PSL_FLAG_WILDCARD = 1 << 1;
 
-static int lookupDomain(QByteArrayView domain)
+class QPublicSuffixDatabase final
+{
+public:
+#if QT_CONFIG(publicsuffix_system)
+    QPublicSuffixDatabase();
+#endif // QT_CONFIG(publicsuffix_system)
+
+    int lookupDomain(QByteArrayView domain) const;
+
+private:
+    QByteArrayView m_data
+#if QT_CONFIG(publicsuffix_qt)
+    {
+        kDafsa, sizeof(kDafsa)
+    }
+#endif // QT_CONFIG(publicsuffix_qt)
+    ;
+
+#if QT_CONFIG(publicsuffix_system)
+    std::unique_ptr<QFile> m_dev;
+    QByteArray m_storage;
+    bool loadFile(const QString &fileName);
+#endif // QT_CONFIG(publicsuffix_system)
+};
+
+int QPublicSuffixDatabase::lookupDomain(QByteArrayView domain) const
+{
+    return LookupStringInFixedSet(reinterpret_cast<const unsigned char *>(m_data.constData()),
+                                     m_data.size(), domain.data(), domain.size());
+}
+
+#if QT_CONFIG(publicsuffix_system)
+
+static QStringList locatePublicSuffixFiles()
+{
+    return QStandardPaths::locateAll(QStandardPaths::GenericDataLocation,
+                                     u"publicsuffix/public_suffix_list.dafsa"_qs);
+}
+
+QPublicSuffixDatabase::QPublicSuffixDatabase()
 {
-    return LookupStringInFixedSet(kDafsa, sizeof(kDafsa), domain.data(), domain.size());
+    for (auto &&fileName : locatePublicSuffixFiles()) {
+        if (loadFile(fileName))
+            return;
+    }
+
+#if QT_CONFIG(publicsuffix_qt)
+    qCDebug(lcTld, "Using builtin publicsuffix list");
+#else
+    qCWarning(lcTld, "No usable publicsuffix file found");
+#endif
 }
 
+bool QPublicSuffixDatabase::loadFile(const QString &fileName)
+{
+    static const QByteArrayView DafsaFileHeader = ".DAFSA@PSL_0   \n";
+
+    qCDebug(lcTld, "Loading publicsuffix file: %s", qUtf8Printable(fileName));
+
+    auto systemFile = std::make_unique<QFile>(fileName);
+
+    if (!systemFile->open(QIODevice::ReadOnly)) {
+        qCDebug(lcTld, "Failed to open publicsuffix file: %s",
+                qUtf8Printable(systemFile->errorString()));
+        return false;
+    }
+
+    auto fileSize = systemFile->size();
+    // Check if there is enough data for header, version byte and some data
+    if (fileSize < DafsaFileHeader.size() + 2) {
+        qCWarning(lcTld, "publicsuffix file is too small: %zu", std::size_t(fileSize));
+        return false;
+    }
+
+    auto header = systemFile->read(DafsaFileHeader.size());
+    if (header != DafsaFileHeader) {
+        qCWarning(lcTld, "Invalid publicsuffix file header: %s", qUtf8Printable(header.toHex()));
+        return false;
+    }
+
+    // Check if the file is UTF-8 compatible
+    if (!systemFile->seek(fileSize - 1)) {
+        qCWarning(lcTld, "Failed to seek to the end of file: %s",
+                  qUtf8Printable(systemFile->errorString()));
+        return false;
+    }
+
+    char version;
+    if (systemFile->read(&version, 1) != 1) {
+        qCWarning(lcTld, "Failed to read publicsuffix version");
+        return false;
+    }
+
+    if (version != 0x01) {
+        qCWarning(lcTld, "Unsupported publicsuffix version: %d", int(version));
+        return false;
+    }
+
+    const auto dataSize = fileSize - DafsaFileHeader.size() - 1;
+    // Try to map the file first
+    auto mappedData = systemFile->map(DafsaFileHeader.size(), dataSize);
+    if (mappedData) {
+        qCDebug(lcTld, "Using mapped system publicsuffix data");
+        systemFile->close();
+        m_data = QByteArrayView(mappedData, dataSize);
+        m_dev = std::move(systemFile);
+        return true;
+    }
+
+    qCDebug(lcTld, "Failed to map publicsuffix file: %s",
+            qUtf8Printable(systemFile->errorString()));
+
+    systemFile->seek(DafsaFileHeader.size());
+    m_storage = systemFile->read(dataSize);
+    if (m_storage.size() != dataSize) {
+        qCWarning(lcTld, "Failed to read publicsuffix file");
+        m_storage.clear();
+        return false;
+    }
+
+    qCDebug(lcTld, "Using system publicsuffix data");
+    m_data = m_storage;
+
+    return true;
+}
+
+Q_GLOBAL_STATIC(QPublicSuffixDatabase, publicSuffix);
+
+#else
+
+static const QPublicSuffixDatabase m_publicSuffix;
+
+#endif // QT_CONFIG(publicsuffix_system)
+
 /*!
     \internal
 
@@ -82,7 +222,14 @@ Q_NETWORK_EXPORT bool qIsEffectiveTLD(QStringView domain)
     QByteArray decodedDomain = domain.toUtf8();
     QByteArrayView domainView(decodedDomain);
 
-    auto ret = lookupDomain(domainView);
+#if QT_CONFIG(publicsuffix_system)
+    if (publicSuffix.isDestroyed())
+        return false;
+#else
+    auto publicSuffix = &m_publicSuffix;
+#endif // QT_CONFIG(publicsuffix_system)
+
+    auto ret = publicSuffix->lookupDomain(domainView);
     if (ret != PSL_NOT_FOUND) {
         if (ret & PSL_FLAG_EXCEPTION) // 1
             return false;
@@ -93,7 +240,7 @@ Q_NETWORK_EXPORT bool qIsEffectiveTLD(QStringView domain)
     const auto dot = domainView.indexOf('.');
     if (dot < 0) // Actual TLD: may be effective if the subject of a wildcard rule:
         return ret != PSL_NOT_FOUND;
-    ret = lookupDomain(domainView.sliced(dot + 1)); // 3
+    ret = publicSuffix->lookupDomain(domainView.sliced(dot + 1)); // 3
     if (ret == PSL_NOT_FOUND)
         return false;
     return (ret & PSL_FLAG_WILDCARD) != 0;
diff --git a/src/network/kernel/qurltlds_p.h.INFO b/src/network/kernel/qurltlds_p.h.INFO
index 02a367a97e..d6162a2a0d 100644
--- a/src/network/kernel/qurltlds_p.h.INFO
+++ b/src/network/kernel/qurltlds_p.h.INFO
@@ -5,6 +5,8 @@ qtbase/src/3rdparty/libpsl/src/psl-make-dafsa in the Qt source tree.
 To regenerate the file, run the following command from qtbase tree:
 
     src/3rdparty/libpsl/src/psl-make-dafsa public_suffix_list.dat src/network/kernel/qurltlds_p.h
+    src/3rdparty/libpsl/src/psl-make-dafsa --output-format=binary public_suffix_list.dat \
+        tests/auto/network/access/qnetworkcookiejar/testdata/publicsuffix/public_suffix_list.dafsa
 
 Those arrays in qurltlds_p.h are derived from the Public
 Suffix List ([2]), which was originally provided by