QMimeDatabase: redesign to use both binary and XML providers

Previously, we would use mime.cache in all mime directories if at
least one of them had such a file (other than the most-local one),
otherwise the "source" XML would be used in all directories.

Now it's possible to use mime.cache in those directories which
have one, and XML in those directories that don't.

Not only is this more correct, it will allow in a subsequent
commit to bundle the binary cache in QtCore's qrc rather than
the very big XML file.

The design change to allow this is that now every provider
only deals with a single directory, and QMimeDatabasePrivate
takes care of creating multiple providers, one for each dir.
This required to move most of the loops from the binary provider
up to QMimeDatabasePrivate itself.

Change-Id: Iac82d5f5836b80d45076e586b903d16fa2525b34
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

1 files changed, 162 insertions, 31 deletions

diff --git a/src/corelib/mimetypes/qmimedatabase.cpp b/src/corelib/mimetypes/qmimedatabase.cpp
index 38b940d6db..864e9edc61 100644
--- a/src/corelib/mimetypes/qmimedatabase.cpp
+++ b/src/corelib/mimetypes/qmimedatabase.cpp
@@ -51,6 +51,7 @@
 #include <QtCore/QFile>
 #include <QtCore/QFileInfo>
 #include <QtCore/QSet>
+#include <QtCore/QStandardPaths>
 #include <QtCore/QBuffer>
 #include <QtCore/QUrl>
 #include <QtCore/QDebug>
@@ -69,14 +70,13 @@ QMimeDatabasePrivate *QMimeDatabasePrivate::instance()
 }
 
 QMimeDatabasePrivate::QMimeDatabasePrivate()
-    : m_provider(0), m_defaultMimeType(QLatin1String("application/octet-stream"))
+    : m_defaultMimeType(QLatin1String("application/octet-stream"))
 {
 }
 
 QMimeDatabasePrivate::~QMimeDatabasePrivate()
 {
-    delete m_provider;
-    m_provider = 0;
+    qDeleteAll(m_providers);
 }
 
 Q_CORE_EXPORT int qmime_secondsBetweenChecks = 5; // exported for the unit test
@@ -89,24 +89,80 @@ bool QMimeDatabasePrivate::shouldCheck()
     return true;
 }
 
-QMimeProviderBase *QMimeDatabasePrivate::provider()
-{
-    Q_ASSERT(!mutex.tryLock()); // caller should have locked mutex
-    if (!m_provider) {
-        QMimeProviderBase *binaryProvider = new QMimeBinaryProvider(this);
-        if (binaryProvider->isValid()) {
-            m_provider = binaryProvider;
+#if defined(Q_OS_UNIX) && !defined(Q_OS_INTEGRITY)
+#define QT_USE_MMAP
+#endif
+
+void QMimeDatabasePrivate::loadProviders()
+{
+    // We use QStandardPaths every time to check if new files appeared
+    QStringList mimeDirs = QStandardPaths::locateAll(QStandardPaths::GenericDataLocation, QLatin1String("mime"), QStandardPaths::LocateDirectory);
+    const auto fdoIterator = std::find_if(mimeDirs.constBegin(), mimeDirs.constEnd(), [](const QString &mimeDir) -> bool {
+        return QFileInfo::exists(mimeDir + QStringLiteral("/packages/freedesktop.org.xml")); }
+    );
+    if (fdoIterator == mimeDirs.constEnd())
+        mimeDirs.prepend(QLatin1String(":/qt-project.org/qmime"));
+    //qDebug() << "mime dirs:" << mimeDirs;
+
+    m_providers.reserve(mimeDirs.size());
+    for (const QString &mimeDir : qAsConst(mimeDirs)) {
+        const QString cacheFile = mimeDir + QStringLiteral("/mime.cache");
+        QFileInfo fileInfo(cacheFile);
+        // Check if we already have a provider for this dir
+        // [This could be optimized by keeping a copy of mimeDirs and comparing the stringlists]
+        const auto it = std::find_if(m_providers.begin(), m_providers.end(), [mimeDir](QMimeProviderBase *prov) { return prov->directory() == mimeDir; });
+        if (it == m_providers.end()) {
+            QMimeProviderBase *provider = nullptr;
+#if defined(QT_USE_MMAP)
+            if (qEnvironmentVariableIsEmpty("QT_NO_MIME_CACHE") && fileInfo.exists()) {
+                provider = new QMimeBinaryProvider(this, mimeDir);
+                //qDebug() << "Created binary provider for" << mimeDir;
+                if (!provider->isValid()) {
+                    delete provider;
+                    provider = nullptr;
+                }
+            }
+#endif
+            if (!provider) {
+                provider = new QMimeXMLProvider(this, mimeDir);
+                //qDebug() << "Created XML provider for" << mimeDir;
+            }
+            m_providers.append(provider);
         } else {
-            delete binaryProvider;
-            m_provider = new QMimeXMLProvider(this);
+            QMimeProviderBase *provider = *it;
+            provider->ensureLoaded();
+            if (!provider->isValid()) {
+                delete provider;
+                provider = new QMimeXMLProvider(this, mimeDir);
+                //qDebug() << "Created XML provider to replace binary provider for" << mimeDir;
+                *it = provider;
+            }
         }
-        m_provider->ensureLoaded();
+    }
+}
+
+QVector<QMimeProviderBase *> QMimeDatabasePrivate::providers()
+{
+    Q_ASSERT(!mutex.tryLock()); // caller should have locked mutex
+    if (m_providers.isEmpty()) {
+        loadProviders();
         m_lastCheck.start();
     } else {
         if (shouldCheck())
-            m_provider->ensureLoaded();
+            loadProviders();
+    }
+    return m_providers;
+}
+
+QString QMimeDatabasePrivate::resolveAlias(const QString &nameOrAlias)
+{
+    const auto allProviders = providers();
+    for (QMimeProviderBase *provider : allProviders) {
+        const QString ret = provider->resolveAlias(nameOrAlias);
+        if (!ret.isEmpty())
+            return ret;
     }
-    return m_provider;
+    return nameOrAlias;
 }
 
 /*!
@@ -115,7 +171,14 @@ QMimeProviderBase *QMimeDatabasePrivate::provider()
  */
 QMimeType QMimeDatabasePrivate::mimeTypeForName(const QString &nameOrAlias)
 {
-    return provider()->mimeTypeForName(provider()->resolveAlias(nameOrAlias));
+    const auto allProviders = providers();
+    const QString mimeName = resolveAlias(nameOrAlias);
+    for (QMimeProviderBase *provider : allProviders) {
+        const QMimeType mime = provider->mimeTypeForName(mimeName);
+        if (mime.isValid())
+            return mime;
+    }
+    return {};
 }
 
 QStringList QMimeDatabasePrivate::mimeTypeForFileName(const QString &fileName)
@@ -123,44 +186,105 @@ QStringList QMimeDatabasePrivate::mimeTypeForFileName(const QString &fileName)
     if (fileName.endsWith(QLatin1Char('/')))
         return QStringList() << QLatin1String("inode/directory");
 
-    QStringList matchingMimeTypes = provider()->findByFileName(QFileInfo(fileName).fileName()).m_matchingMimeTypes;
+    const QString shortName = QFileInfo(fileName).fileName();
+    const QMimeGlobMatchResult result = findByFileName(shortName);
+    QStringList matchingMimeTypes = result.m_matchingMimeTypes;
     matchingMimeTypes.sort(); // make it deterministic
     return matchingMimeTypes;
 }
 
 QMimeGlobMatchResult QMimeDatabasePrivate::findByFileName(const QString &fileName)
 {
-    return provider()->findByFileName(fileName);
+    QMimeGlobMatchResult result;
+    // TODO this parses in the order (local, global). Check that it handles "NOGLOBS" correctly.
+    const auto allProviders = providers();
+    for (QMimeProviderBase *provider : allProviders)
+        provider->addFileNameMatches(fileName, result);
+    return result;
 }
 
 void QMimeDatabasePrivate::loadMimeTypePrivate(QMimeTypePrivate &mimePrivate)
 {
     QMutexLocker locker(&mutex);
-    provider()->loadMimeTypePrivate(mimePrivate);
+    if (!mimePrivate.loaded) { // XML provider sets loaded=true, binary provider does this on demand
+        Q_ASSERT(mimePrivate.fromCache);
+        QMimeBinaryProvider::loadMimeTypePrivate(mimePrivate);
+    }
 }
 
 void QMimeDatabasePrivate::loadGenericIcon(QMimeTypePrivate &mimePrivate)
 {
     QMutexLocker locker(&mutex);
-    provider()->loadGenericIcon(mimePrivate);
+    if (mimePrivate.fromCache) {
+        mimePrivate.genericIconName.clear();
+        const auto allProviders = providers();
+        for (QMimeProviderBase *provider : allProviders) {
+            provider->loadGenericIcon(mimePrivate);
+            if (!mimePrivate.genericIconName.isEmpty())
+                break;
+        }
+    }
 }
 
 void QMimeDatabasePrivate::loadIcon(QMimeTypePrivate &mimePrivate)
 {
     QMutexLocker locker(&mutex);
-    provider()->loadIcon(mimePrivate);
+    if (mimePrivate.fromCache) {
+        mimePrivate.iconName.clear();
+        const auto allProviders = providers();
+        for (QMimeProviderBase *provider : allProviders) {
+            provider->loadIcon(mimePrivate);
+            if (!mimePrivate.iconName.isEmpty())
+                break;
+        }
+    }
 }
 
-QStringList QMimeDatabasePrivate::parents(const QString &mimeName)
+static QString fallbackParent(const QString &mimeTypeName)
+{
+    const QStringRef myGroup = mimeTypeName.leftRef(mimeTypeName.indexOf(QLatin1Char('/')));
+    // All text/* types are subclasses of text/plain.
+    if (myGroup == QLatin1String("text") && mimeTypeName != QLatin1String("text/plain"))
+        return QLatin1String("text/plain");
+    // All real-file mimetypes implicitly derive from application/octet-stream
+    if (myGroup != QLatin1String("inode") &&
+        // ignore non-file extensions
+        myGroup != QLatin1String("all") && myGroup != QLatin1String("fonts") && myGroup != QLatin1String("print") && myGroup != QLatin1String("uri")
+        && mimeTypeName != QLatin1String("application/octet-stream")) {
+        return QLatin1String("application/octet-stream");
+    }
+    return QString();
+}
+
+QStringList QMimeDatabasePrivate::mimeParents(const QString &mimeName)
 {
     QMutexLocker locker(&mutex);
-    return provider()->parents(mimeName);
+    return parents(mimeName);
+}
+
+QStringList QMimeDatabasePrivate::parents(const QString &mimeName)
+{
+    Q_ASSERT(!mutex.tryLock());
+    QStringList result;
+    const auto allProviders = providers();
+    for (QMimeProviderBase *provider : allProviders)
+        provider->addParents(mimeName, result);
+    if (result.isEmpty()) {
+        const QString parent = fallbackParent(mimeName);
+        if (!parent.isEmpty())
+            result.append(parent);
+    }
+    return result;
 }
 
 QStringList QMimeDatabasePrivate::listAliases(const QString &mimeName)
 {
     QMutexLocker locker(&mutex);
-    return provider()->listAliases(mimeName);
+    QStringList result;
+    const auto allProviders = providers();
+    for (QMimeProviderBase *provider : allProviders)
+        provider->addAliases(mimeName, result);
+    return result;
 }
 
 bool QMimeDatabasePrivate::mimeInherits(const QString &mime, const QString &parent)
@@ -196,7 +320,10 @@ QMimeType QMimeDatabasePrivate::findByData(const QByteArray &data, int *accuracy
     }
 
     *accuracyPtr = 0;
-    QMimeType candidate = provider()->findByMagic(data, accuracyPtr);
+    QMimeType candidate;
+    const auto allProviders = providers();
+    for (QMimeProviderBase *provider : allProviders)
+        provider->findByMagic(data, accuracyPtr, candidate);
 
     if (candidate.isValid())
         return candidate;
@@ -223,7 +350,7 @@ QMimeType QMimeDatabasePrivate::mimeTypeForFileNameAndData(const QString &fileNa
     if (fileName.endsWith(QLatin1Char('/')))
         candidatesByName.addMatch(QLatin1String("inode/directory"), 100, QString());
     else
-        candidatesByName = provider()->findByFileName(QFileInfo(fileName).fileName());
+        candidatesByName = findByFileName(QFileInfo(fileName).fileName());
     if (candidatesByName.m_allMatchingMimeTypes.count() == 1) {
         *accuracyPtr = 100;
         const QMimeType mime = mimeTypeForName(candidatesByName.m_matchingMimeTypes.at(0));
@@ -273,21 +400,25 @@ QMimeType QMimeDatabasePrivate::mimeTypeForFileNameAndData(const QString &fileNa
 
 QList<QMimeType> QMimeDatabasePrivate::allMimeTypes()
 {
-    return provider()->allMimeTypes();
+    QList<QMimeType> result;
+    const auto allProviders = providers();
+    for (QMimeProviderBase *provider : allProviders)
+        provider->addAllMimeTypes(result);
+    return result;
 }
 
 bool QMimeDatabasePrivate::inherits(const QString &mime, const QString &parent)
 {
-    const QString resolvedParent = provider()->resolveAlias(parent);
-    //Q_ASSERT(provider()->resolveAlias(mime) == mime);
+    const QString resolvedParent = resolveAlias(parent);
     std::stack<QString, QStringList> toCheck;
     toCheck.push(mime);
     while (!toCheck.empty()) {
         if (toCheck.top() == resolvedParent)
             return true;
-        const auto parents = provider()->parents(toCheck.top());
+        const QString mimeName = toCheck.top();
         toCheck.pop();
-        for (const QString &par : parents)
+        const auto parentList = parents(mimeName);
+        for (const QString &par : parentList)
             toCheck.push(par);
     }
     return false;