MIME: Make the internal database direct content, not a Qt resource

This saves us from having to bootstrap rcc in regular (non-cross)
compilations, as it can now link to QtCore. Actually un-bootstrapping
rcc is left as an exercise for the reader.

This commit discovered that MSVC cannot handle constexpr arrays bigger
than 256 kB, at which point it simply starts claiming that the constant
expressions using it are not constexpr. ICC has a similar problem at
64 kB, but it tells you why ("note: type "const unsigned char [65537]"
too large for constant-expression evaluation").

Note also that this requires gzip or zstd to be in PATH for compression
to happen. RCC linked to zlib, which is always present due to the
bundled copy. gzip's presence is not likely to be a problem on Unix
systems, but could be for Windows users, especially MSVC ones. If gzip
is not present, QtCore's size will increase by about 1910 kB of
read-only (sharable) data.

Change-Id: I2b1955a995ad40f3b89afffd15a3e65a94670242
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

9 files changed, 371 insertions, 28 deletions

diff --git a/src/corelib/mimetypes/mime/generate.bat b/src/corelib/mimetypes/mime/generate.bat
new file mode 100644
index 0000000000..f63fc63693
--- /dev/null
+++ b/src/corelib/mimetypes/mime/generate.bat
@@ -0,0 +1,73 @@
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+::
+:: Copyright (C) 2019 Intel Corporation.
+:: Contact: https://www.qt.io/licensing/
+::
+:: This file is part of the tools applications of the Qt Toolkit.
+::
+:: $QT_BEGIN_LICENSE:GPL-EXCEPT$
+:: Commercial License Usage
+:: Licensees holding valid commercial Qt licenses may use this file in
+:: accordance with the commercial license agreement provided with the
+:: Software or, alternatively, in accordance with the terms contained in
+:: a written agreement between you and The Qt Company. For licensing terms
+:: and conditions see https://www.qt.io/terms-conditions. For further
+:: information use the contact form at https://www.qt.io/contact-us.
+::
+:: GNU General Public License Usage
+:: Alternatively, this file may be used under the terms of the GNU
+:: General Public License version 3 as published by the Free Software
+:: Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+:: included in the packaging of this file. Please review the following
+:: information to ensure the GNU General Public License requirements will
+:: be met: https://www.gnu.org/licenses/gpl-3.0.html.
+::
+:: $QT_END_LICENSE$
+::
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+
+@echo off
+setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS
+set me=%~dp0
+
+:: Check if certain tools are in PATH
+for %%C in (gzip.exe zstd.exe perl.exe) do set %%C=%%~$PATH:C
+
+:: If perl is in PATH, just let it do everything
+if not "%perl.exe%" == "" goto PuntToPerl
+
+set COMPRESS=
+set MACRO=MIME_DATABASE_IS_UNCOMPRESSED
+if not "%gzip.exe%" == "" (
+    set COMPRESS=gzip -9
+    set MACRO=MIME_DATABASE_IS_GZIP
+)
+
+:: Check if zstd support was enabled
+if /i "%~1" == "--zstd" (
+    shift
+    if not "%zstd.exe%" == "" (
+        set COMPRESS=zstd -19
+        set MACRO=MIME_DATABASE_IS_ZSTD
+    )
+)
+
+if not "%COMPRESS%" == "" goto CompressedCommon
+
+:: No Compression and no Perl
+:: Just hex-dump with Powershell
+powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %1 %1
+exit /b %errorlevel%
+
+:CompressedCommon
+:: Compress to a temporary file, then hex-dump using Powershell
+echo #define %MACRO%
+set tempfile=generate%RANDOM%.tmp
+%COMPRESS% < %1 > %tempfile%
+powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %tempfile% %1
+del %tempfile%
+exit /b %errorlevel%
+
+:PuntToPerl
+perl %me%generate.pl %*
+exit /b %errorlevel%
diff --git a/src/corelib/mimetypes/mime/generate.pl b/src/corelib/mimetypes/mime/generate.pl
new file mode 100644
index 0000000000..0f87d61f8e
--- /dev/null
+++ b/src/corelib/mimetypes/mime/generate.pl
@@ -0,0 +1,113 @@
+#!/usr/bin/perl
+#############################################################################
+##
+## Copyright (C) 2019 Intel Corporation.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is the build configuration utility of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+use strict;
+use warnings;
+use Config;
+local $/;               # Enable "slurp" mode
+
+sub checkCommand($) {
+    use File::Spec::Functions;
+    my $cmd = $_[0] . $Config{_exe};
+    for my $path (path()) {
+        return 1 if -x catfile($path, $cmd);
+    }
+    return 0;
+}
+
+my $data;
+my $compress;
+my $macro;
+my $zlib = eval 'use Compress::Zlib; use IO::Compress::Gzip; return 1;';
+my $fname = shift @ARGV;
+
+if ($zlib) {
+    # Prefer internal zlib support (useful on Windows where gzip.exe isn't
+    # always presnet)
+    $macro = "MIME_DATABASE_IS_GZIP";
+} elsif (checkCommand("gzip")) {
+    # No builtin support for compression (old Perl?)
+    $compress = "gzip -c9";
+    $macro = "MIME_DATABASE_IS_GZIP";
+}
+
+# Check if Qt is being built with zstd support
+if ($fname eq "--zstd") {
+    $fname = shift @ARGV;
+    if (checkCommand("zstd")) {
+        $compress = "zstd -cq19 --single-thread";
+        $macro = "MIME_DATABASE_IS_ZSTD";
+    }
+}
+
+# Check if xml (from xmlstarlet) is in $PATH
+my $cmd;
+if (checkCommand("xml")) {
+    # Minify the data before compressing
+    $cmd = "xml sel -D -B -t -c / $fname";
+    $cmd .= "| $compress" if $compress;
+} elsif ($compress) {
+    $cmd = "$compress < $fname"
+}
+if ($cmd) {
+    # Run the command and read everything
+    open CMD, "$cmd |";
+    $data = <CMD>;
+    close CMD;
+    die("Failed to run $cmd") if ($? >> 8);
+} else {
+    # No command, just read the file
+    open F, "<$fname";
+    $data = <F>;
+    close F;
+}
+
+# Do we need to compress with zlib?
+if (!$compress && $zlib) {
+    $data = eval q{
+        use Compress::Zlib;
+        use IO::Compress::Gzip qw(gzip);
+        my $compressed;
+        gzip \$data => \$compressed,
+            Minimal => 1,
+            Level => Z_BEST_COMPRESSION;
+        return $compressed;
+    };
+}
+
+# Now print as hex
+printf "#define %s\n", $macro if $macro;
+printf "static const unsigned char mimetype_database[] = {";
+my $i = 0;
+map {
+    printf "\n  " if $i++ % 12 == 0;
+    printf "0x%02x, ", ord $_
+} split //, $data;
+printf "\n};\n";
+printf "static constexpr size_t MimeTypeDatabaseOriginalSize = %d;\n",
+    (stat $fname)[7];
diff --git a/src/corelib/mimetypes/mime/hexdump.ps1 b/src/corelib/mimetypes/mime/hexdump.ps1
new file mode 100644
index 0000000000..25ce8138fa
--- /dev/null
+++ b/src/corelib/mimetypes/mime/hexdump.ps1
@@ -0,0 +1,43 @@
+#############################################################################
+##
+## Copyright (C) 2019 Intel Corporation.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is the build configuration utility of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+param([String]$path, [String]$orig)
+
+"static const unsigned char mimetype_database[] = {"
+ForEach ($byte in Get-Content -Encoding byte -ReadCount 16 -path $path) {
+#    if (($byte -eq 0).count -ne 16) {
+        $hex = $byte | Foreach-Object {
+            " 0x" + ("{0:x}" -f $_).PadLeft( 2, "0" ) + ","
+        }
+        "    $hex"
+#    }
+}
+"};"
+
+$file = Get-Childitem -file $orig
+"static constexpr size_t MimeTypeDatabaseOriginalSize = " + $file.length + ";"
diff --git a/src/corelib/mimetypes/mimetypes.pri b/src/corelib/mimetypes/mimetypes.pri
index 62bbe348e4..8cbe7b69ae 100644
--- a/src/corelib/mimetypes/mimetypes.pri
+++ b/src/corelib/mimetypes/mimetypes.pri
@@ -21,5 +21,32 @@ qtConfig(mimetype) {
         mimetypes/qmimeglobpattern.cpp \
         mimetypes/qmimeprovider.cpp
 
-    qtConfig(mimetype-database): RESOURCES += mimetypes/mimetypes.qrc
+    MIME_DATABASE = mimetypes/mime/packages/freedesktop.org.xml
+    OTHER_FILES += $$MIME_DATABASE
+
+    qtConfig(mimetype-database) {
+        outpath = .rcc
+        debug_and_release {
+            CONFIG(debug, debug|release): outpath = .rcc/debug
+            else:                         outpath = .rcc/release
+        }
+
+        mimedb.depends = $$PWD/mime/generate.pl
+        equals(MAKEFILE_GENERATOR, MSVC.NET)|equals(MAKEFILE_GENERATOR, MSBUILD)|isEmpty(QMAKE_SH) {
+            mimedb.commands = cmd /c $$shell_path($$PWD/mime/generate.bat)
+            mimedb.depends += $$PWD/mime/generate.bat $$PWD/mime/hexdump.ps1
+        } else {
+            mimedb.commands = perl $${mimedb.depends}
+        }
+
+        qtConfig(zstd): mimedb.commands += --zstd
+        mimedb.commands += ${QMAKE_FILE_IN} > ${QMAKE_FILE_OUT}
+
+        mimedb.output = $$outpath/qmimeprovider_database.cpp
+        mimedb.input = MIME_DATABASE
+        mimedb.variable_out = INCLUDED_SOURCES
+        QMAKE_EXTRA_COMPILERS += mimedb
+        INCLUDEPATH += $$outpath
+        unset(outpath)
+    }
 }
diff --git a/src/corelib/mimetypes/qmimedatabase.cpp b/src/corelib/mimetypes/qmimedatabase.cpp
index 1fbcc31fae..10b2190966 100644
--- a/src/corelib/mimetypes/qmimedatabase.cpp
+++ b/src/corelib/mimetypes/qmimedatabase.cpp
@@ -102,13 +102,18 @@ void QMimeDatabasePrivate::loadProviders()
     const auto fdoIterator = std::find_if(mimeDirs.constBegin(), mimeDirs.constEnd(), [](const QString &mimeDir) -> bool {
         return QFileInfo::exists(mimeDir + QStringLiteral("/packages/freedesktop.org.xml")); }
     );
-    if (fdoIterator == mimeDirs.constEnd())
-        mimeDirs.prepend(QLatin1String(":/qt-project.org/qmime"));
     //qDebug() << "mime dirs:" << mimeDirs;
 
     Providers currentProviders;
     std::swap(m_providers, currentProviders);
-    m_providers.reserve(mimeDirs.size());
+
+    if (QMimeXMLProvider::InternalDatabaseAvailable && fdoIterator == mimeDirs.constEnd()) {
+        m_providers.reserve(mimeDirs.size() + 1);
+        m_providers.push_back(Providers::value_type(new QMimeXMLProvider(this, QMimeXMLProvider::InternalDatabase)));
+    } else {
+        m_providers.reserve(mimeDirs.size());
+    }
+
     for (const QString &mimeDir : qAsConst(mimeDirs)) {
         const QString cacheFile = mimeDir + QStringLiteral("/mime.cache");
         QFileInfo fileInfo(cacheFile);
diff --git a/src/corelib/mimetypes/qmimeprovider.cpp b/src/corelib/mimetypes/qmimeprovider.cpp
index c61759025c..4aee772366 100644
--- a/src/corelib/mimetypes/qmimeprovider.cpp
+++ b/src/corelib/mimetypes/qmimeprovider.cpp
@@ -1,7 +1,8 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
-** Copyright (C) 2015 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure <david.faure@kdab.com>
+** Copyright (C) 2018 The Qt Company Ltd.
+** Copyright (C) 2018 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure <david.faure@kdab.com>
+** Copyright (C) 2019 Intel Corporation.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtCore module of the Qt Toolkit.
@@ -45,6 +46,7 @@
 #include "qmimemagicrulematcher_p.h"
 
 #include <QXmlStreamReader>
+#include <QBuffer>
 #include <QDir>
 #include <QFile>
 #include <QByteArrayMatcher>
@@ -52,12 +54,33 @@
 #include <QDateTime>
 #include <QtEndian>
 
-static void initResources()
-{
 #if QT_CONFIG(mimetype_database)
-    Q_INIT_RESOURCE(mimetypes);
+#  if defined(Q_CC_MSVC)
+#    pragma section(".qtmimedatabase", read, shared)
+__declspec(allocate(".qtmimedatabase")) __declspec(align(4096))
+#  elif defined(Q_OS_DARWIN)
+__attribute__((section("__TEXT,.qtmimedatabase"), aligned(4096)))
+#  elif (defined(Q_OF_ELF) || defined(Q_OS_WIN)) && defined(Q_CC_GNU)
+__attribute__((section(".qtmimedatabase"), aligned(4096)))
+#  endif
+
+#  include "qmimeprovider_database.cpp"
+
+#  ifdef MIME_DATABASE_IS_ZSTD
+#    if !QT_CONFIG(zstd)
+#      error "MIME database is zstd but no support compiled in!"
+#    endif
+#    include <zstd.h>
+#  endif
+#  ifdef MIME_DATABASE_IS_GZIP
+#    ifdef QT_NO_COMPRESS
+#      error "MIME database is zlib but no support compiled in!"
+#    endif
+#    define ZLIB_CONST
+#    include <zconf.h>
+#    include <zlib.h>
+#  endif
 #endif
-}
 
 QT_BEGIN_NAMESPACE
 
@@ -597,10 +620,55 @@ void QMimeBinaryProvider::loadGenericIcon(QMimeTypePrivate &data)
 
 ////
 
+#if QT_CONFIG(mimetype_database)
+static QString internalMimeFileName()
+{
+    return QStringLiteral("<internal MIME data>");
+}
+
+QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum)
+    : QMimeProviderBase(db, internalMimeFileName())
+{
+    Q_STATIC_ASSERT_X(sizeof(mimetype_database), "Bundled MIME database is empty");
+    Q_STATIC_ASSERT_X(sizeof(mimetype_database) <= MimeTypeDatabaseOriginalSize,
+                      "Compressed MIME database is larger than the original size");
+    Q_STATIC_ASSERT_X(MimeTypeDatabaseOriginalSize <= 16*1024*1024,
+                      "Bundled MIME database is too big");
+    const char *data = reinterpret_cast<const char *>(mimetype_database);
+    qsizetype size = MimeTypeDatabaseOriginalSize;
+
+#ifdef MIME_DATABASE_IS_ZSTD
+    // uncompress with libzstd
+    std::unique_ptr<char []> uncompressed(new char[size]);
+    size = ZSTD_decompress(uncompressed.get(), size, mimetype_database, sizeof(mimetype_database));
+    Q_ASSERT(!ZSTD_isError(size));
+    data = uncompressed.get();
+#elif defined(MIME_DATABASE_IS_GZIP)
+    std::unique_ptr<char []> uncompressed(new char[size]);
+    z_stream zs = {};
+    zs.next_in = mimetype_database;
+    zs.avail_in = sizeof(mimetype_database);
+    zs.next_out = reinterpret_cast<Bytef *>(uncompressed.get());
+    zs.avail_out = size;
+
+    int res = inflateInit2(&zs, MAX_WBITS | 32);
+    Q_ASSERT(res == Z_OK);
+    res = inflate(&zs, Z_FINISH);
+    Q_ASSERT(res == Z_STREAM_END);
+    res = inflateEnd(&zs);
+    Q_ASSERT(res == Z_OK);
+
+    data = uncompressed.get();
+    size = zs.total_out;
+#endif
+
+    load(data, size);
+}
+#endif
+
 QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory)
     : QMimeProviderBase(db, directory)
 {
-    initResources();
     ensureLoaded();
 }
 
@@ -692,6 +760,19 @@ bool QMimeXMLProvider::load(const QString &fileName, QString *errorMessage)
     return parser.parse(&file, fileName, errorMessage);
 }
 
+#if QT_CONFIG(mimetype_database)
+void QMimeXMLProvider::load(const char *data, qsizetype len)
+{
+    QBuffer buffer;
+    buffer.setData(QByteArray::fromRawData(data, len));
+    buffer.open(QIODevice::ReadOnly);
+    QString errorMessage;
+    QMimeTypeParser parser(*this);
+    if (!parser.parse(&buffer, internalMimeFileName(), &errorMessage))
+        qWarning("QMimeDatabase: Error loading internal MIME data\n%s", qPrintable(errorMessage));
+}
+#endif
+
 void QMimeXMLProvider::addGlobPattern(const QMimeGlobPattern &glob)
 {
     m_mimeTypeGlobs.addGlob(glob);
diff --git a/src/corelib/mimetypes/qmimeprovider_p.h b/src/corelib/mimetypes/qmimeprovider_p.h
index b6268210c0..0629df8a95 100644
--- a/src/corelib/mimetypes/qmimeprovider_p.h
+++ b/src/corelib/mimetypes/qmimeprovider_p.h
@@ -132,6 +132,16 @@ private:
 class QMimeXMLProvider : public QMimeProviderBase
 {
 public:
+    enum InternalDatabaseEnum { InternalDatabase };
+#if QT_CONFIG(mimetype_database)
+    enum : bool { InternalDatabaseAvailable = true };
+    QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum);
+#else
+    enum : bool { InternalDatabaseAvailable = false };
+    QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum)
+        : QMimeProviderBase(db, QString())
+    { Q_UNREACHABLE() };
+#endif
     QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory);
     ~QMimeXMLProvider();
 
@@ -156,6 +166,7 @@ public:
 
 private:
     void load(const QString &fileName);
+    void load(const char *data, qsizetype len);
 
     typedef QHash<QString, QMimeType> NameMimeTypeMap;
     NameMimeTypeMap m_nameMimeTypeMap;
diff --git a/src/src.pro b/src/src.pro
index 8ff3ec4c1f..592f0cf644 100644
--- a/src/src.pro
+++ b/src/src.pro
@@ -70,7 +70,7 @@ src_winmain.depends = sub-corelib  # just for the module .pri file
 
 src_corelib.subdir = $$PWD/corelib
 src_corelib.target = sub-corelib
-src_corelib.depends = src_tools_moc src_tools_rcc src_tools_tracegen
+src_corelib.depends = src_tools_moc src_tools_tracegen
 
 src_xml.subdir = $$PWD/xml
 src_xml.target = sub-xml
@@ -118,7 +118,7 @@ src_angle.target = sub-angle
 
 src_gui.subdir = $$PWD/gui
 src_gui.target = sub-gui
-src_gui.depends = src_corelib
+src_gui.depends = src_corelib src_tools_rcc
 
 src_platformheaders.subdir = $$PWD/platformheaders
 src_platformheaders.target = sub-platformheaders
diff --git a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp
index 44c8c4b681..bdbc1c6928 100644
--- a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp
+++ b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp
@@ -30,7 +30,6 @@
 
 #include <QtTest/QtTest>
 #include <QtCore/QCoreApplication>
-#include <QtCore/QMimeDatabase>
 
 class tst_QResourceEngine: public QObject
 {
@@ -119,24 +118,20 @@ void tst_QResourceEngine::checkStructure_data()
 
     QStringList rootContents;
     rootContents << QLatin1String("aliasdir")
+#if defined(Q_OS_ANDROID) && !defined(Q_OS_ANDROID_EMBEDDED)
+                 << QLatin1String("android_testdata")
+#endif
                  << QLatin1String("otherdir")
-                 << QLatin1String("qt-project.org")
                  << QLatin1String("runtime_resource")
                  << QLatin1String("searchpath1")
                  << QLatin1String("searchpath2")
                  << QLatin1String("secondary_root")
                  << QLatin1String("staticplugin")
                  << QLatin1String("test")
-                 << QLatin1String("withoutslashes");
-
-#if defined(Q_OS_ANDROID) && !defined(Q_OS_ANDROID_EMBEDDED)
-    rootContents.insert(1, QLatin1String("android_testdata"));
-#endif
-
 #if defined(BUILTIN_TESTDATA)
-    rootContents.insert(9, QLatin1String("testqrc"));
+                 << QLatin1String("testqrc")
 #endif
-
+                 << QLatin1String("withoutslashes");
 
     QTest::newRow("root dir")          << QString(":/")
                                        << QByteArray()
@@ -352,11 +347,6 @@ void tst_QResourceEngine::checkStructure()
     QFETCH(QLocale, locale);
     QFETCH(qlonglong, contentsSize);
 
-    // We rely on the existence of the root "qt-project.org" in resources. For
-    // static builds on MSVC these resources are only added if they are used.
-    QMimeDatabase db;
-    Q_UNUSED(db);
-
     bool directory = (containedDirs.size() + containedFiles.size() > 0);
     QLocale::setDefault(locale);