Merge remote-tracking branch 'origin/dev' into merge-dev

Change-Id: I31b761cfd5ea01373c60d02a5da8c33398d34739

7 files changed, 367 insertions, 14 deletions

diff --git a/src/corelib/mimetypes/mime/generate.bat b/src/corelib/mimetypes/mime/generate.bat
new file mode 100644
index 0000000000..f63fc63693
--- /dev/null
+++ b/src/corelib/mimetypes/mime/generate.bat
@@ -0,0 +1,73 @@
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+::
+:: Copyright (C) 2019 Intel Corporation.
+:: Contact: https://www.qt.io/licensing/
+::
+:: This file is part of the tools applications of the Qt Toolkit.
+::
+:: $QT_BEGIN_LICENSE:GPL-EXCEPT$
+:: Commercial License Usage
+:: Licensees holding valid commercial Qt licenses may use this file in
+:: accordance with the commercial license agreement provided with the
+:: Software or, alternatively, in accordance with the terms contained in
+:: a written agreement between you and The Qt Company. For licensing terms
+:: and conditions see https://www.qt.io/terms-conditions. For further
+:: information use the contact form at https://www.qt.io/contact-us.
+::
+:: GNU General Public License Usage
+:: Alternatively, this file may be used under the terms of the GNU
+:: General Public License version 3 as published by the Free Software
+:: Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+:: included in the packaging of this file. Please review the following
+:: information to ensure the GNU General Public License requirements will
+:: be met: https://www.gnu.org/licenses/gpl-3.0.html.
+::
+:: $QT_END_LICENSE$
+::
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+
+@echo off
+setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS
+set me=%~dp0
+
+:: Check if certain tools are in PATH
+for %%C in (gzip.exe zstd.exe perl.exe) do set %%C=%%~$PATH:C
+
+:: If perl is in PATH, just let it do everything
+if not "%perl.exe%" == "" goto PuntToPerl
+
+set COMPRESS=
+set MACRO=MIME_DATABASE_IS_UNCOMPRESSED
+if not "%gzip.exe%" == "" (
+    set COMPRESS=gzip -9
+    set MACRO=MIME_DATABASE_IS_GZIP
+)
+
+:: Check if zstd support was enabled
+if /i "%~1" == "--zstd" (
+    shift
+    if not "%zstd.exe%" == "" (
+        set COMPRESS=zstd -19
+        set MACRO=MIME_DATABASE_IS_ZSTD
+    )
+)
+
+if not "%COMPRESS%" == "" goto CompressedCommon
+
+:: No Compression and no Perl
+:: Just hex-dump with Powershell
+powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %1 %1
+exit /b %errorlevel%
+
+:CompressedCommon
+:: Compress to a temporary file, then hex-dump using Powershell
+echo #define %MACRO%
+set tempfile=generate%RANDOM%.tmp
+%COMPRESS% < %1 > %tempfile%
+powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %tempfile% %1
+del %tempfile%
+exit /b %errorlevel%
+
+:PuntToPerl
+perl %me%generate.pl %*
+exit /b %errorlevel%
diff --git a/src/corelib/mimetypes/mime/generate.pl b/src/corelib/mimetypes/mime/generate.pl
new file mode 100644
index 0000000000..0f87d61f8e
--- /dev/null
+++ b/src/corelib/mimetypes/mime/generate.pl
@@ -0,0 +1,113 @@
+#!/usr/bin/perl
+#############################################################################
+##
+## Copyright (C) 2019 Intel Corporation.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is the build configuration utility of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+use strict;
+use warnings;
+use Config;
+local $/;               # Enable "slurp" mode
+
+sub checkCommand($) {
+    use File::Spec::Functions;
+    my $cmd = $_[0] . $Config{_exe};
+    for my $path (path()) {
+        return 1 if -x catfile($path, $cmd);
+    }
+    return 0;
+}
+
+my $data;
+my $compress;
+my $macro;
+my $zlib = eval 'use Compress::Zlib; use IO::Compress::Gzip; return 1;';
+my $fname = shift @ARGV;
+
+if ($zlib) {
+    # Prefer internal zlib support (useful on Windows where gzip.exe isn't
+    # always presnet)
+    $macro = "MIME_DATABASE_IS_GZIP";
+} elsif (checkCommand("gzip")) {
+    # No builtin support for compression (old Perl?)
+    $compress = "gzip -c9";
+    $macro = "MIME_DATABASE_IS_GZIP";
+}
+
+# Check if Qt is being built with zstd support
+if ($fname eq "--zstd") {
+    $fname = shift @ARGV;
+    if (checkCommand("zstd")) {
+        $compress = "zstd -cq19 --single-thread";
+        $macro = "MIME_DATABASE_IS_ZSTD";
+    }
+}
+
+# Check if xml (from xmlstarlet) is in $PATH
+my $cmd;
+if (checkCommand("xml")) {
+    # Minify the data before compressing
+    $cmd = "xml sel -D -B -t -c / $fname";
+    $cmd .= "| $compress" if $compress;
+} elsif ($compress) {
+    $cmd = "$compress < $fname"
+}
+if ($cmd) {
+    # Run the command and read everything
+    open CMD, "$cmd |";
+    $data = <CMD>;
+    close CMD;
+    die("Failed to run $cmd") if ($? >> 8);
+} else {
+    # No command, just read the file
+    open F, "<$fname";
+    $data = <F>;
+    close F;
+}
+
+# Do we need to compress with zlib?
+if (!$compress && $zlib) {
+    $data = eval q{
+        use Compress::Zlib;
+        use IO::Compress::Gzip qw(gzip);
+        my $compressed;
+        gzip \$data => \$compressed,
+            Minimal => 1,
+            Level => Z_BEST_COMPRESSION;
+        return $compressed;
+    };
+}
+
+# Now print as hex
+printf "#define %s\n", $macro if $macro;
+printf "static const unsigned char mimetype_database[] = {";
+my $i = 0;
+map {
+    printf "\n  " if $i++ % 12 == 0;
+    printf "0x%02x, ", ord $_
+} split //, $data;
+printf "\n};\n";
+printf "static constexpr size_t MimeTypeDatabaseOriginalSize = %d;\n",
+    (stat $fname)[7];
diff --git a/src/corelib/mimetypes/mime/hexdump.ps1 b/src/corelib/mimetypes/mime/hexdump.ps1
new file mode 100644
index 0000000000..25ce8138fa
--- /dev/null
+++ b/src/corelib/mimetypes/mime/hexdump.ps1
@@ -0,0 +1,43 @@
+#############################################################################
+##
+## Copyright (C) 2019 Intel Corporation.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is the build configuration utility of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+param([String]$path, [String]$orig)
+
+"static const unsigned char mimetype_database[] = {"
+ForEach ($byte in Get-Content -Encoding byte -ReadCount 16 -path $path) {
+#    if (($byte -eq 0).count -ne 16) {
+        $hex = $byte | Foreach-Object {
+            " 0x" + ("{0:x}" -f $_).PadLeft( 2, "0" ) + ","
+        }
+        "    $hex"
+#    }
+}
+"};"
+
+$file = Get-Childitem -file $orig
+"static constexpr size_t MimeTypeDatabaseOriginalSize = " + $file.length + ";"
diff --git a/src/corelib/mimetypes/mimetypes.pri b/src/corelib/mimetypes/mimetypes.pri
index 62bbe348e4..8cbe7b69ae 100644
--- a/src/corelib/mimetypes/mimetypes.pri
+++ b/src/corelib/mimetypes/mimetypes.pri
@@ -21,5 +21,32 @@ qtConfig(mimetype) {
         mimetypes/qmimeglobpattern.cpp \
         mimetypes/qmimeprovider.cpp
 
-    qtConfig(mimetype-database): RESOURCES += mimetypes/mimetypes.qrc
+    MIME_DATABASE = mimetypes/mime/packages/freedesktop.org.xml
+    OTHER_FILES += $$MIME_DATABASE
+
+    qtConfig(mimetype-database) {
+        outpath = .rcc
+        debug_and_release {
+            CONFIG(debug, debug|release): outpath = .rcc/debug
+            else:                         outpath = .rcc/release
+        }
+
+        mimedb.depends = $$PWD/mime/generate.pl
+        equals(MAKEFILE_GENERATOR, MSVC.NET)|equals(MAKEFILE_GENERATOR, MSBUILD)|isEmpty(QMAKE_SH) {
+            mimedb.commands = cmd /c $$shell_path($$PWD/mime/generate.bat)
+            mimedb.depends += $$PWD/mime/generate.bat $$PWD/mime/hexdump.ps1
+        } else {
+            mimedb.commands = perl $${mimedb.depends}
+        }
+
+        qtConfig(zstd): mimedb.commands += --zstd
+        mimedb.commands += ${QMAKE_FILE_IN} > ${QMAKE_FILE_OUT}
+
+        mimedb.output = $$outpath/qmimeprovider_database.cpp
+        mimedb.input = MIME_DATABASE
+        mimedb.variable_out = INCLUDED_SOURCES
+        QMAKE_EXTRA_COMPILERS += mimedb
+        INCLUDEPATH += $$outpath
+        unset(outpath)
+    }
 }
diff --git a/src/corelib/mimetypes/qmimedatabase.cpp b/src/corelib/mimetypes/qmimedatabase.cpp
index ce84a15831..f1d0dc859e 100644
--- a/src/corelib/mimetypes/qmimedatabase.cpp
+++ b/src/corelib/mimetypes/qmimedatabase.cpp
@@ -102,13 +102,18 @@ void QMimeDatabasePrivate::loadProviders()
     const auto fdoIterator = std::find_if(mimeDirs.constBegin(), mimeDirs.constEnd(), [](const QString &mimeDir) -> bool {
         return QFileInfo::exists(mimeDir + QStringLiteral("/packages/freedesktop.org.xml")); }
     );
-    if (fdoIterator == mimeDirs.constEnd())
-        mimeDirs.prepend(QLatin1String(":/qt-project.org/qmime"));
     //qDebug() << "mime dirs:" << mimeDirs;
 
     Providers currentProviders;
     std::swap(m_providers, currentProviders);
-    m_providers.reserve(mimeDirs.size());
+
+    if (QMimeXMLProvider::InternalDatabaseAvailable && fdoIterator == mimeDirs.constEnd()) {
+        m_providers.reserve(mimeDirs.size() + 1);
+        m_providers.push_back(Providers::value_type(new QMimeXMLProvider(this, QMimeXMLProvider::InternalDatabase)));
+    } else {
+        m_providers.reserve(mimeDirs.size());
+    }
+
     for (const QString &mimeDir : qAsConst(mimeDirs)) {
         const QString cacheFile = mimeDir + QStringLiteral("/mime.cache");
         QFileInfo fileInfo(cacheFile);
@@ -487,7 +492,7 @@ QMimeDatabase::QMimeDatabase() :
  */
 QMimeDatabase::~QMimeDatabase()
 {
-    d = 0;
+    d = nullptr;
 }
 
 /*!
diff --git a/src/corelib/mimetypes/qmimeprovider.cpp b/src/corelib/mimetypes/qmimeprovider.cpp
index a3a6b9615c..4aee772366 100644
--- a/src/corelib/mimetypes/qmimeprovider.cpp
+++ b/src/corelib/mimetypes/qmimeprovider.cpp
@@ -1,7 +1,8 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
-** Copyright (C) 2015 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure <david.faure@kdab.com>
+** Copyright (C) 2018 The Qt Company Ltd.
+** Copyright (C) 2018 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure <david.faure@kdab.com>
+** Copyright (C) 2019 Intel Corporation.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtCore module of the Qt Toolkit.
@@ -45,6 +46,7 @@
 #include "qmimemagicrulematcher_p.h"
 
 #include <QXmlStreamReader>
+#include <QBuffer>
 #include <QDir>
 #include <QFile>
 #include <QByteArrayMatcher>
@@ -52,12 +54,33 @@
 #include <QDateTime>
 #include <QtEndian>
 
-static void initResources()
-{
 #if QT_CONFIG(mimetype_database)
-    Q_INIT_RESOURCE(mimetypes);
+#  if defined(Q_CC_MSVC)
+#    pragma section(".qtmimedatabase", read, shared)
+__declspec(allocate(".qtmimedatabase")) __declspec(align(4096))
+#  elif defined(Q_OS_DARWIN)
+__attribute__((section("__TEXT,.qtmimedatabase"), aligned(4096)))
+#  elif (defined(Q_OF_ELF) || defined(Q_OS_WIN)) && defined(Q_CC_GNU)
+__attribute__((section(".qtmimedatabase"), aligned(4096)))
+#  endif
+
+#  include "qmimeprovider_database.cpp"
+
+#  ifdef MIME_DATABASE_IS_ZSTD
+#    if !QT_CONFIG(zstd)
+#      error "MIME database is zstd but no support compiled in!"
+#    endif
+#    include <zstd.h>
+#  endif
+#  ifdef MIME_DATABASE_IS_GZIP
+#    ifdef QT_NO_COMPRESS
+#      error "MIME database is zlib but no support compiled in!"
+#    endif
+#    define ZLIB_CONST
+#    include <zconf.h>
+#    include <zlib.h>
+#  endif
 #endif
-}
 
 QT_BEGIN_NAMESPACE
 
@@ -130,7 +153,7 @@ bool QMimeBinaryProvider::CacheFile::reload()
     if (file.isOpen()) {
         file.close();
     }
-    data = 0;
+    data = nullptr;
     return load();
 }
 
@@ -306,7 +329,7 @@ bool QMimeBinaryProvider::matchMagicRule(QMimeBinaryProvider::CacheFile *cacheFi
         const int valueLength = cacheFile->getUint32(off + 12);
         const int valueOffset = cacheFile->getUint32(off + 16);
         const int maskOffset = cacheFile->getUint32(off + 20);
-        const char *mask = maskOffset ? cacheFile->getCharStar(maskOffset) : NULL;
+        const char *mask = maskOffset ? cacheFile->getCharStar(maskOffset) : nullptr;
 
         if (!QMimeMagicRule::matchSubstring(dataPtr, dataSize, rangeStart, rangeLength, valueLength, cacheFile->getCharStar(valueOffset), mask))
             continue;
@@ -597,10 +620,55 @@ void QMimeBinaryProvider::loadGenericIcon(QMimeTypePrivate &data)
 
 ////
 
+#if QT_CONFIG(mimetype_database)
+static QString internalMimeFileName()
+{
+    return QStringLiteral("<internal MIME data>");
+}
+
+QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum)
+    : QMimeProviderBase(db, internalMimeFileName())
+{
+    Q_STATIC_ASSERT_X(sizeof(mimetype_database), "Bundled MIME database is empty");
+    Q_STATIC_ASSERT_X(sizeof(mimetype_database) <= MimeTypeDatabaseOriginalSize,
+                      "Compressed MIME database is larger than the original size");
+    Q_STATIC_ASSERT_X(MimeTypeDatabaseOriginalSize <= 16*1024*1024,
+                      "Bundled MIME database is too big");
+    const char *data = reinterpret_cast<const char *>(mimetype_database);
+    qsizetype size = MimeTypeDatabaseOriginalSize;
+
+#ifdef MIME_DATABASE_IS_ZSTD
+    // uncompress with libzstd
+    std::unique_ptr<char []> uncompressed(new char[size]);
+    size = ZSTD_decompress(uncompressed.get(), size, mimetype_database, sizeof(mimetype_database));
+    Q_ASSERT(!ZSTD_isError(size));
+    data = uncompressed.get();
+#elif defined(MIME_DATABASE_IS_GZIP)
+    std::unique_ptr<char []> uncompressed(new char[size]);
+    z_stream zs = {};
+    zs.next_in = mimetype_database;
+    zs.avail_in = sizeof(mimetype_database);
+    zs.next_out = reinterpret_cast<Bytef *>(uncompressed.get());
+    zs.avail_out = size;
+
+    int res = inflateInit2(&zs, MAX_WBITS | 32);
+    Q_ASSERT(res == Z_OK);
+    res = inflate(&zs, Z_FINISH);
+    Q_ASSERT(res == Z_STREAM_END);
+    res = inflateEnd(&zs);
+    Q_ASSERT(res == Z_OK);
+
+    data = uncompressed.get();
+    size = zs.total_out;
+#endif
+
+    load(data, size);
+}
+#endif
+
 QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory)
     : QMimeProviderBase(db, directory)
 {
-    initResources();
     ensureLoaded();
 }
 
@@ -692,6 +760,19 @@ bool QMimeXMLProvider::load(const QString &fileName, QString *errorMessage)
     return parser.parse(&file, fileName, errorMessage);
 }
 
+#if QT_CONFIG(mimetype_database)
+void QMimeXMLProvider::load(const char *data, qsizetype len)
+{
+    QBuffer buffer;
+    buffer.setData(QByteArray::fromRawData(data, len));
+    buffer.open(QIODevice::ReadOnly);
+    QString errorMessage;
+    QMimeTypeParser parser(*this);
+    if (!parser.parse(&buffer, internalMimeFileName(), &errorMessage))
+        qWarning("QMimeDatabase: Error loading internal MIME data\n%s", qPrintable(errorMessage));
+}
+#endif
+
 void QMimeXMLProvider::addGlobPattern(const QMimeGlobPattern &glob)
 {
     m_mimeTypeGlobs.addGlob(glob);
diff --git a/src/corelib/mimetypes/qmimeprovider_p.h b/src/corelib/mimetypes/qmimeprovider_p.h
index b6268210c0..0629df8a95 100644
--- a/src/corelib/mimetypes/qmimeprovider_p.h
+++ b/src/corelib/mimetypes/qmimeprovider_p.h
@@ -132,6 +132,16 @@ private:
 class QMimeXMLProvider : public QMimeProviderBase
 {
 public:
+    enum InternalDatabaseEnum { InternalDatabase };
+#if QT_CONFIG(mimetype_database)
+    enum : bool { InternalDatabaseAvailable = true };
+    QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum);
+#else
+    enum : bool { InternalDatabaseAvailable = false };
+    QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum)
+        : QMimeProviderBase(db, QString())
+    { Q_UNREACHABLE() };
+#endif
     QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory);
     ~QMimeXMLProvider();
 
@@ -156,6 +166,7 @@ public:
 
 private:
     void load(const QString &fileName);
+    void load(const char *data, qsizetype len);
 
     typedef QHash<QString, QMimeType> NameMimeTypeMap;
     NameMimeTypeMap m_nameMimeTypeMap;