diff options
-rw-r--r-- | src/corelib/mimetypes/mime/generate.bat | 73 | ||||
-rw-r--r-- | src/corelib/mimetypes/mime/generate.pl | 113 | ||||
-rw-r--r-- | src/corelib/mimetypes/mime/hexdump.ps1 | 43 | ||||
-rw-r--r-- | src/corelib/mimetypes/mimetypes.pri | 29 | ||||
-rw-r--r-- | src/corelib/mimetypes/qmimedatabase.cpp | 11 | ||||
-rw-r--r-- | src/corelib/mimetypes/qmimeprovider.cpp | 95 | ||||
-rw-r--r-- | src/corelib/mimetypes/qmimeprovider_p.h | 11 | ||||
-rw-r--r-- | src/src.pro | 4 | ||||
-rw-r--r-- | tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp | 20 |
9 files changed, 371 insertions, 28 deletions
diff --git a/src/corelib/mimetypes/mime/generate.bat b/src/corelib/mimetypes/mime/generate.bat new file mode 100644 index 0000000000..f63fc63693 --- /dev/null +++ b/src/corelib/mimetypes/mime/generate.bat @@ -0,0 +1,73 @@ +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:: +:: Copyright (C) 2019 Intel Corporation. +:: Contact: https://www.qt.io/licensing/ +:: +:: This file is part of the tools applications of the Qt Toolkit. +:: +:: $QT_BEGIN_LICENSE:GPL-EXCEPT$ +:: Commercial License Usage +:: Licensees holding valid commercial Qt licenses may use this file in +:: accordance with the commercial license agreement provided with the +:: Software or, alternatively, in accordance with the terms contained in +:: a written agreement between you and The Qt Company. For licensing terms +:: and conditions see https://www.qt.io/terms-conditions. For further +:: information use the contact form at https://www.qt.io/contact-us. +:: +:: GNU General Public License Usage +:: Alternatively, this file may be used under the terms of the GNU +:: General Public License version 3 as published by the Free Software +:: Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +:: included in the packaging of this file. Please review the following +:: information to ensure the GNU General Public License requirements will +:: be met: https://www.gnu.org/licenses/gpl-3.0.html. +:: +:: $QT_END_LICENSE$ +:: +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +@echo off +setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS +set me=%~dp0 + +:: Check if certain tools are in PATH +for %%C in (gzip.exe zstd.exe perl.exe) do set %%C=%%~$PATH:C + +:: If perl is in PATH, just let it do everything +if not "%perl.exe%" == "" goto PuntToPerl + +set COMPRESS= +set MACRO=MIME_DATABASE_IS_UNCOMPRESSED +if not "%gzip.exe%" == "" ( + set COMPRESS=gzip -9 + set MACRO=MIME_DATABASE_IS_GZIP +) + +:: Check if zstd support was enabled +if /i "%~1" == "--zstd" ( + shift + if not "%zstd.exe%" == "" ( + set COMPRESS=zstd -19 + set MACRO=MIME_DATABASE_IS_ZSTD + ) +) + +if not "%COMPRESS%" == "" goto CompressedCommon + +:: No Compression and no Perl +:: Just hex-dump with Powershell +powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %1 %1 +exit /b %errorlevel% + +:CompressedCommon +:: Compress to a temporary file, then hex-dump using Powershell +echo #define %MACRO% +set tempfile=generate%RANDOM%.tmp +%COMPRESS% < %1 > %tempfile% +powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %tempfile% %1 +del %tempfile% +exit /b %errorlevel% + +:PuntToPerl +perl %me%generate.pl %* +exit /b %errorlevel% diff --git a/src/corelib/mimetypes/mime/generate.pl b/src/corelib/mimetypes/mime/generate.pl new file mode 100644 index 0000000000..0f87d61f8e --- /dev/null +++ b/src/corelib/mimetypes/mime/generate.pl @@ -0,0 +1,113 @@ +#!/usr/bin/perl +############################################################################# +## +## Copyright (C) 2019 Intel Corporation. +## Contact: https://www.qt.io/licensing/ +## +## This file is the build configuration utility of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# +use strict; +use warnings; +use Config; +local $/; # Enable "slurp" mode + +sub checkCommand($) { + use File::Spec::Functions; + my $cmd = $_[0] . $Config{_exe}; + for my $path (path()) { + return 1 if -x catfile($path, $cmd); + } + return 0; +} + +my $data; +my $compress; +my $macro; +my $zlib = eval 'use Compress::Zlib; use IO::Compress::Gzip; return 1;'; +my $fname = shift @ARGV; + +if ($zlib) { + # Prefer internal zlib support (useful on Windows where gzip.exe isn't + # always presnet) + $macro = "MIME_DATABASE_IS_GZIP"; +} elsif (checkCommand("gzip")) { + # No builtin support for compression (old Perl?) + $compress = "gzip -c9"; + $macro = "MIME_DATABASE_IS_GZIP"; +} + +# Check if Qt is being built with zstd support +if ($fname eq "--zstd") { + $fname = shift @ARGV; + if (checkCommand("zstd")) { + $compress = "zstd -cq19 --single-thread"; + $macro = "MIME_DATABASE_IS_ZSTD"; + } +} + +# Check if xml (from xmlstarlet) is in $PATH +my $cmd; +if (checkCommand("xml")) { + # Minify the data before compressing + $cmd = "xml sel -D -B -t -c / $fname"; + $cmd .= "| $compress" if $compress; +} elsif ($compress) { + $cmd = "$compress < $fname" +} +if ($cmd) { + # Run the command and read everything + open CMD, "$cmd |"; + $data = <CMD>; + close CMD; + die("Failed to run $cmd") if ($? >> 8); +} else { + # No command, just read the file + open F, "<$fname"; + $data = <F>; + close F; +} + +# Do we need to compress with zlib? +if (!$compress && $zlib) { + $data = eval q{ + use Compress::Zlib; + use IO::Compress::Gzip qw(gzip); + my $compressed; + gzip \$data => \$compressed, + Minimal => 1, + Level => Z_BEST_COMPRESSION; + return $compressed; + }; +} + +# Now print as hex +printf "#define %s\n", $macro if $macro; +printf "static const unsigned char mimetype_database[] = {"; +my $i = 0; +map { + printf "\n " if $i++ % 12 == 0; + printf "0x%02x, ", ord $_ +} split //, $data; +printf "\n};\n"; +printf "static constexpr size_t MimeTypeDatabaseOriginalSize = %d;\n", + (stat $fname)[7]; diff --git a/src/corelib/mimetypes/mime/hexdump.ps1 b/src/corelib/mimetypes/mime/hexdump.ps1 new file mode 100644 index 0000000000..25ce8138fa --- /dev/null +++ b/src/corelib/mimetypes/mime/hexdump.ps1 @@ -0,0 +1,43 @@ +############################################################################# +## +## Copyright (C) 2019 Intel Corporation. +## Contact: https://www.qt.io/licensing/ +## +## This file is the build configuration utility of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +param([String]$path, [String]$orig) + +"static const unsigned char mimetype_database[] = {" +ForEach ($byte in Get-Content -Encoding byte -ReadCount 16 -path $path) { +# if (($byte -eq 0).count -ne 16) { + $hex = $byte | Foreach-Object { + " 0x" + ("{0:x}" -f $_).PadLeft( 2, "0" ) + "," + } + " $hex" +# } +} +"};" + +$file = Get-Childitem -file $orig +"static constexpr size_t MimeTypeDatabaseOriginalSize = " + $file.length + ";" diff --git a/src/corelib/mimetypes/mimetypes.pri b/src/corelib/mimetypes/mimetypes.pri index 62bbe348e4..8cbe7b69ae 100644 --- a/src/corelib/mimetypes/mimetypes.pri +++ b/src/corelib/mimetypes/mimetypes.pri @@ -21,5 +21,32 @@ qtConfig(mimetype) { mimetypes/qmimeglobpattern.cpp \ mimetypes/qmimeprovider.cpp - qtConfig(mimetype-database): RESOURCES += mimetypes/mimetypes.qrc + MIME_DATABASE = mimetypes/mime/packages/freedesktop.org.xml + OTHER_FILES += $$MIME_DATABASE + + qtConfig(mimetype-database) { + outpath = .rcc + debug_and_release { + CONFIG(debug, debug|release): outpath = .rcc/debug + else: outpath = .rcc/release + } + + mimedb.depends = $$PWD/mime/generate.pl + equals(MAKEFILE_GENERATOR, MSVC.NET)|equals(MAKEFILE_GENERATOR, MSBUILD)|isEmpty(QMAKE_SH) { + mimedb.commands = cmd /c $$shell_path($$PWD/mime/generate.bat) + mimedb.depends += $$PWD/mime/generate.bat $$PWD/mime/hexdump.ps1 + } else { + mimedb.commands = perl $${mimedb.depends} + } + + qtConfig(zstd): mimedb.commands += --zstd + mimedb.commands += ${QMAKE_FILE_IN} > ${QMAKE_FILE_OUT} + + mimedb.output = $$outpath/qmimeprovider_database.cpp + mimedb.input = MIME_DATABASE + mimedb.variable_out = INCLUDED_SOURCES + QMAKE_EXTRA_COMPILERS += mimedb + INCLUDEPATH += $$outpath + unset(outpath) + } } diff --git a/src/corelib/mimetypes/qmimedatabase.cpp b/src/corelib/mimetypes/qmimedatabase.cpp index 1fbcc31fae..10b2190966 100644 --- a/src/corelib/mimetypes/qmimedatabase.cpp +++ b/src/corelib/mimetypes/qmimedatabase.cpp @@ -102,13 +102,18 @@ void QMimeDatabasePrivate::loadProviders() const auto fdoIterator = std::find_if(mimeDirs.constBegin(), mimeDirs.constEnd(), [](const QString &mimeDir) -> bool { return QFileInfo::exists(mimeDir + QStringLiteral("/packages/freedesktop.org.xml")); } ); - if (fdoIterator == mimeDirs.constEnd()) - mimeDirs.prepend(QLatin1String(":/qt-project.org/qmime")); //qDebug() << "mime dirs:" << mimeDirs; Providers currentProviders; std::swap(m_providers, currentProviders); - m_providers.reserve(mimeDirs.size()); + + if (QMimeXMLProvider::InternalDatabaseAvailable && fdoIterator == mimeDirs.constEnd()) { + m_providers.reserve(mimeDirs.size() + 1); + m_providers.push_back(Providers::value_type(new QMimeXMLProvider(this, QMimeXMLProvider::InternalDatabase))); + } else { + m_providers.reserve(mimeDirs.size()); + } + for (const QString &mimeDir : qAsConst(mimeDirs)) { const QString cacheFile = mimeDir + QStringLiteral("/mime.cache"); QFileInfo fileInfo(cacheFile); diff --git a/src/corelib/mimetypes/qmimeprovider.cpp b/src/corelib/mimetypes/qmimeprovider.cpp index c61759025c..4aee772366 100644 --- a/src/corelib/mimetypes/qmimeprovider.cpp +++ b/src/corelib/mimetypes/qmimeprovider.cpp @@ -1,7 +1,8 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. -** Copyright (C) 2015 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure <david.faure@kdab.com> +** Copyright (C) 2018 The Qt Company Ltd. +** Copyright (C) 2018 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure <david.faure@kdab.com> +** Copyright (C) 2019 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -45,6 +46,7 @@ #include "qmimemagicrulematcher_p.h" #include <QXmlStreamReader> +#include <QBuffer> #include <QDir> #include <QFile> #include <QByteArrayMatcher> @@ -52,12 +54,33 @@ #include <QDateTime> #include <QtEndian> -static void initResources() -{ #if QT_CONFIG(mimetype_database) - Q_INIT_RESOURCE(mimetypes); +# if defined(Q_CC_MSVC) +# pragma section(".qtmimedatabase", read, shared) +__declspec(allocate(".qtmimedatabase")) __declspec(align(4096)) +# elif defined(Q_OS_DARWIN) +__attribute__((section("__TEXT,.qtmimedatabase"), aligned(4096))) +# elif (defined(Q_OF_ELF) || defined(Q_OS_WIN)) && defined(Q_CC_GNU) +__attribute__((section(".qtmimedatabase"), aligned(4096))) +# endif + +# include "qmimeprovider_database.cpp" + +# ifdef MIME_DATABASE_IS_ZSTD +# if !QT_CONFIG(zstd) +# error "MIME database is zstd but no support compiled in!" +# endif +# include <zstd.h> +# endif +# ifdef MIME_DATABASE_IS_GZIP +# ifdef QT_NO_COMPRESS +# error "MIME database is zlib but no support compiled in!" +# endif +# define ZLIB_CONST +# include <zconf.h> +# include <zlib.h> +# endif #endif -} QT_BEGIN_NAMESPACE @@ -597,10 +620,55 @@ void QMimeBinaryProvider::loadGenericIcon(QMimeTypePrivate &data) //// +#if QT_CONFIG(mimetype_database) +static QString internalMimeFileName() +{ + return QStringLiteral("<internal MIME data>"); +} + +QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum) + : QMimeProviderBase(db, internalMimeFileName()) +{ + Q_STATIC_ASSERT_X(sizeof(mimetype_database), "Bundled MIME database is empty"); + Q_STATIC_ASSERT_X(sizeof(mimetype_database) <= MimeTypeDatabaseOriginalSize, + "Compressed MIME database is larger than the original size"); + Q_STATIC_ASSERT_X(MimeTypeDatabaseOriginalSize <= 16*1024*1024, + "Bundled MIME database is too big"); + const char *data = reinterpret_cast<const char *>(mimetype_database); + qsizetype size = MimeTypeDatabaseOriginalSize; + +#ifdef MIME_DATABASE_IS_ZSTD + // uncompress with libzstd + std::unique_ptr<char []> uncompressed(new char[size]); + size = ZSTD_decompress(uncompressed.get(), size, mimetype_database, sizeof(mimetype_database)); + Q_ASSERT(!ZSTD_isError(size)); + data = uncompressed.get(); +#elif defined(MIME_DATABASE_IS_GZIP) + std::unique_ptr<char []> uncompressed(new char[size]); + z_stream zs = {}; + zs.next_in = mimetype_database; + zs.avail_in = sizeof(mimetype_database); + zs.next_out = reinterpret_cast<Bytef *>(uncompressed.get()); + zs.avail_out = size; + + int res = inflateInit2(&zs, MAX_WBITS | 32); + Q_ASSERT(res == Z_OK); + res = inflate(&zs, Z_FINISH); + Q_ASSERT(res == Z_STREAM_END); + res = inflateEnd(&zs); + Q_ASSERT(res == Z_OK); + + data = uncompressed.get(); + size = zs.total_out; +#endif + + load(data, size); +} +#endif + QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory) : QMimeProviderBase(db, directory) { - initResources(); ensureLoaded(); } @@ -692,6 +760,19 @@ bool QMimeXMLProvider::load(const QString &fileName, QString *errorMessage) return parser.parse(&file, fileName, errorMessage); } +#if QT_CONFIG(mimetype_database) +void QMimeXMLProvider::load(const char *data, qsizetype len) +{ + QBuffer buffer; + buffer.setData(QByteArray::fromRawData(data, len)); + buffer.open(QIODevice::ReadOnly); + QString errorMessage; + QMimeTypeParser parser(*this); + if (!parser.parse(&buffer, internalMimeFileName(), &errorMessage)) + qWarning("QMimeDatabase: Error loading internal MIME data\n%s", qPrintable(errorMessage)); +} +#endif + void QMimeXMLProvider::addGlobPattern(const QMimeGlobPattern &glob) { m_mimeTypeGlobs.addGlob(glob); diff --git a/src/corelib/mimetypes/qmimeprovider_p.h b/src/corelib/mimetypes/qmimeprovider_p.h index b6268210c0..0629df8a95 100644 --- a/src/corelib/mimetypes/qmimeprovider_p.h +++ b/src/corelib/mimetypes/qmimeprovider_p.h @@ -132,6 +132,16 @@ private: class QMimeXMLProvider : public QMimeProviderBase { public: + enum InternalDatabaseEnum { InternalDatabase }; +#if QT_CONFIG(mimetype_database) + enum : bool { InternalDatabaseAvailable = true }; + QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum); +#else + enum : bool { InternalDatabaseAvailable = false }; + QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum) + : QMimeProviderBase(db, QString()) + { Q_UNREACHABLE() }; +#endif QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory); ~QMimeXMLProvider(); @@ -156,6 +166,7 @@ public: private: void load(const QString &fileName); + void load(const char *data, qsizetype len); typedef QHash<QString, QMimeType> NameMimeTypeMap; NameMimeTypeMap m_nameMimeTypeMap; diff --git a/src/src.pro b/src/src.pro index 8ff3ec4c1f..592f0cf644 100644 --- a/src/src.pro +++ b/src/src.pro @@ -70,7 +70,7 @@ src_winmain.depends = sub-corelib # just for the module .pri file src_corelib.subdir = $$PWD/corelib src_corelib.target = sub-corelib -src_corelib.depends = src_tools_moc src_tools_rcc src_tools_tracegen +src_corelib.depends = src_tools_moc src_tools_tracegen src_xml.subdir = $$PWD/xml src_xml.target = sub-xml @@ -118,7 +118,7 @@ src_angle.target = sub-angle src_gui.subdir = $$PWD/gui src_gui.target = sub-gui -src_gui.depends = src_corelib +src_gui.depends = src_corelib src_tools_rcc src_platformheaders.subdir = $$PWD/platformheaders src_platformheaders.target = sub-platformheaders diff --git a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp index 44c8c4b681..bdbc1c6928 100644 --- a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp +++ b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp @@ -30,7 +30,6 @@ #include <QtTest/QtTest> #include <QtCore/QCoreApplication> -#include <QtCore/QMimeDatabase> class tst_QResourceEngine: public QObject { @@ -119,24 +118,20 @@ void tst_QResourceEngine::checkStructure_data() QStringList rootContents; rootContents << QLatin1String("aliasdir") +#if defined(Q_OS_ANDROID) && !defined(Q_OS_ANDROID_EMBEDDED) + << QLatin1String("android_testdata") +#endif << QLatin1String("otherdir") - << QLatin1String("qt-project.org") << QLatin1String("runtime_resource") << QLatin1String("searchpath1") << QLatin1String("searchpath2") << QLatin1String("secondary_root") << QLatin1String("staticplugin") << QLatin1String("test") - << QLatin1String("withoutslashes"); - -#if defined(Q_OS_ANDROID) && !defined(Q_OS_ANDROID_EMBEDDED) - rootContents.insert(1, QLatin1String("android_testdata")); -#endif - #if defined(BUILTIN_TESTDATA) - rootContents.insert(9, QLatin1String("testqrc")); + << QLatin1String("testqrc") #endif - + << QLatin1String("withoutslashes"); QTest::newRow("root dir") << QString(":/") << QByteArray() @@ -352,11 +347,6 @@ void tst_QResourceEngine::checkStructure() QFETCH(QLocale, locale); QFETCH(qlonglong, contentsSize); - // We rely on the existence of the root "qt-project.org" in resources. For - // static builds on MSVC these resources are only added if they are used. - QMimeDatabase db; - Q_UNUSED(db); - bool directory = (containedDirs.size() + containedFiles.size() > 0); QLocale::setDefault(locale); |