From 0d4dea728ff3b7f15a7ddb22d94c54087e98f53d Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 31 May 2019 15:52:52 -0700 Subject: MIME: Make the internal database direct content, not a Qt resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This saves us from having to bootstrap rcc in regular (non-cross) compilations, as it can now link to QtCore. Actually un-bootstrapping rcc is left as an exercise for the reader. This commit discovered that MSVC cannot handle constexpr arrays bigger than 256 kB, at which point it simply starts claiming that the constant expressions using it are not constexpr. ICC has a similar problem at 64 kB, but it tells you why ("note: type "const unsigned char [65537]" too large for constant-expression evaluation"). Note also that this requires gzip or zstd to be in PATH for compression to happen. RCC linked to zlib, which is always present due to the bundled copy. gzip's presence is not likely to be a problem on Unix systems, but could be for Windows users, especially MSVC ones. If gzip is not present, QtCore's size will increase by about 1910 kB of read-only (sharable) data. Change-Id: I2b1955a995ad40f3b89afffd15a3e65a94670242 Reviewed-by: Edward Welbourne --- src/corelib/mimetypes/mime/generate.bat | 73 +++++++++++++ src/corelib/mimetypes/mime/generate.pl | 113 +++++++++++++++++++++ src/corelib/mimetypes/mime/hexdump.ps1 | 43 ++++++++ src/corelib/mimetypes/mimetypes.pri | 29 +++++- src/corelib/mimetypes/qmimedatabase.cpp | 11 +- src/corelib/mimetypes/qmimeprovider.cpp | 95 +++++++++++++++-- src/corelib/mimetypes/qmimeprovider_p.h | 11 ++ src/src.pro | 4 +- .../io/qresourceengine/tst_qresourceengine.cpp | 20 +--- 9 files changed, 371 insertions(+), 28 deletions(-) create mode 100644 src/corelib/mimetypes/mime/generate.bat create mode 100644 src/corelib/mimetypes/mime/generate.pl create mode 100644 src/corelib/mimetypes/mime/hexdump.ps1 diff --git a/src/corelib/mimetypes/mime/generate.bat b/src/corelib/mimetypes/mime/generate.bat new file mode 100644 index 0000000000..f63fc63693 --- /dev/null +++ b/src/corelib/mimetypes/mime/generate.bat @@ -0,0 +1,73 @@ +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:: +:: Copyright (C) 2019 Intel Corporation. +:: Contact: https://www.qt.io/licensing/ +:: +:: This file is part of the tools applications of the Qt Toolkit. +:: +:: $QT_BEGIN_LICENSE:GPL-EXCEPT$ +:: Commercial License Usage +:: Licensees holding valid commercial Qt licenses may use this file in +:: accordance with the commercial license agreement provided with the +:: Software or, alternatively, in accordance with the terms contained in +:: a written agreement between you and The Qt Company. For licensing terms +:: and conditions see https://www.qt.io/terms-conditions. For further +:: information use the contact form at https://www.qt.io/contact-us. +:: +:: GNU General Public License Usage +:: Alternatively, this file may be used under the terms of the GNU +:: General Public License version 3 as published by the Free Software +:: Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +:: included in the packaging of this file. Please review the following +:: information to ensure the GNU General Public License requirements will +:: be met: https://www.gnu.org/licenses/gpl-3.0.html. +:: +:: $QT_END_LICENSE$ +:: +::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +@echo off +setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS +set me=%~dp0 + +:: Check if certain tools are in PATH +for %%C in (gzip.exe zstd.exe perl.exe) do set %%C=%%~$PATH:C + +:: If perl is in PATH, just let it do everything +if not "%perl.exe%" == "" goto PuntToPerl + +set COMPRESS= +set MACRO=MIME_DATABASE_IS_UNCOMPRESSED +if not "%gzip.exe%" == "" ( + set COMPRESS=gzip -9 + set MACRO=MIME_DATABASE_IS_GZIP +) + +:: Check if zstd support was enabled +if /i "%~1" == "--zstd" ( + shift + if not "%zstd.exe%" == "" ( + set COMPRESS=zstd -19 + set MACRO=MIME_DATABASE_IS_ZSTD + ) +) + +if not "%COMPRESS%" == "" goto CompressedCommon + +:: No Compression and no Perl +:: Just hex-dump with Powershell +powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %1 %1 +exit /b %errorlevel% + +:CompressedCommon +:: Compress to a temporary file, then hex-dump using Powershell +echo #define %MACRO% +set tempfile=generate%RANDOM%.tmp +%COMPRESS% < %1 > %tempfile% +powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %tempfile% %1 +del %tempfile% +exit /b %errorlevel% + +:PuntToPerl +perl %me%generate.pl %* +exit /b %errorlevel% diff --git a/src/corelib/mimetypes/mime/generate.pl b/src/corelib/mimetypes/mime/generate.pl new file mode 100644 index 0000000000..0f87d61f8e --- /dev/null +++ b/src/corelib/mimetypes/mime/generate.pl @@ -0,0 +1,113 @@ +#!/usr/bin/perl +############################################################################# +## +## Copyright (C) 2019 Intel Corporation. +## Contact: https://www.qt.io/licensing/ +## +## This file is the build configuration utility of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# +use strict; +use warnings; +use Config; +local $/; # Enable "slurp" mode + +sub checkCommand($) { + use File::Spec::Functions; + my $cmd = $_[0] . $Config{_exe}; + for my $path (path()) { + return 1 if -x catfile($path, $cmd); + } + return 0; +} + +my $data; +my $compress; +my $macro; +my $zlib = eval 'use Compress::Zlib; use IO::Compress::Gzip; return 1;'; +my $fname = shift @ARGV; + +if ($zlib) { + # Prefer internal zlib support (useful on Windows where gzip.exe isn't + # always presnet) + $macro = "MIME_DATABASE_IS_GZIP"; +} elsif (checkCommand("gzip")) { + # No builtin support for compression (old Perl?) + $compress = "gzip -c9"; + $macro = "MIME_DATABASE_IS_GZIP"; +} + +# Check if Qt is being built with zstd support +if ($fname eq "--zstd") { + $fname = shift @ARGV; + if (checkCommand("zstd")) { + $compress = "zstd -cq19 --single-thread"; + $macro = "MIME_DATABASE_IS_ZSTD"; + } +} + +# Check if xml (from xmlstarlet) is in $PATH +my $cmd; +if (checkCommand("xml")) { + # Minify the data before compressing + $cmd = "xml sel -D -B -t -c / $fname"; + $cmd .= "| $compress" if $compress; +} elsif ($compress) { + $cmd = "$compress < $fname" +} +if ($cmd) { + # Run the command and read everything + open CMD, "$cmd |"; + $data = ; + close CMD; + die("Failed to run $cmd") if ($? >> 8); +} else { + # No command, just read the file + open F, "<$fname"; + $data = ; + close F; +} + +# Do we need to compress with zlib? +if (!$compress && $zlib) { + $data = eval q{ + use Compress::Zlib; + use IO::Compress::Gzip qw(gzip); + my $compressed; + gzip \$data => \$compressed, + Minimal => 1, + Level => Z_BEST_COMPRESSION; + return $compressed; + }; +} + +# Now print as hex +printf "#define %s\n", $macro if $macro; +printf "static const unsigned char mimetype_database[] = {"; +my $i = 0; +map { + printf "\n " if $i++ % 12 == 0; + printf "0x%02x, ", ord $_ +} split //, $data; +printf "\n};\n"; +printf "static constexpr size_t MimeTypeDatabaseOriginalSize = %d;\n", + (stat $fname)[7]; diff --git a/src/corelib/mimetypes/mime/hexdump.ps1 b/src/corelib/mimetypes/mime/hexdump.ps1 new file mode 100644 index 0000000000..25ce8138fa --- /dev/null +++ b/src/corelib/mimetypes/mime/hexdump.ps1 @@ -0,0 +1,43 @@ +############################################################################# +## +## Copyright (C) 2019 Intel Corporation. +## Contact: https://www.qt.io/licensing/ +## +## This file is the build configuration utility of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +param([String]$path, [String]$orig) + +"static const unsigned char mimetype_database[] = {" +ForEach ($byte in Get-Content -Encoding byte -ReadCount 16 -path $path) { +# if (($byte -eq 0).count -ne 16) { + $hex = $byte | Foreach-Object { + " 0x" + ("{0:x}" -f $_).PadLeft( 2, "0" ) + "," + } + " $hex" +# } +} +"};" + +$file = Get-Childitem -file $orig +"static constexpr size_t MimeTypeDatabaseOriginalSize = " + $file.length + ";" diff --git a/src/corelib/mimetypes/mimetypes.pri b/src/corelib/mimetypes/mimetypes.pri index 62bbe348e4..8cbe7b69ae 100644 --- a/src/corelib/mimetypes/mimetypes.pri +++ b/src/corelib/mimetypes/mimetypes.pri @@ -21,5 +21,32 @@ qtConfig(mimetype) { mimetypes/qmimeglobpattern.cpp \ mimetypes/qmimeprovider.cpp - qtConfig(mimetype-database): RESOURCES += mimetypes/mimetypes.qrc + MIME_DATABASE = mimetypes/mime/packages/freedesktop.org.xml + OTHER_FILES += $$MIME_DATABASE + + qtConfig(mimetype-database) { + outpath = .rcc + debug_and_release { + CONFIG(debug, debug|release): outpath = .rcc/debug + else: outpath = .rcc/release + } + + mimedb.depends = $$PWD/mime/generate.pl + equals(MAKEFILE_GENERATOR, MSVC.NET)|equals(MAKEFILE_GENERATOR, MSBUILD)|isEmpty(QMAKE_SH) { + mimedb.commands = cmd /c $$shell_path($$PWD/mime/generate.bat) + mimedb.depends += $$PWD/mime/generate.bat $$PWD/mime/hexdump.ps1 + } else { + mimedb.commands = perl $${mimedb.depends} + } + + qtConfig(zstd): mimedb.commands += --zstd + mimedb.commands += ${QMAKE_FILE_IN} > ${QMAKE_FILE_OUT} + + mimedb.output = $$outpath/qmimeprovider_database.cpp + mimedb.input = MIME_DATABASE + mimedb.variable_out = INCLUDED_SOURCES + QMAKE_EXTRA_COMPILERS += mimedb + INCLUDEPATH += $$outpath + unset(outpath) + } } diff --git a/src/corelib/mimetypes/qmimedatabase.cpp b/src/corelib/mimetypes/qmimedatabase.cpp index 1fbcc31fae..10b2190966 100644 --- a/src/corelib/mimetypes/qmimedatabase.cpp +++ b/src/corelib/mimetypes/qmimedatabase.cpp @@ -102,13 +102,18 @@ void QMimeDatabasePrivate::loadProviders() const auto fdoIterator = std::find_if(mimeDirs.constBegin(), mimeDirs.constEnd(), [](const QString &mimeDir) -> bool { return QFileInfo::exists(mimeDir + QStringLiteral("/packages/freedesktop.org.xml")); } ); - if (fdoIterator == mimeDirs.constEnd()) - mimeDirs.prepend(QLatin1String(":/qt-project.org/qmime")); //qDebug() << "mime dirs:" << mimeDirs; Providers currentProviders; std::swap(m_providers, currentProviders); - m_providers.reserve(mimeDirs.size()); + + if (QMimeXMLProvider::InternalDatabaseAvailable && fdoIterator == mimeDirs.constEnd()) { + m_providers.reserve(mimeDirs.size() + 1); + m_providers.push_back(Providers::value_type(new QMimeXMLProvider(this, QMimeXMLProvider::InternalDatabase))); + } else { + m_providers.reserve(mimeDirs.size()); + } + for (const QString &mimeDir : qAsConst(mimeDirs)) { const QString cacheFile = mimeDir + QStringLiteral("/mime.cache"); QFileInfo fileInfo(cacheFile); diff --git a/src/corelib/mimetypes/qmimeprovider.cpp b/src/corelib/mimetypes/qmimeprovider.cpp index c61759025c..4aee772366 100644 --- a/src/corelib/mimetypes/qmimeprovider.cpp +++ b/src/corelib/mimetypes/qmimeprovider.cpp @@ -1,7 +1,8 @@ /**************************************************************************** ** -** Copyright (C) 2016 The Qt Company Ltd. -** Copyright (C) 2015 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure +** Copyright (C) 2018 The Qt Company Ltd. +** Copyright (C) 2018 Klaralvdalens Datakonsult AB, a KDAB Group company, info@kdab.com, author David Faure +** Copyright (C) 2019 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -45,6 +46,7 @@ #include "qmimemagicrulematcher_p.h" #include +#include #include #include #include @@ -52,12 +54,33 @@ #include #include -static void initResources() -{ #if QT_CONFIG(mimetype_database) - Q_INIT_RESOURCE(mimetypes); +# if defined(Q_CC_MSVC) +# pragma section(".qtmimedatabase", read, shared) +__declspec(allocate(".qtmimedatabase")) __declspec(align(4096)) +# elif defined(Q_OS_DARWIN) +__attribute__((section("__TEXT,.qtmimedatabase"), aligned(4096))) +# elif (defined(Q_OF_ELF) || defined(Q_OS_WIN)) && defined(Q_CC_GNU) +__attribute__((section(".qtmimedatabase"), aligned(4096))) +# endif + +# include "qmimeprovider_database.cpp" + +# ifdef MIME_DATABASE_IS_ZSTD +# if !QT_CONFIG(zstd) +# error "MIME database is zstd but no support compiled in!" +# endif +# include +# endif +# ifdef MIME_DATABASE_IS_GZIP +# ifdef QT_NO_COMPRESS +# error "MIME database is zlib but no support compiled in!" +# endif +# define ZLIB_CONST +# include +# include +# endif #endif -} QT_BEGIN_NAMESPACE @@ -597,10 +620,55 @@ void QMimeBinaryProvider::loadGenericIcon(QMimeTypePrivate &data) //// +#if QT_CONFIG(mimetype_database) +static QString internalMimeFileName() +{ + return QStringLiteral(""); +} + +QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum) + : QMimeProviderBase(db, internalMimeFileName()) +{ + Q_STATIC_ASSERT_X(sizeof(mimetype_database), "Bundled MIME database is empty"); + Q_STATIC_ASSERT_X(sizeof(mimetype_database) <= MimeTypeDatabaseOriginalSize, + "Compressed MIME database is larger than the original size"); + Q_STATIC_ASSERT_X(MimeTypeDatabaseOriginalSize <= 16*1024*1024, + "Bundled MIME database is too big"); + const char *data = reinterpret_cast(mimetype_database); + qsizetype size = MimeTypeDatabaseOriginalSize; + +#ifdef MIME_DATABASE_IS_ZSTD + // uncompress with libzstd + std::unique_ptr uncompressed(new char[size]); + size = ZSTD_decompress(uncompressed.get(), size, mimetype_database, sizeof(mimetype_database)); + Q_ASSERT(!ZSTD_isError(size)); + data = uncompressed.get(); +#elif defined(MIME_DATABASE_IS_GZIP) + std::unique_ptr uncompressed(new char[size]); + z_stream zs = {}; + zs.next_in = mimetype_database; + zs.avail_in = sizeof(mimetype_database); + zs.next_out = reinterpret_cast(uncompressed.get()); + zs.avail_out = size; + + int res = inflateInit2(&zs, MAX_WBITS | 32); + Q_ASSERT(res == Z_OK); + res = inflate(&zs, Z_FINISH); + Q_ASSERT(res == Z_STREAM_END); + res = inflateEnd(&zs); + Q_ASSERT(res == Z_OK); + + data = uncompressed.get(); + size = zs.total_out; +#endif + + load(data, size); +} +#endif + QMimeXMLProvider::QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory) : QMimeProviderBase(db, directory) { - initResources(); ensureLoaded(); } @@ -692,6 +760,19 @@ bool QMimeXMLProvider::load(const QString &fileName, QString *errorMessage) return parser.parse(&file, fileName, errorMessage); } +#if QT_CONFIG(mimetype_database) +void QMimeXMLProvider::load(const char *data, qsizetype len) +{ + QBuffer buffer; + buffer.setData(QByteArray::fromRawData(data, len)); + buffer.open(QIODevice::ReadOnly); + QString errorMessage; + QMimeTypeParser parser(*this); + if (!parser.parse(&buffer, internalMimeFileName(), &errorMessage)) + qWarning("QMimeDatabase: Error loading internal MIME data\n%s", qPrintable(errorMessage)); +} +#endif + void QMimeXMLProvider::addGlobPattern(const QMimeGlobPattern &glob) { m_mimeTypeGlobs.addGlob(glob); diff --git a/src/corelib/mimetypes/qmimeprovider_p.h b/src/corelib/mimetypes/qmimeprovider_p.h index b6268210c0..0629df8a95 100644 --- a/src/corelib/mimetypes/qmimeprovider_p.h +++ b/src/corelib/mimetypes/qmimeprovider_p.h @@ -132,6 +132,16 @@ private: class QMimeXMLProvider : public QMimeProviderBase { public: + enum InternalDatabaseEnum { InternalDatabase }; +#if QT_CONFIG(mimetype_database) + enum : bool { InternalDatabaseAvailable = true }; + QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum); +#else + enum : bool { InternalDatabaseAvailable = false }; + QMimeXMLProvider(QMimeDatabasePrivate *db, InternalDatabaseEnum) + : QMimeProviderBase(db, QString()) + { Q_UNREACHABLE() }; +#endif QMimeXMLProvider(QMimeDatabasePrivate *db, const QString &directory); ~QMimeXMLProvider(); @@ -156,6 +166,7 @@ public: private: void load(const QString &fileName); + void load(const char *data, qsizetype len); typedef QHash NameMimeTypeMap; NameMimeTypeMap m_nameMimeTypeMap; diff --git a/src/src.pro b/src/src.pro index 8ff3ec4c1f..592f0cf644 100644 --- a/src/src.pro +++ b/src/src.pro @@ -70,7 +70,7 @@ src_winmain.depends = sub-corelib # just for the module .pri file src_corelib.subdir = $$PWD/corelib src_corelib.target = sub-corelib -src_corelib.depends = src_tools_moc src_tools_rcc src_tools_tracegen +src_corelib.depends = src_tools_moc src_tools_tracegen src_xml.subdir = $$PWD/xml src_xml.target = sub-xml @@ -118,7 +118,7 @@ src_angle.target = sub-angle src_gui.subdir = $$PWD/gui src_gui.target = sub-gui -src_gui.depends = src_corelib +src_gui.depends = src_corelib src_tools_rcc src_platformheaders.subdir = $$PWD/platformheaders src_platformheaders.target = sub-platformheaders diff --git a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp index 44c8c4b681..bdbc1c6928 100644 --- a/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp +++ b/tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp @@ -30,7 +30,6 @@ #include #include -#include class tst_QResourceEngine: public QObject { @@ -119,24 +118,20 @@ void tst_QResourceEngine::checkStructure_data() QStringList rootContents; rootContents << QLatin1String("aliasdir") +#if defined(Q_OS_ANDROID) && !defined(Q_OS_ANDROID_EMBEDDED) + << QLatin1String("android_testdata") +#endif << QLatin1String("otherdir") - << QLatin1String("qt-project.org") << QLatin1String("runtime_resource") << QLatin1String("searchpath1") << QLatin1String("searchpath2") << QLatin1String("secondary_root") << QLatin1String("staticplugin") << QLatin1String("test") - << QLatin1String("withoutslashes"); - -#if defined(Q_OS_ANDROID) && !defined(Q_OS_ANDROID_EMBEDDED) - rootContents.insert(1, QLatin1String("android_testdata")); -#endif - #if defined(BUILTIN_TESTDATA) - rootContents.insert(9, QLatin1String("testqrc")); + << QLatin1String("testqrc") #endif - + << QLatin1String("withoutslashes"); QTest::newRow("root dir") << QString(":/") << QByteArray() @@ -352,11 +347,6 @@ void tst_QResourceEngine::checkStructure() QFETCH(QLocale, locale); QFETCH(qlonglong, contentsSize); - // We rely on the existence of the root "qt-project.org" in resources. For - // static builds on MSVC these resources are only added if they are used. - QMimeDatabase db; - Q_UNUSED(db); - bool directory = (containedDirs.size() + containedFiles.size() > 0); QLocale::setDefault(locale); -- cgit v1.2.3