From 430f202718f7311c541a7243870719e2df456f2f Mon Sep 17 00:00:00 2001 From: Pierre Rossi Date: Wed, 20 Nov 2013 13:45:03 +0100 Subject: Ship repacked .pak files We used to wrap various .pak files in qrc files, but it turns out to be very memory inefficient compared to chromium's approach of mmaping those files. Drop the pak->qrc logic and instead add some pure gyp targets to repack the resources. We then install those with qmake and look them up with QLibraryInfo. Change-Id: I6dd3cedf6afa626ed181463911fef8885c9e9add Reviewed-by: Andras Becsi --- src/core/core.gyp | 12 +++ src/core/core.pro | 12 ++- src/core/core_gyp_generator.pro | 10 +-- src/core/core_resources.qrc | 5 -- src/core/devtools.qrc | 1 - src/core/qtwebengine.gypi | 7 +- src/core/resource_bundle_qt.cpp | 113 +++---------------------- src/core/resources/repack_resources.gypi | 23 +++++ src/core/resources/resources.gyp | 18 ++++ src/core/resources/resources.pro | 5 -- tools/buildscripts/build_resources.py | 103 ---------------------- tools/buildscripts/gyp_qtwebengine | 2 +- tools/qmake/mkspecs/features/gyp_generator.prf | 20 +---- 13 files changed, 79 insertions(+), 252 deletions(-) create mode 100644 src/core/core.gyp delete mode 100644 src/core/core_resources.qrc create mode 100644 src/core/resources/repack_resources.gypi create mode 100644 src/core/resources/resources.gyp delete mode 100644 src/core/resources/resources.pro delete mode 100755 tools/buildscripts/build_resources.py diff --git a/src/core/core.gyp b/src/core/core.gyp new file mode 100644 index 000000000..ea5478cf1 --- /dev/null +++ b/src/core/core.gyp @@ -0,0 +1,12 @@ +{ + 'targets': [ + { + 'target_name': 'qtwebengine', + 'type': 'none', + 'dependencies': [ + 'core_generated.gyp:*', + 'resources/resources.gyp:*', + ], + }, + ] +} diff --git a/src/core/core.pro b/src/core/core.pro index b6a856545..3c3110257 100644 --- a/src/core/core.pro +++ b/src/core/core.pro @@ -13,10 +13,16 @@ gyp_configure_target.depends = gyp_configure_host # gyp_run.pro calls gyp through gyp_qtwebengine on the qmake step, and ninja on the make step. gyp_run.file = gyp_run.pro -gyp_run.depends = resources core_gyp_generator gyp_configure_host gyp_configure_target +gyp_run.depends = core_gyp_generator gyp_configure_host gyp_configure_target SUBDIRS += core_gyp_generator \ gyp_configure_host \ gyp_configure_target \ - gyp_run \ - resources \ + gyp_run + +REPACK_DIR = $$getOutDir()/$$getConfigDir()/gen/repack +resources.files = $$REPACK_DIR/qtwebengine_resources.pak +resources.path = $$[QT_INSTALL_DATA] + +INSTALLS += resources + diff --git a/src/core/core_gyp_generator.pro b/src/core/core_gyp_generator.pro index d145f7b07..a13e8e26e 100644 --- a/src/core/core_gyp_generator.pro +++ b/src/core/core_gyp_generator.pro @@ -2,8 +2,7 @@ # We want the gyp generation step to happen after all the other config steps. For that we need to prepend # our gyp_generator.prf feature to the CONFIG variable since it is processed backwards CONFIG = gyp_generator $$CONFIG -GYPFILE = $$PWD/core.gyp -GYPDEPENDENCIES += <(chromium_src_dir)/content/browser/devtools/devtools_resources.gyp:devtools_resources +GYPFILE = $$PWD/core_generated.gyp GYPINCLUDES += qtwebengine.gypi TEMPLATE = lib @@ -19,9 +18,6 @@ QT += qml quick QT_PRIVATE += qml-private quick-private gui-private core-private qtHaveModule(v8): QT_PRIVATE += v8-private -COPY_FILES = <(SHARED_INTERMEDIATE_DIR)/webkit/devtools_resources.pak -COPY_DESTINATIONS = resources/ - # Defining keywords such as 'signal' clashes with the chromium code base. DEFINES += QT_NO_KEYWORDS \ Q_FORWARD_DECLARE_OBJC_CLASS=QT_FORWARD_DECLARE_CLASS @@ -32,9 +28,7 @@ PER_CONFIG_DEFINES = QTWEBENGINEPROCESS_PATH=\\\"$$getOutDir()/%config/$$QTWEBEN # Keep Skia happy CONFIG(release, debug|release): DEFINES += NDEBUG -RESOURCES += core_resources.qrc devtools.qrc -# We need this to find the include files generated for the .pak resource files. -INCLUDEPATH += $$absolute_path(resources, $$PWD) +RESOURCES += devtools.qrc # something fishy with qmake in 5.2 ? INCLUDEPATH += $$[QT_INSTALL_HEADERS] diff --git a/src/core/core_resources.qrc b/src/core/core_resources.qrc deleted file mode 100644 index 9dbdf90af..000000000 --- a/src/core/core_resources.qrc +++ /dev/null @@ -1,5 +0,0 @@ - - - resources/net_resources.pak - - diff --git a/src/core/devtools.qrc b/src/core/devtools.qrc index 96d24761d..6656304ac 100644 --- a/src/core/devtools.qrc +++ b/src/core/devtools.qrc @@ -1,6 +1,5 @@ - resources/devtools_resources.pak resources/devtools_discovery_page.html diff --git a/src/core/qtwebengine.gypi b/src/core/qtwebengine.gypi index f13e781f5..5e36383ff 100644 --- a/src/core/qtwebengine.gypi +++ b/src/core/qtwebengine.gypi @@ -32,13 +32,13 @@ '<(chromium_src_dir)/ui/ui.gyp:ui', '<(chromium_src_dir)/url/url.gyp:url_lib', '<(chromium_src_dir)/v8/tools/gyp/v8.gyp:v8', - '<(chromium_src_dir)/webkit/webkit_resources.gyp:webkit_resources', '<(chromium_src_dir)/webkit/support/webkit_support.gyp:webkit_support', '<(chromium_src_dir)/third_party/WebKit/Source/web/web.gyp:webkit', ], 'include_dirs': [ - '<(qtwebengine_src_dir)', + '<(qtwebengine_root)/src/core', '<(chromium_src_dir)', + '<(SHARED_INTERMEDIATE_DIR)/net', # Needed to include grit/net_resources.h ], # Chromium code defines those in common.gypi, do the same for our code that include Chromium headers. 'defines': [ @@ -60,9 +60,6 @@ 'resource_include_dirs': [ '<(SHARED_INTERMEDIATE_DIR)/webkit', ], - 'dependencies': [ - '<(DEPTH)/webkit/webkit_resources.gyp:webkit_strings', - ], 'configurations': { 'Debug_Base': { 'msvs_settings': { diff --git a/src/core/resource_bundle_qt.cpp b/src/core/resource_bundle_qt.cpp index 044d00511..ea893d330 100644 --- a/src/core/resource_bundle_qt.cpp +++ b/src/core/resource_bundle_qt.cpp @@ -39,116 +39,25 @@ ** ****************************************************************************/ +#include "base/command_line.h" +#include "content/public/common/content_switches.h" #include "ui/base/resource/resource_bundle.h" -#include "ui/base/resource/data_pack.h" +#include "type_conversion.h" -#include -#include +#include +#include +#include namespace ui { -// ********************* data_pack.cc ********************* -// This is duplicated code originating from data_pack.cc. -// It should instead be moved to a header file and be included -// in both places. - -static const uint32 kFileFormatVersion = 4; -static const size_t kHeaderLength = 2 * sizeof(uint32) + sizeof(uint8); - -#pragma pack(push,2) -struct DataPackEntry { - uint16 resource_id; - uint32 file_offset; - - static int CompareById(const void* void_key, const void* void_entry) { - uint16 key = *reinterpret_cast(void_key); - const DataPackEntry* entry = - reinterpret_cast(void_entry); - if (key < entry->resource_id) { - return -1; - } else if (key > entry->resource_id) { - return 1; - } else { - return 0; - } - } -}; -#pragma pack(pop) -// ******************* data_pack.cc end ******************* - -class UI_EXPORT DataPackQt : public DataPack { - public: - DataPackQt(ui::ScaleFactor scale_factor) - : DataPack(scale_factor) - , m_data(NULL) - , m_resourceCount(0) { } - - virtual ~DataPackQt() { } - - bool LoadFromByteArray(const QByteArray& data) - { - m_data = data; - - if (kHeaderLength > static_cast(m_data.size())) - return false; - - const uint32* ptr = reinterpret_cast(m_data.data()); - uint32 version = ptr[0]; - if (version != kFileFormatVersion) { - LOG(ERROR) << "Bad data pack version: got " << version << ", expected " << kFileFormatVersion; - return false; - } - - m_resourceCount = ptr[1]; - return true; - } - - virtual bool HasResource(uint16 resource_id) const OVERRIDE - { - return !!bsearch(&resource_id, m_data.data() + kHeaderLength, m_resourceCount, sizeof(DataPackEntry), DataPackEntry::CompareById); - } - - virtual bool GetStringPiece(uint16 resource_id, base::StringPiece* data) const OVERRIDE - { - #if defined(__BYTE_ORDER) // Linux check - COMPILE_ASSERT(__BYTE_ORDER == __LITTLE_ENDIAN, datapack_assumes_little_endian); - #elif defined(__BIG_ENDIAN__) // Mac check - #error DataPack assumes little endian - #endif - - const DataPackEntry* target = reinterpret_cast(bsearch(&resource_id, m_data.data() + kHeaderLength, m_resourceCount, sizeof(DataPackEntry), DataPackEntry::CompareById)); - if (!target) - return false; - - const DataPackEntry* next_entry = target + 1; - size_t length = next_entry->file_offset - target->file_offset; - - data->set(m_data.data() + target->file_offset, length); - return true; - } - - private: - QByteArray m_data; - size_t m_resourceCount; - DISALLOW_COPY_AND_ASSIGN(DataPackQt); -}; - - void ResourceBundle::LoadCommonResources() { - QStringList resources; - resources << ":/data/resources.pak" << ":/data/devtools.pak"; - Q_FOREACH (const QString& pak, resources) { - QFile pak_file(pak); - if (!pak_file.open(QIODevice::ReadOnly)) { - qWarning("Resource file %s not loaded", qPrintable(pak)); - continue; - } + // Loading these resources probably only makes sense for the browser process + if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kProcessType)) + return; - scoped_ptr data_pack(new DataPackQt(SCALE_FACTOR_100P)); - if (data_pack->LoadFromByteArray(pak_file.readAll())) - AddDataPack(data_pack.release()); - } + // We repacked the resources we need and installed them. now let chromium mmap that file. + AddDataPackFromPath(base::FilePath(toFilePathString(QLibraryInfo::location(QLibraryInfo::DataPath) % QStringLiteral("/qtwebengine_resources.pak"))) , SCALE_FACTOR_100P); } // As GetLocaleFilePath is excluded for Mac in resource_bundle.cc, diff --git a/src/core/resources/repack_resources.gypi b/src/core/resources/repack_resources.gypi new file mode 100644 index 000000000..49bd52b8e --- /dev/null +++ b/src/core/resources/repack_resources.gypi @@ -0,0 +1,23 @@ +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +{ + 'action_name': 'repack_resources', + 'variables': { + 'repack_path': '<(chromium_src_dir)/tools/grit/grit/format/repack.py', + 'pak_inputs': [ + '<(SHARED_INTERMEDIATE_DIR)/net/net_resources.pak', + '<(SHARED_INTERMEDIATE_DIR)/webkit/devtools_resources.pak', +# FIXME: we'll probably want those as well +# '<(SHARED_INTERMEDIATE_DIR)/ui/ui_resources/webui_resources.pak', + ], + }, + 'inputs': [ + '<(repack_path)', + '<@(pak_inputs)', + ], + 'outputs': [ + '<(SHARED_INTERMEDIATE_DIR)/repack/qtwebengine_resources.pak', + ], + 'action': ['python', '<(repack_path)', '<@(_outputs)', '<@(pak_inputs)'], +} diff --git a/src/core/resources/resources.gyp b/src/core/resources/resources.gyp new file mode 100644 index 000000000..4747829ae --- /dev/null +++ b/src/core/resources/resources.gyp @@ -0,0 +1,18 @@ +{ + 'dependencies': [ + '<(chromium_src_dir)/webkit/webkit_resources.gyp:webkit_resources', + '<(chromium_src_dir)/content/browser/devtools/devtools_resources.gyp:devtools_resources', + ], + 'targets': [ + { + 'target_name': 'qtwebengine_resources', + 'type': 'none', + 'actions' : [ + { + 'action_name': 'repack_resources', + 'includes': [ 'repack_resources.gypi' ], + }, + ] + } + ] +} diff --git a/src/core/resources/resources.pro b/src/core/resources/resources.pro deleted file mode 100644 index b0eba1246..000000000 --- a/src/core/resources/resources.pro +++ /dev/null @@ -1,5 +0,0 @@ -# This is a dummy .pro file used to prepare chromium .pak resource files. -# These files will then be bundled using the Qt Resource System. -TEMPLATE = aux - -system("python $$QTWEBENGINE_ROOT/tools/buildscripts/build_resources.py") diff --git a/tools/buildscripts/build_resources.py b/tools/buildscripts/build_resources.py deleted file mode 100755 index 56d8ed404..000000000 --- a/tools/buildscripts/build_resources.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - -############################################################################# -# -# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). -# Contact: http://www.qt-project.org/legal -# -# This file is part of the QtWebEngine module of the Qt Toolkit. -# -# $QT_BEGIN_LICENSE:LGPL$ -# Commercial License Usage -# Licensees holding valid commercial Qt licenses may use this file in -# accordance with the commercial license agreement provided with the -# Software or, alternatively, in accordance with the terms contained in -# a written agreement between you and Digia. For licensing terms and -# conditions see http://qt.digia.com/licensing. For further information -# use the contact form at http://qt.digia.com/contact-us. -# -# GNU Lesser General Public License Usage -# Alternatively, this file may be used under the terms of the GNU Lesser -# General Public License version 2.1 as published by the Free Software -# Foundation and appearing in the file LICENSE.LGPL included in the -# packaging of this file. Please review the following information to -# ensure the GNU Lesser General Public License version 2.1 requirements -# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -# -# In addition, as a special exception, Digia gives you certain additional -# rights. These rights are described in the Digia Qt LGPL Exception -# version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -# -# GNU General Public License Usage -# Alternatively, this file may be used under the terms of the GNU -# General Public License version 3.0 as published by the Free Software -# Foundation and appearing in the file LICENSE.GPL included in the -# packaging of this file. Please review the following information to -# ensure the GNU General Public License version 3.0 requirements will be -# met: http://www.gnu.org/copyleft/gpl.html. -# -# -# $QT_END_LICENSE$ -# -############################################################################# - -import glob -import os -import subprocess -import sys -import string -import time - -qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) - - -chrome_src = subprocess.check_output("git config qtwebengine.chromiumsrcdir || true", shell=True).strip() -if chrome_src: - chrome_src = os.path.join(qtwebengine_root, chrome_src) -if not chrome_src or not os.path.isdir(chrome_src): - chrome_src = os.path.join(qtwebengine_root, 'src/3rdparty/chromium') - print 'CHROMIUM_SRC_DIR not set, falling back to ' + chrome_src - -grit_tool = os.path.join(chrome_src, 'tools/grit/grit.py') -resources_subdir = os.path.join(qtwebengine_root, 'src/core/resources') - -def checkNeedForRebuild(grd_file): - grit_files = subprocess.check_output(['python', grit_tool, '-i', grd_file, 'buildinfo']).splitlines() - - dependencies = [grd_file] - data_packages = [] - for line in grit_files: - if line.startswith('input|'): - dependencies.append(line.split('|')[1]) - if line.startswith('data_package|'): - data_packages.append(line.split('|')[1]) - - target_timestamp = 0 - for data_package in data_packages: - data_package_file = os.path.join(resources_subdir, data_package) - if not os.path.isfile(data_package_file): - return True - - data_package_timestamp = os.path.getmtime(data_package_file) - if data_package_timestamp < target_timestamp or target_timestamp == 0: - target_timestamp = data_package_timestamp - - for dependency in dependencies: - dependency_timestamp = os.path.getmtime(dependency) - if (dependency_timestamp > target_timestamp): - return True - return False - -def rebuildPakFile(grd_file): - print 'Rebuilding resource file for:' + grd_file - resource_ids_file = os.path.join(chrome_src, 'tools/gritsettings/resource_ids') - subprocess.call(['python', grit_tool, '-i', grd_file, 'build', '-f', resource_ids_file, '-o', resources_subdir]) - -def rebuildIfNeeded(grd_file): - grd_file = os.path.join(chrome_src, grd_file) - if checkNeedForRebuild(grd_file): - rebuildPakFile(grd_file) - - -# The grd_file is specified relative to the chromium source directory. -rebuildIfNeeded('net/base/net_resources.grd') diff --git a/tools/buildscripts/gyp_qtwebengine b/tools/buildscripts/gyp_qtwebengine index 48491e600..b7dd5ea12 100755 --- a/tools/buildscripts/gyp_qtwebengine +++ b/tools/buildscripts/gyp_qtwebengine @@ -135,7 +135,7 @@ if __name__ == '__main__': args.extend(["--toplevel-dir=" + toplevel]) # Chromium specific Hack: for Chromium to build, the depth has to be set to the chromium src dir. args.extend(["--depth=" + chrome_src]) - args.extend(['-D', 'qtwebengine_src_dir=' + qtwebengine_src]) + args.extend(['-D', 'qtwebengine_root=' + qtwebengine_root]) args.extend(['-D', 'chromium_src_dir=' + chrome_src]) if 'qt_cross_compile=1' in sys.argv: diff --git a/tools/qmake/mkspecs/features/gyp_generator.prf b/tools/qmake/mkspecs/features/gyp_generator.prf index 96de5e280..6ae886a33 100644 --- a/tools/qmake/mkspecs/features/gyp_generator.prf +++ b/tools/qmake/mkspecs/features/gyp_generator.prf @@ -114,11 +114,6 @@ macx { GYP_CONTENTS += " ]," \ " }," -!isEmpty(GYPDEPENDENCIES) { - GYP_CONTENTS += " 'dependencies': [" - for (dep, GYPDEPENDENCIES): GYP_CONTENTS += " '$$dep'," - GYP_CONTENTS += " ]," -} !isEmpty(DEFINES) { GYP_CONTENTS += " 'defines': [" for (define, DEFINES): GYP_CONTENTS += " '$$define'," @@ -172,20 +167,7 @@ GYP_CONTENTS += " ]," GYP_CONTENTS += " ]," } -# Some needed files (like devtools_resources.pak) are both _generated_ as part of the build process and are _needed_ as part of the build process. -!isEmpty(COPY_FILES) { - GYP_CONTENTS += " 'copies': [" - for (index, 0..$$size(COPY_FILES)) { - copyFile = $$member(COPY_FILES, $$index) - !isEmpty(copyFile) { - copyDestination = $$member(COPY_DESTINATIONS, $$index) - GYP_CONTENTS += " {'destination': '$$copyDestination', 'files': ['$$copyFile']}," - } - } - GYP_CONTENTS += " ]," -} - -# Generate the actions for moc, copy +# Generate the actions for moc and rcc GYP_CONTENTS += " 'actions': [" for(resourcefile, RESOURCES): GYP_CONTENTS += $$rccAction($$resourcefile) for(header, MOCABLE_HEADERS): GYP_CONTENTS += $$mocAction($$header) -- cgit v1.2.3