Ship repacked .pak files

We used to wrap various .pak files in qrc files, but it turns out to be
very memory inefficient compared to chromium's approach of mmaping
those files.

Drop the pak->qrc logic and instead add some pure gyp targets to repack
the resources.
We then install those with qmake and look them up with QLibraryInfo.

Change-Id: I6dd3cedf6afa626ed181463911fef8885c9e9add
Reviewed-by: Andras Becsi <andras.becsi@digia.com>

13 files changed, 79 insertions, 252 deletions

diff --git a/src/core/core.gyp b/src/core/core.gyp
new file mode 100644
index 000000000..ea5478cf1
--- /dev/null
+++ b/src/core/core.gyp
@@ -0,0 +1,12 @@
+{
+  'targets': [
+  {
+    'target_name': 'qtwebengine',
+    'type': 'none',
+    'dependencies': [
+      'core_generated.gyp:*',
+      'resources/resources.gyp:*',
+    ],
+  },
+  ]
+}
diff --git a/src/core/core.pro b/src/core/core.pro
index b6a856545..3c3110257 100644
--- a/src/core/core.pro
+++ b/src/core/core.pro
@@ -13,10 +13,16 @@ gyp_configure_target.depends = gyp_configure_host
 # gyp_run.pro calls gyp through gyp_qtwebengine on the qmake step, and ninja on the make step.
 gyp_run.file = gyp_run.pro
 
-gyp_run.depends = resources core_gyp_generator gyp_configure_host gyp_configure_target
+gyp_run.depends = core_gyp_generator gyp_configure_host gyp_configure_target
 
 SUBDIRS += core_gyp_generator \
            gyp_configure_host \
            gyp_configure_target \
-           gyp_run \
-           resources \
+           gyp_run
+
+REPACK_DIR = $$getOutDir()/$$getConfigDir()/gen/repack
+resources.files = $$REPACK_DIR/qtwebengine_resources.pak
+resources.path = $$[QT_INSTALL_DATA]
+
+INSTALLS += resources
+
diff --git a/src/core/core_gyp_generator.pro b/src/core/core_gyp_generator.pro
index d145f7b07..a13e8e26e 100644
--- a/src/core/core_gyp_generator.pro
+++ b/src/core/core_gyp_generator.pro
@@ -2,8 +2,7 @@
 # We want the gyp generation step to happen after all the other config steps. For that we need to prepend
 # our gyp_generator.prf feature to the CONFIG variable since it is processed backwards
 CONFIG = gyp_generator $$CONFIG
-GYPFILE = $$PWD/core.gyp
-GYPDEPENDENCIES += <(chromium_src_dir)/content/browser/devtools/devtools_resources.gyp:devtools_resources
+GYPFILE = $$PWD/core_generated.gyp
 GYPINCLUDES += qtwebengine.gypi
 
 TEMPLATE = lib
@@ -19,9 +18,6 @@ QT += qml quick
 QT_PRIVATE += qml-private quick-private gui-private core-private
 qtHaveModule(v8): QT_PRIVATE += v8-private
 
-COPY_FILES = <(SHARED_INTERMEDIATE_DIR)/webkit/devtools_resources.pak
-COPY_DESTINATIONS = resources/
-
 # Defining keywords such as 'signal' clashes with the chromium code base.
 DEFINES += QT_NO_KEYWORDS \
            Q_FORWARD_DECLARE_OBJC_CLASS=QT_FORWARD_DECLARE_CLASS
@@ -32,9 +28,7 @@ PER_CONFIG_DEFINES = QTWEBENGINEPROCESS_PATH=\\\"$$getOutDir()/%config/$$QTWEBEN
 # Keep Skia happy
 CONFIG(release, debug|release): DEFINES += NDEBUG
 
-RESOURCES += core_resources.qrc devtools.qrc
-# We need this to find the include files generated for the .pak resource files.
-INCLUDEPATH += $$absolute_path(resources, $$PWD)
+RESOURCES += devtools.qrc
 
 # something fishy with qmake in 5.2 ?
 INCLUDEPATH += $$[QT_INSTALL_HEADERS]
diff --git a/src/core/core_resources.qrc b/src/core/core_resources.qrc
deleted file mode 100644
index 9dbdf90af..000000000
--- a/src/core/core_resources.qrc
+++ /dev/null
@@ -1,5 +0,0 @@
-<!DOCTYPE RCC><RCC version="1.0">
-<qresource prefix="data">
-    <file alias="resources.pak">resources/net_resources.pak</file>
-</qresource>
-</RCC>
diff --git a/src/core/devtools.qrc b/src/core/devtools.qrc
index 96d24761d..6656304ac 100644
--- a/src/core/devtools.qrc
+++ b/src/core/devtools.qrc
@@ -1,6 +1,5 @@
 <!DOCTYPE RCC><RCC version="1.0">
 <qresource prefix="data">
-    <file alias="devtools.pak">resources/devtools_resources.pak</file>
     <file alias="discovery_page.html">resources/devtools_discovery_page.html</file>
 </qresource>
 </RCC>
diff --git a/src/core/qtwebengine.gypi b/src/core/qtwebengine.gypi
index f13e781f5..5e36383ff 100644
--- a/src/core/qtwebengine.gypi
+++ b/src/core/qtwebengine.gypi
@@ -32,13 +32,13 @@
       '<(chromium_src_dir)/ui/ui.gyp:ui',
       '<(chromium_src_dir)/url/url.gyp:url_lib',
       '<(chromium_src_dir)/v8/tools/gyp/v8.gyp:v8',
-      '<(chromium_src_dir)/webkit/webkit_resources.gyp:webkit_resources',
       '<(chromium_src_dir)/webkit/support/webkit_support.gyp:webkit_support',
       '<(chromium_src_dir)/third_party/WebKit/Source/web/web.gyp:webkit',
     ],
     'include_dirs': [
-      '<(qtwebengine_src_dir)',
+      '<(qtwebengine_root)/src/core',
       '<(chromium_src_dir)',
+      '<(SHARED_INTERMEDIATE_DIR)/net', # Needed to include grit/net_resources.h
     ],
     # Chromium code defines those in common.gypi, do the same for our code that include Chromium headers.
     'defines': [
@@ -60,9 +60,6 @@
       'resource_include_dirs': [
         '<(SHARED_INTERMEDIATE_DIR)/webkit',
       ],
-      'dependencies': [
-        '<(DEPTH)/webkit/webkit_resources.gyp:webkit_strings',
-      ],
       'configurations': {
         'Debug_Base': {
           'msvs_settings': {
diff --git a/src/core/resource_bundle_qt.cpp b/src/core/resource_bundle_qt.cpp
index 044d00511..ea893d330 100644
--- a/src/core/resource_bundle_qt.cpp
+++ b/src/core/resource_bundle_qt.cpp
@@ -39,116 +39,25 @@
 **
 ****************************************************************************/
 
+#include "base/command_line.h"
+#include "content/public/common/content_switches.h"
 #include "ui/base/resource/resource_bundle.h"
-#include "ui/base/resource/data_pack.h"
+#include "type_conversion.h"
 
-#include <QFile>
-#include <QStringList>
+#include <QFileInfo>
+#include <QLibraryInfo>
+#include <QStringBuilder>
 
 namespace ui {
 
-// ********************* data_pack.cc *********************
-// This is duplicated code originating from data_pack.cc.
-// It should instead be moved to a header file and be included
-// in both places.
-
-static const uint32 kFileFormatVersion = 4;
-static const size_t kHeaderLength = 2 * sizeof(uint32) + sizeof(uint8);
-
-#pragma pack(push,2)
-struct DataPackEntry {
-  uint16 resource_id;
-  uint32 file_offset;
-
-  static int CompareById(const void* void_key, const void* void_entry) {
-    uint16 key = *reinterpret_cast<const uint16*>(void_key);
-    const DataPackEntry* entry =
-        reinterpret_cast<const DataPackEntry*>(void_entry);
-    if (key < entry->resource_id) {
-      return -1;
-    } else if (key > entry->resource_id) {
-      return 1;
-    } else {
-      return 0;
-    }
-  }
-};
-#pragma pack(pop)
-// ******************* data_pack.cc end *******************
-
-class UI_EXPORT DataPackQt : public DataPack {
-    public:
-        DataPackQt(ui::ScaleFactor scale_factor)
-            : DataPack(scale_factor)
-            , m_data(NULL)
-            , m_resourceCount(0) { }
-
-        virtual ~DataPackQt() { }
-
-        bool LoadFromByteArray(const QByteArray& data)
-        {
-            m_data = data;
-
-            if (kHeaderLength > static_cast<size_t>(m_data.size()))
-                return false;
-
-            const uint32* ptr = reinterpret_cast<const uint32*>(m_data.data());
-            uint32 version = ptr[0];
-            if (version != kFileFormatVersion) {
-                LOG(ERROR) << "Bad data pack version: got " << version << ", expected " << kFileFormatVersion;
-                return false;
-            }
-
-            m_resourceCount = ptr[1];
-            return true;
-        }
-
-        virtual bool HasResource(uint16 resource_id) const OVERRIDE
-        {
-            return !!bsearch(&resource_id, m_data.data() + kHeaderLength, m_resourceCount, sizeof(DataPackEntry), DataPackEntry::CompareById);
-        }
-
-        virtual bool GetStringPiece(uint16 resource_id, base::StringPiece* data) const OVERRIDE
-        {
-            #if defined(__BYTE_ORDER) // Linux check
-                COMPILE_ASSERT(__BYTE_ORDER == __LITTLE_ENDIAN, datapack_assumes_little_endian);
-            #elif defined(__BIG_ENDIAN__) // Mac check
-                #error DataPack assumes little endian
-            #endif
-
-            const DataPackEntry* target = reinterpret_cast<const DataPackEntry*>(bsearch(&resource_id, m_data.data() + kHeaderLength, m_resourceCount, sizeof(DataPackEntry), DataPackEntry::CompareById));
-            if (!target)
-                return false;
-
-            const DataPackEntry* next_entry = target + 1;
-            size_t length = next_entry->file_offset - target->file_offset;
-
-            data->set(m_data.data() + target->file_offset, length);
-            return true;
-        }
-
-    private:
-        QByteArray m_data;
-        size_t m_resourceCount;
-        DISALLOW_COPY_AND_ASSIGN(DataPackQt);
-};
-
-
 void ResourceBundle::LoadCommonResources()
 {
-    QStringList resources;
-    resources << ":/data/resources.pak" << ":/data/devtools.pak";
-    Q_FOREACH (const QString& pak, resources) {
-        QFile pak_file(pak);
-        if (!pak_file.open(QIODevice::ReadOnly)) {
-            qWarning("Resource file %s not loaded", qPrintable(pak));
-            continue;
-        }
+    // Loading these resources probably only makes sense for the browser process
+    if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kProcessType))
+        return;
 
-        scoped_ptr<DataPackQt> data_pack(new DataPackQt(SCALE_FACTOR_100P));
-        if (data_pack->LoadFromByteArray(pak_file.readAll()))
-            AddDataPack(data_pack.release());
-    }
+    // We repacked the resources we need and installed them. now let chromium mmap that file.
+    AddDataPackFromPath(base::FilePath(toFilePathString(QLibraryInfo::location(QLibraryInfo::DataPath) % QStringLiteral("/qtwebengine_resources.pak"))) , SCALE_FACTOR_100P);
 }
 
 // As GetLocaleFilePath is excluded for Mac in resource_bundle.cc,
diff --git a/src/core/resources/repack_resources.gypi b/src/core/resources/repack_resources.gypi
new file mode 100644
index 000000000..49bd52b8e
--- /dev/null
+++ b/src/core/resources/repack_resources.gypi
@@ -0,0 +1,23 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+{
+  'action_name': 'repack_resources',
+  'variables': {
+    'repack_path': '<(chromium_src_dir)/tools/grit/grit/format/repack.py',
+    'pak_inputs': [
+      '<(SHARED_INTERMEDIATE_DIR)/net/net_resources.pak',
+      '<(SHARED_INTERMEDIATE_DIR)/webkit/devtools_resources.pak',
+# FIXME: we'll probably want those as well
+#      '<(SHARED_INTERMEDIATE_DIR)/ui/ui_resources/webui_resources.pak',
+    ],
+  },
+  'inputs': [
+    '<(repack_path)',
+    '<@(pak_inputs)',
+  ],
+  'outputs': [
+    '<(SHARED_INTERMEDIATE_DIR)/repack/qtwebengine_resources.pak',
+  ],
+  'action': ['python', '<(repack_path)', '<@(_outputs)', '<@(pak_inputs)'],
+}
diff --git a/src/core/resources/resources.gyp b/src/core/resources/resources.gyp
new file mode 100644
index 000000000..4747829ae
--- /dev/null
+++ b/src/core/resources/resources.gyp
@@ -0,0 +1,18 @@
+{
+  'dependencies': [
+      '<(chromium_src_dir)/webkit/webkit_resources.gyp:webkit_resources',
+      '<(chromium_src_dir)/content/browser/devtools/devtools_resources.gyp:devtools_resources',
+  ],
+  'targets': [
+  {
+    'target_name': 'qtwebengine_resources',
+    'type': 'none',
+    'actions' : [
+      {
+        'action_name': 'repack_resources',
+        'includes': [ 'repack_resources.gypi' ],
+      },
+    ]
+  }
+  ]
+}
diff --git a/src/core/resources/resources.pro b/src/core/resources/resources.pro
deleted file mode 100644
index b0eba1246..000000000
--- a/src/core/resources/resources.pro
+++ /dev/null
@@ -1,5 +0,0 @@
-# This is a dummy .pro file used to prepare chromium .pak resource files.
-# These files will then be bundled using the Qt Resource System.
-TEMPLATE = aux
-
-system("python $$QTWEBENGINE_ROOT/tools/buildscripts/build_resources.py")
diff --git a/tools/buildscripts/build_resources.py b/tools/buildscripts/build_resources.py
deleted file mode 100755
index 56d8ed404..000000000
--- a/tools/buildscripts/build_resources.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-
-#############################################################################
-#
-# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
-# Contact: http://www.qt-project.org/legal
-#
-# This file is part of the QtWebEngine module of the Qt Toolkit.
-#
-# $QT_BEGIN_LICENSE:LGPL$
-# Commercial License Usage
-# Licensees holding valid commercial Qt licenses may use this file in
-# accordance with the commercial license agreement provided with the
-# Software or, alternatively, in accordance with the terms contained in
-# a written agreement between you and Digia.  For licensing terms and
-# conditions see http://qt.digia.com/licensing.  For further information
-# use the contact form at http://qt.digia.com/contact-us.
-#
-# GNU Lesser General Public License Usage
-# Alternatively, this file may be used under the terms of the GNU Lesser
-# General Public License version 2.1 as published by the Free Software
-# Foundation and appearing in the file LICENSE.LGPL included in the
-# packaging of this file.  Please review the following information to
-# ensure the GNU Lesser General Public License version 2.1 requirements
-# will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-#
-# In addition, as a special exception, Digia gives you certain additional
-# rights.  These rights are described in the Digia Qt LGPL Exception
-# version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
-#
-# GNU General Public License Usage
-# Alternatively, this file may be used under the terms of the GNU
-# General Public License version 3.0 as published by the Free Software
-# Foundation and appearing in the file LICENSE.GPL included in the
-# packaging of this file.  Please review the following information to
-# ensure the GNU General Public License version 3.0 requirements will be
-# met: http://www.gnu.org/copyleft/gpl.html.
-#
-#
-# $QT_END_LICENSE$
-#
-#############################################################################
-
-import glob
-import os
-import subprocess
-import sys
-import string
-import time
-
-qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
-
-
-chrome_src = subprocess.check_output("git config qtwebengine.chromiumsrcdir || true", shell=True).strip()
-if chrome_src:
-  chrome_src = os.path.join(qtwebengine_root, chrome_src)
-if not chrome_src or not os.path.isdir(chrome_src):
-  chrome_src = os.path.join(qtwebengine_root, 'src/3rdparty/chromium')
-  print 'CHROMIUM_SRC_DIR not set, falling back to ' + chrome_src
-
-grit_tool = os.path.join(chrome_src, 'tools/grit/grit.py')
-resources_subdir = os.path.join(qtwebengine_root, 'src/core/resources')
-
-def checkNeedForRebuild(grd_file):
-    grit_files = subprocess.check_output(['python', grit_tool, '-i', grd_file, 'buildinfo']).splitlines()
-
-    dependencies = [grd_file]
-    data_packages = []
-    for line in grit_files:
-        if line.startswith('input|'):
-            dependencies.append(line.split('|')[1])
-        if line.startswith('data_package|'):
-            data_packages.append(line.split('|')[1])
-
-    target_timestamp = 0
-    for data_package in data_packages:
-        data_package_file = os.path.join(resources_subdir, data_package)
-        if not os.path.isfile(data_package_file):
-            return True
-
-        data_package_timestamp = os.path.getmtime(data_package_file)
-        if data_package_timestamp < target_timestamp or target_timestamp == 0:
-            target_timestamp = data_package_timestamp
-
-    for dependency in dependencies:
-        dependency_timestamp = os.path.getmtime(dependency)
-        if (dependency_timestamp > target_timestamp):
-            return True
-    return False
-
-def rebuildPakFile(grd_file):
-    print 'Rebuilding resource file for:' + grd_file
-    resource_ids_file = os.path.join(chrome_src, 'tools/gritsettings/resource_ids')
-    subprocess.call(['python', grit_tool, '-i', grd_file, 'build', '-f', resource_ids_file, '-o', resources_subdir])
-
-def rebuildIfNeeded(grd_file):
-    grd_file = os.path.join(chrome_src, grd_file)
-    if checkNeedForRebuild(grd_file):
-        rebuildPakFile(grd_file)
-
-
-# The grd_file is specified relative to the chromium source directory.
-rebuildIfNeeded('net/base/net_resources.grd')
diff --git a/tools/buildscripts/gyp_qtwebengine b/tools/buildscripts/gyp_qtwebengine
index 48491e600..b7dd5ea12 100755
--- a/tools/buildscripts/gyp_qtwebengine
+++ b/tools/buildscripts/gyp_qtwebengine
@@ -135,7 +135,7 @@ if __name__ == '__main__':
   args.extend(["--toplevel-dir=" + toplevel])
   # Chromium specific Hack: for Chromium to build, the depth has to be set to the chromium src dir.
   args.extend(["--depth=" + chrome_src])
-  args.extend(['-D', 'qtwebengine_src_dir=' + qtwebengine_src])
+  args.extend(['-D', 'qtwebengine_root=' + qtwebengine_root])
   args.extend(['-D', 'chromium_src_dir=' + chrome_src])
 
   if 'qt_cross_compile=1' in sys.argv:
diff --git a/tools/qmake/mkspecs/features/gyp_generator.prf b/tools/qmake/mkspecs/features/gyp_generator.prf
index 96de5e280..6ae886a33 100644
--- a/tools/qmake/mkspecs/features/gyp_generator.prf
+++ b/tools/qmake/mkspecs/features/gyp_generator.prf
@@ -114,11 +114,6 @@ macx {
 GYP_CONTENTS += "      ]," \
              "    },"
 
-!isEmpty(GYPDEPENDENCIES) {
-  GYP_CONTENTS += "    'dependencies': ["
-  for (dep, GYPDEPENDENCIES): GYP_CONTENTS += "      '$$dep',"
-  GYP_CONTENTS += "    ],"
-}
 !isEmpty(DEFINES) {
   GYP_CONTENTS += "    'defines': ["
   for (define, DEFINES): GYP_CONTENTS += "      '$$define',"
@@ -172,20 +167,7 @@ GYP_CONTENTS += "    ],"
   GYP_CONTENTS += "    ],"
 }
 
-# Some needed files (like devtools_resources.pak) are both _generated_ as part of the build process and are _needed_ as part of the build process.
-!isEmpty(COPY_FILES) {
-  GYP_CONTENTS += "    'copies': ["
-  for (index, 0..$$size(COPY_FILES)) {
-    copyFile = $$member(COPY_FILES, $$index)
-    !isEmpty(copyFile) {
-      copyDestination = $$member(COPY_DESTINATIONS, $$index)
-      GYP_CONTENTS += "        {'destination': '$$copyDestination', 'files': ['$$copyFile']},"
-    }
-  }
-  GYP_CONTENTS += "    ],"
-}
-
-# Generate the actions for moc, copy
+# Generate the actions for moc and rcc
 GYP_CONTENTS += "    'actions': ["
 for(resourcefile, RESOURCES): GYP_CONTENTS += $$rccAction($$resourcefile)
 for(header, MOCABLE_HEADERS): GYP_CONTENTS += $$mocAction($$header)