Ship repacked .pak files

We used to wrap various .pak files in qrc files, but it turns out to be
very memory inefficient compared to chromium's approach of mmaping
those files.

Drop the pak->qrc logic and instead add some pure gyp targets to repack
the resources.
We then install those with qmake and look them up with QLibraryInfo.

Change-Id: I6dd3cedf6afa626ed181463911fef8885c9e9add
Reviewed-by: Andras Becsi <andras.becsi@digia.com>

10 files changed, 77 insertions, 129 deletions

diff --git a/src/core/core.gyp b/src/core/core.gyp
new file mode 100644
index 000000000..ea5478cf1
--- /dev/null
+++ b/src/core/core.gyp
@@ -0,0 +1,12 @@
+{
+  'targets': [
+  {
+    'target_name': 'qtwebengine',
+    'type': 'none',
+    'dependencies': [
+      'core_generated.gyp:*',
+      'resources/resources.gyp:*',
+    ],
+  },
+  ]
+}
diff --git a/src/core/core.pro b/src/core/core.pro
index b6a856545..3c3110257 100644
--- a/src/core/core.pro
+++ b/src/core/core.pro
@@ -13,10 +13,16 @@ gyp_configure_target.depends = gyp_configure_host
 # gyp_run.pro calls gyp through gyp_qtwebengine on the qmake step, and ninja on the make step.
 gyp_run.file = gyp_run.pro
 
-gyp_run.depends = resources core_gyp_generator gyp_configure_host gyp_configure_target
+gyp_run.depends = core_gyp_generator gyp_configure_host gyp_configure_target
 
 SUBDIRS += core_gyp_generator \
            gyp_configure_host \
            gyp_configure_target \
-           gyp_run \
-           resources \
+           gyp_run
+
+REPACK_DIR = $$getOutDir()/$$getConfigDir()/gen/repack
+resources.files = $$REPACK_DIR/qtwebengine_resources.pak
+resources.path = $$[QT_INSTALL_DATA]
+
+INSTALLS += resources
+
diff --git a/src/core/core_gyp_generator.pro b/src/core/core_gyp_generator.pro
index d145f7b07..a13e8e26e 100644
--- a/src/core/core_gyp_generator.pro
+++ b/src/core/core_gyp_generator.pro
@@ -2,8 +2,7 @@
 # We want the gyp generation step to happen after all the other config steps. For that we need to prepend
 # our gyp_generator.prf feature to the CONFIG variable since it is processed backwards
 CONFIG = gyp_generator $$CONFIG
-GYPFILE = $$PWD/core.gyp
-GYPDEPENDENCIES += <(chromium_src_dir)/content/browser/devtools/devtools_resources.gyp:devtools_resources
+GYPFILE = $$PWD/core_generated.gyp
 GYPINCLUDES += qtwebengine.gypi
 
 TEMPLATE = lib
@@ -19,9 +18,6 @@ QT += qml quick
 QT_PRIVATE += qml-private quick-private gui-private core-private
 qtHaveModule(v8): QT_PRIVATE += v8-private
 
-COPY_FILES = <(SHARED_INTERMEDIATE_DIR)/webkit/devtools_resources.pak
-COPY_DESTINATIONS = resources/
-
 # Defining keywords such as 'signal' clashes with the chromium code base.
 DEFINES += QT_NO_KEYWORDS \
            Q_FORWARD_DECLARE_OBJC_CLASS=QT_FORWARD_DECLARE_CLASS
@@ -32,9 +28,7 @@ PER_CONFIG_DEFINES = QTWEBENGINEPROCESS_PATH=\\\"$$getOutDir()/%config/$$QTWEBEN
 # Keep Skia happy
 CONFIG(release, debug|release): DEFINES += NDEBUG
 
-RESOURCES += core_resources.qrc devtools.qrc
-# We need this to find the include files generated for the .pak resource files.
-INCLUDEPATH += $$absolute_path(resources, $$PWD)
+RESOURCES += devtools.qrc
 
 # something fishy with qmake in 5.2 ?
 INCLUDEPATH += $$[QT_INSTALL_HEADERS]
diff --git a/src/core/core_resources.qrc b/src/core/core_resources.qrc
deleted file mode 100644
index 9dbdf90af..000000000
--- a/src/core/core_resources.qrc
+++ /dev/null
@@ -1,5 +0,0 @@
-<!DOCTYPE RCC><RCC version="1.0">
-<qresource prefix="data">
-    <file alias="resources.pak">resources/net_resources.pak</file>
-</qresource>
-</RCC>
diff --git a/src/core/devtools.qrc b/src/core/devtools.qrc
index 96d24761d..6656304ac 100644
--- a/src/core/devtools.qrc
+++ b/src/core/devtools.qrc
@@ -1,6 +1,5 @@
 <!DOCTYPE RCC><RCC version="1.0">
 <qresource prefix="data">
-    <file alias="devtools.pak">resources/devtools_resources.pak</file>
     <file alias="discovery_page.html">resources/devtools_discovery_page.html</file>
 </qresource>
 </RCC>
diff --git a/src/core/qtwebengine.gypi b/src/core/qtwebengine.gypi
index f13e781f5..5e36383ff 100644
--- a/src/core/qtwebengine.gypi
+++ b/src/core/qtwebengine.gypi
@@ -32,13 +32,13 @@
       '<(chromium_src_dir)/ui/ui.gyp:ui',
       '<(chromium_src_dir)/url/url.gyp:url_lib',
       '<(chromium_src_dir)/v8/tools/gyp/v8.gyp:v8',
-      '<(chromium_src_dir)/webkit/webkit_resources.gyp:webkit_resources',
       '<(chromium_src_dir)/webkit/support/webkit_support.gyp:webkit_support',
       '<(chromium_src_dir)/third_party/WebKit/Source/web/web.gyp:webkit',
     ],
     'include_dirs': [
-      '<(qtwebengine_src_dir)',
+      '<(qtwebengine_root)/src/core',
       '<(chromium_src_dir)',
+      '<(SHARED_INTERMEDIATE_DIR)/net', # Needed to include grit/net_resources.h
     ],
     # Chromium code defines those in common.gypi, do the same for our code that include Chromium headers.
     'defines': [
@@ -60,9 +60,6 @@
       'resource_include_dirs': [
         '<(SHARED_INTERMEDIATE_DIR)/webkit',
       ],
-      'dependencies': [
-        '<(DEPTH)/webkit/webkit_resources.gyp:webkit_strings',
-      ],
       'configurations': {
         'Debug_Base': {
           'msvs_settings': {
diff --git a/src/core/resource_bundle_qt.cpp b/src/core/resource_bundle_qt.cpp
index 044d00511..ea893d330 100644
--- a/src/core/resource_bundle_qt.cpp
+++ b/src/core/resource_bundle_qt.cpp
@@ -39,116 +39,25 @@
 **
 ****************************************************************************/
 
+#include "base/command_line.h"
+#include "content/public/common/content_switches.h"
 #include "ui/base/resource/resource_bundle.h"
-#include "ui/base/resource/data_pack.h"
+#include "type_conversion.h"
 
-#include <QFile>
-#include <QStringList>
+#include <QFileInfo>
+#include <QLibraryInfo>
+#include <QStringBuilder>
 
 namespace ui {
 
-// ********************* data_pack.cc *********************
-// This is duplicated code originating from data_pack.cc.
-// It should instead be moved to a header file and be included
-// in both places.
-
-static const uint32 kFileFormatVersion = 4;
-static const size_t kHeaderLength = 2 * sizeof(uint32) + sizeof(uint8);
-
-#pragma pack(push,2)
-struct DataPackEntry {
-  uint16 resource_id;
-  uint32 file_offset;
-
-  static int CompareById(const void* void_key, const void* void_entry) {
-    uint16 key = *reinterpret_cast<const uint16*>(void_key);
-    const DataPackEntry* entry =
-        reinterpret_cast<const DataPackEntry*>(void_entry);
-    if (key < entry->resource_id) {
-      return -1;
-    } else if (key > entry->resource_id) {
-      return 1;
-    } else {
-      return 0;
-    }
-  }
-};
-#pragma pack(pop)
-// ******************* data_pack.cc end *******************
-
-class UI_EXPORT DataPackQt : public DataPack {
-    public:
-        DataPackQt(ui::ScaleFactor scale_factor)
-            : DataPack(scale_factor)
-            , m_data(NULL)
-            , m_resourceCount(0) { }
-
-        virtual ~DataPackQt() { }
-
-        bool LoadFromByteArray(const QByteArray& data)
-        {
-            m_data = data;
-
-            if (kHeaderLength > static_cast<size_t>(m_data.size()))
-                return false;
-
-            const uint32* ptr = reinterpret_cast<const uint32*>(m_data.data());
-            uint32 version = ptr[0];
-            if (version != kFileFormatVersion) {
-                LOG(ERROR) << "Bad data pack version: got " << version << ", expected " << kFileFormatVersion;
-                return false;
-            }
-
-            m_resourceCount = ptr[1];
-            return true;
-        }
-
-        virtual bool HasResource(uint16 resource_id) const OVERRIDE
-        {
-            return !!bsearch(&resource_id, m_data.data() + kHeaderLength, m_resourceCount, sizeof(DataPackEntry), DataPackEntry::CompareById);
-        }
-
-        virtual bool GetStringPiece(uint16 resource_id, base::StringPiece* data) const OVERRIDE
-        {
-            #if defined(__BYTE_ORDER) // Linux check
-                COMPILE_ASSERT(__BYTE_ORDER == __LITTLE_ENDIAN, datapack_assumes_little_endian);
-            #elif defined(__BIG_ENDIAN__) // Mac check
-                #error DataPack assumes little endian
-            #endif
-
-            const DataPackEntry* target = reinterpret_cast<const DataPackEntry*>(bsearch(&resource_id, m_data.data() + kHeaderLength, m_resourceCount, sizeof(DataPackEntry), DataPackEntry::CompareById));
-            if (!target)
-                return false;
-
-            const DataPackEntry* next_entry = target + 1;
-            size_t length = next_entry->file_offset - target->file_offset;
-
-            data->set(m_data.data() + target->file_offset, length);
-            return true;
-        }
-
-    private:
-        QByteArray m_data;
-        size_t m_resourceCount;
-        DISALLOW_COPY_AND_ASSIGN(DataPackQt);
-};
-
-
 void ResourceBundle::LoadCommonResources()
 {
-    QStringList resources;
-    resources << ":/data/resources.pak" << ":/data/devtools.pak";
-    Q_FOREACH (const QString& pak, resources) {
-        QFile pak_file(pak);
-        if (!pak_file.open(QIODevice::ReadOnly)) {
-            qWarning("Resource file %s not loaded", qPrintable(pak));
-            continue;
-        }
+    // Loading these resources probably only makes sense for the browser process
+    if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kProcessType))
+        return;
 
-        scoped_ptr<DataPackQt> data_pack(new DataPackQt(SCALE_FACTOR_100P));
-        if (data_pack->LoadFromByteArray(pak_file.readAll()))
-            AddDataPack(data_pack.release());
-    }
+    // We repacked the resources we need and installed them. now let chromium mmap that file.
+    AddDataPackFromPath(base::FilePath(toFilePathString(QLibraryInfo::location(QLibraryInfo::DataPath) % QStringLiteral("/qtwebengine_resources.pak"))) , SCALE_FACTOR_100P);
 }
 
 // As GetLocaleFilePath is excluded for Mac in resource_bundle.cc,
diff --git a/src/core/resources/repack_resources.gypi b/src/core/resources/repack_resources.gypi
new file mode 100644
index 000000000..49bd52b8e
--- /dev/null
+++ b/src/core/resources/repack_resources.gypi
@@ -0,0 +1,23 @@
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+{
+  'action_name': 'repack_resources',
+  'variables': {
+    'repack_path': '<(chromium_src_dir)/tools/grit/grit/format/repack.py',
+    'pak_inputs': [
+      '<(SHARED_INTERMEDIATE_DIR)/net/net_resources.pak',
+      '<(SHARED_INTERMEDIATE_DIR)/webkit/devtools_resources.pak',
+# FIXME: we'll probably want those as well
+#      '<(SHARED_INTERMEDIATE_DIR)/ui/ui_resources/webui_resources.pak',
+    ],
+  },
+  'inputs': [
+    '<(repack_path)',
+    '<@(pak_inputs)',
+  ],
+  'outputs': [
+    '<(SHARED_INTERMEDIATE_DIR)/repack/qtwebengine_resources.pak',
+  ],
+  'action': ['python', '<(repack_path)', '<@(_outputs)', '<@(pak_inputs)'],
+}
diff --git a/src/core/resources/resources.gyp b/src/core/resources/resources.gyp
new file mode 100644
index 000000000..4747829ae
--- /dev/null
+++ b/src/core/resources/resources.gyp
@@ -0,0 +1,18 @@
+{
+  'dependencies': [
+      '<(chromium_src_dir)/webkit/webkit_resources.gyp:webkit_resources',
+      '<(chromium_src_dir)/content/browser/devtools/devtools_resources.gyp:devtools_resources',
+  ],
+  'targets': [
+  {
+    'target_name': 'qtwebengine_resources',
+    'type': 'none',
+    'actions' : [
+      {
+        'action_name': 'repack_resources',
+        'includes': [ 'repack_resources.gypi' ],
+      },
+    ]
+  }
+  ]
+}
diff --git a/src/core/resources/resources.pro b/src/core/resources/resources.pro
deleted file mode 100644
index b0eba1246..000000000
--- a/src/core/resources/resources.pro
+++ /dev/null
@@ -1,5 +0,0 @@
-# This is a dummy .pro file used to prepare chromium .pak resource files.
-# These files will then be bundled using the Qt Resource System.
-TEMPLATE = aux
-
-system("python $$QTWEBENGINE_ROOT/tools/buildscripts/build_resources.py")