Bundle pinyin dictionary in the plugin

This change modifies the pinyin library and adds support for loading
the dictionary from the qt resource system.

[ChangeLog] The pinyin dictionary is now bundled in the plugin by default.

Task-number: QTBUG-66198
Change-Id: I17459c4bde89cc4ccd96ad5c97393a5ad502b902
Reviewed-by: Andy Shaw <andy.shaw@qt.io>
Reviewed-by: Mitch Curtis <mitch.curtis@qt.io>

13 files changed, 76 insertions, 65 deletions

diff --git a/src/virtualkeyboard/3rdparty/pinyin/include/dictlist.h b/src/virtualkeyboard/3rdparty/pinyin/include/dictlist.h
index 27fa6d8e..1c1daef4 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/include/dictlist.h
+++ b/src/virtualkeyboard/3rdparty/pinyin/include/dictlist.h
@@ -80,7 +80,7 @@ class DictList {
   ~DictList();
 
   bool save_list(FILE *fp);
-  bool load_list(FILE *fp);
+  bool load_list(QFile *fp);
 
 #ifdef ___BUILD_MODEL___
   // Init the list from the LemmaEntry array.
diff --git a/src/virtualkeyboard/3rdparty/pinyin/include/dicttrie.h b/src/virtualkeyboard/3rdparty/pinyin/include/dicttrie.h
index 75b7ee05..86a8ee25 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/include/dicttrie.h
+++ b/src/virtualkeyboard/3rdparty/pinyin/include/dicttrie.h
@@ -22,6 +22,7 @@
 #include "./dictdef.h"
 #include "./dictlist.h"
 #include "./searchutility.h"
+#include <QFile>
 
 namespace ime_pinyin {
 
@@ -105,7 +106,7 @@ class DictTrie : AtomDictBase {
 
   void free_resource(bool free_dict_list);
 
-  bool load_dict(FILE *fp);
+  bool load_dict(QFile *fp);
 
   // Given a LmaNodeLE0 node, extract the lemmas specified by it, and fill
   // them into the lpi_items buffer.
diff --git a/src/virtualkeyboard/3rdparty/pinyin/include/ngram.h b/src/virtualkeyboard/3rdparty/pinyin/include/ngram.h
index 7adb46d8..1d3a86e6 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/include/ngram.h
+++ b/src/virtualkeyboard/3rdparty/pinyin/include/ngram.h
@@ -20,6 +20,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include "./dictdef.h"
+#include <QFile>
 
 namespace ime_pinyin {
 
@@ -71,7 +72,7 @@ class NGram {
   static NGram& get_instance();
 
   bool save_ngram(FILE *fp);
-  bool load_ngram(FILE *fp);
+  bool load_ngram(QFile *fp);
 
   // Set the total frequency of all none system dictionaries.
   void set_total_freq_none_sys(size_t freq_none_sys);
diff --git a/src/virtualkeyboard/3rdparty/pinyin/include/spellingtrie.h b/src/virtualkeyboard/3rdparty/pinyin/include/spellingtrie.h
index 03510ed3..f943a24d 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/include/spellingtrie.h
+++ b/src/virtualkeyboard/3rdparty/pinyin/include/spellingtrie.h
@@ -20,6 +20,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include "./dictdef.h"
+#include <QFile>
 
 namespace ime_pinyin {
 
@@ -233,7 +234,7 @@ class SpellingTrie {
   bool save_spl_trie(FILE *fp);
 
   // Load from the file stream
-  bool load_spl_trie(FILE *fp);
+  bool load_spl_trie(QFile *fp);
 
   // Get the number of spellings
   size_t get_spelling_num();
diff --git a/src/virtualkeyboard/3rdparty/pinyin/pinyin.pro b/src/virtualkeyboard/3rdparty/pinyin/pinyin.pro
index 9ad9a318..6b528867 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/pinyin.pro
+++ b/src/virtualkeyboard/3rdparty/pinyin/pinyin.pro
@@ -50,10 +50,8 @@ OTHER_FILES +=\
 
 load(qt_helper_lib)
 
-# On Windows, the library uses Qt for platform abstraction.
-win32 {
-    CONFIG += qt
-    QT = core
-} else {
+CONFIG += qt
+QT = core
+!win32 {
     CONFIG *= thread
 }
diff --git a/src/virtualkeyboard/3rdparty/pinyin/share/dictlist.cpp b/src/virtualkeyboard/3rdparty/pinyin/share/dictlist.cpp
index 64d8d085..a5238aeb 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/share/dictlist.cpp
+++ b/src/virtualkeyboard/3rdparty/pinyin/share/dictlist.cpp
@@ -408,21 +408,21 @@ bool DictList::save_list(FILE *fp) {
   return true;
 }
 
-bool DictList::load_list(FILE *fp) {
+bool DictList::load_list(QFile *fp) {
   if (NULL == fp)
     return false;
 
   initialized_ = false;
 
-  if (fread(&scis_num_, sizeof(uint32), 1, fp) != 1)
+  if (fp->read((char *)&scis_num_, sizeof(uint32)) != sizeof(uint32))
     return false;
 
-  if (fread(start_pos_, sizeof(uint32), kMaxLemmaSize + 1, fp) !=
-      kMaxLemmaSize + 1)
+  if (fp->read((char *)start_pos_, sizeof(uint32) * (kMaxLemmaSize + 1)) !=
+      sizeof(uint32) * (kMaxLemmaSize + 1))
     return false;
 
-  if (fread(start_id_, sizeof(uint32), kMaxLemmaSize + 1, fp) !=
-      kMaxLemmaSize + 1)
+  if (fp->read((char *)start_id_, sizeof(uint32) * (kMaxLemmaSize + 1)) !=
+      sizeof(uint32) * (kMaxLemmaSize + 1))
     return false;
 
   free_resource();
@@ -430,14 +430,14 @@ bool DictList::load_list(FILE *fp) {
   if (!alloc_resource(start_pos_[kMaxLemmaSize], scis_num_))
     return false;
 
-  if (fread(scis_hz_, sizeof(char16), scis_num_, fp) != scis_num_)
+  if (fp->read((char *)scis_hz_, sizeof(char16) * scis_num_) != sizeof(char16) * scis_num_)
     return false;
 
-  if (fread(scis_splid_, sizeof(SpellingId), scis_num_, fp) != scis_num_)
+  if (fp->read((char *)scis_splid_, sizeof(SpellingId) * scis_num_) != sizeof(SpellingId) * scis_num_)
     return false;
 
-  if (fread(buf_, sizeof(char16), start_pos_[kMaxLemmaSize], fp) !=
-      start_pos_[kMaxLemmaSize])
+  if (fp->read((char *)buf_, sizeof(char16) * start_pos_[kMaxLemmaSize]) !=
+      sizeof(char16) * start_pos_[kMaxLemmaSize])
     return false;
 
   initialized_ = true;
diff --git a/src/virtualkeyboard/3rdparty/pinyin/share/dicttrie.cpp b/src/virtualkeyboard/3rdparty/pinyin/share/dicttrie.cpp
index 0cdd0982..e7d194f6 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/share/dicttrie.cpp
+++ b/src/virtualkeyboard/3rdparty/pinyin/share/dicttrie.cpp
@@ -165,19 +165,19 @@ bool DictTrie::save_dict(const char *filename) {
 }
 #endif  // ___BUILD_MODEL___
 
-bool DictTrie::load_dict(FILE *fp) {
+bool DictTrie::load_dict(QFile *fp) {
   if (NULL == fp)
     return false;
-  if (fread(&lma_node_num_le0_, sizeof(uint32), 1, fp) != 1)
+  if (fp->read((char *)&lma_node_num_le0_, sizeof(uint32)) != sizeof(uint32))
     return false;
 
-  if (fread(&lma_node_num_ge1_, sizeof(uint32), 1, fp) != 1)
+  if (fp->read((char *)&lma_node_num_ge1_, sizeof(uint32)) != sizeof(uint32))
     return false;
 
-  if (fread(&lma_idx_buf_len_, sizeof(uint32), 1, fp) != 1)
+  if (fp->read((char *)&lma_idx_buf_len_, sizeof(uint32)) != sizeof(uint32))
     return false;
 
-  if (fread(&top_lmas_num_, sizeof(uint32), 1, fp) != 1 ||
+  if (fp->read((char *)&top_lmas_num_, sizeof(uint32)) != sizeof(uint32) ||
      top_lmas_num_ >= lma_idx_buf_len_)
      return false;
 
@@ -206,16 +206,16 @@ bool DictTrie::load_dict(FILE *fp) {
     return false;
   }
 
-  if (fread(root_, sizeof(LmaNodeLE0), lma_node_num_le0_, fp)
-      != lma_node_num_le0_)
+  if (fp->read((char *)root_, sizeof(LmaNodeLE0) * lma_node_num_le0_)
+      != sizeof(LmaNodeLE0) * lma_node_num_le0_)
     return false;
 
-  if (fread(nodes_ge1_, sizeof(LmaNodeGE1), lma_node_num_ge1_, fp)
-      != lma_node_num_ge1_)
+  if (fp->read((char *)nodes_ge1_, sizeof(LmaNodeGE1) * lma_node_num_ge1_)
+      != sizeof(LmaNodeGE1) * lma_node_num_ge1_)
     return false;
 
-  if (fread(lma_idx_buf_, sizeof(unsigned char), lma_idx_buf_len_, fp) !=
-      lma_idx_buf_len_)
+  if (fp->read((char *)lma_idx_buf_, sizeof(unsigned char) * lma_idx_buf_len_) !=
+      sizeof(unsigned char) * lma_idx_buf_len_)
     return false;
 
   // The quick index for the first level sons
@@ -245,15 +245,15 @@ bool DictTrie::load_dict(const char *filename, LemmaIdType start_id,
   if (NULL == filename || end_id <= start_id)
     return false;
 
-  FILE *fp = fopen(filename, "rb");
-  if (NULL == fp)
+  QFile file(filename);
+  if (!file.open(QIODevice::ReadOnly))
     return false;
+  QFile *fp = &file;
 
   free_resource(true);
 
   dict_list_ = new DictList();
   if (NULL == dict_list_) {
-    fclose(fp);
     return false;
   }
 
@@ -264,11 +264,9 @@ bool DictTrie::load_dict(const char *filename, LemmaIdType start_id,
       !load_dict(fp) || !ngram.load_ngram(fp) ||
       total_lma_num_ > end_id - start_id + 1) {
     free_resource(true);
-    fclose(fp);
     return false;
   }
 
-  fclose(fp);
   return true;
 }
 
@@ -278,12 +276,12 @@ bool DictTrie::load_dict_fd(int sys_fd, long start_offset,
   if (start_offset < 0 || length <= 0 || end_id <= start_id)
     return false;
 
-  FILE *fp = fdopen(sys_fd, "rb");
-  if (NULL == fp)
+  QFile file;
+  if (!file.open(sys_fd, QIODevice::ReadOnly))
     return false;
+  QFile *fp = &file;
 
-  if (-1 == fseek(fp, start_offset, SEEK_SET)) {
-    fclose(fp);
+  if (!fp->seek(start_offset)) {
     return false;
   }
 
@@ -291,7 +289,6 @@ bool DictTrie::load_dict_fd(int sys_fd, long start_offset,
 
   dict_list_ = new DictList();
   if (NULL == dict_list_) {
-    fclose(fp);
     return false;
   }
 
@@ -300,14 +297,12 @@ bool DictTrie::load_dict_fd(int sys_fd, long start_offset,
 
   if (!spl_trie.load_spl_trie(fp) || !dict_list_->load_list(fp) ||
       !load_dict(fp) || !ngram.load_ngram(fp) ||
-      ftell(fp) < start_offset + length ||
+      fp->pos() < start_offset + length ||
       total_lma_num_ > end_id - start_id + 1) {
     free_resource(true);
-    fclose(fp);
     return false;
   }
 
-  fclose(fp);
   return true;
 }
 
diff --git a/src/virtualkeyboard/3rdparty/pinyin/share/ngram.cpp b/src/virtualkeyboard/3rdparty/pinyin/share/ngram.cpp
index 6aec850b..39155bfb 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/share/ngram.cpp
+++ b/src/virtualkeyboard/3rdparty/pinyin/share/ngram.cpp
@@ -177,13 +177,13 @@ bool NGram::save_ngram(FILE *fp) {
   return true;
 }
 
-bool NGram::load_ngram(FILE *fp) {
+bool NGram::load_ngram(QFile *fp) {
   if (NULL == fp)
     return false;
 
   initialized_ = false;
 
-  if (fread(&idx_num_, sizeof(uint32), 1, fp) != 1 )
+  if (fp->read((char *)&idx_num_, sizeof(uint32)) != sizeof(uint32) )
     return false;
 
   if (NULL != lma_freq_idx_)
@@ -200,11 +200,11 @@ bool NGram::load_ngram(FILE *fp) {
   if (NULL == lma_freq_idx_ || NULL == freq_codes_)
     return false;
 
-  if (fread(freq_codes_, sizeof(LmaScoreType), kCodeBookSize, fp) !=
-      kCodeBookSize)
+  if (fp->read((char *)freq_codes_, sizeof(LmaScoreType) * kCodeBookSize) !=
+      sizeof(LmaScoreType) * kCodeBookSize)
     return false;
 
-  if (fread(lma_freq_idx_, sizeof(CODEBOOK_TYPE), idx_num_, fp) != idx_num_)
+  if (fp->read((char *)lma_freq_idx_, sizeof(CODEBOOK_TYPE) * idx_num_) != sizeof(CODEBOOK_TYPE) * idx_num_)
     return false;
 
   initialized_ = true;
diff --git a/src/virtualkeyboard/3rdparty/pinyin/share/spellingtrie.cpp b/src/virtualkeyboard/3rdparty/pinyin/share/spellingtrie.cpp
index e01c89a5..355c56be 100644
--- a/src/virtualkeyboard/3rdparty/pinyin/share/spellingtrie.cpp
+++ b/src/virtualkeyboard/3rdparty/pinyin/share/spellingtrie.cpp
@@ -663,20 +663,20 @@ bool SpellingTrie::save_spl_trie(FILE *fp) {
   return true;
 }
 
-bool SpellingTrie::load_spl_trie(FILE *fp) {
+bool SpellingTrie::load_spl_trie(QFile *fp) {
   if (NULL == fp)
     return false;
 
-  if (fread(&spelling_size_, sizeof(uint32), 1, fp) != 1)
+  if (fp->read((char *)&spelling_size_, sizeof(uint32)) != sizeof(uint32))
     return false;
 
-  if (fread(&spelling_num_, sizeof(uint32), 1, fp) != 1)
+  if (fp->read((char *)&spelling_num_, sizeof(uint32)) != sizeof(uint32))
     return false;
 
-  if (fread(&score_amplifier_, sizeof(float), 1, fp) != 1)
+  if (fp->read((char *)&score_amplifier_, sizeof(float)) != sizeof(float))
     return false;
 
-  if (fread(&average_score_, sizeof(unsigned char), 1, fp) != 1)
+  if (fp->read((char *)&average_score_, sizeof(unsigned char)) != sizeof(unsigned char))
     return false;
 
   if (NULL != spelling_buf_)
@@ -686,8 +686,7 @@ bool SpellingTrie::load_spl_trie(FILE *fp) {
   if (NULL == spelling_buf_)
     return false;
 
-  if (fread(spelling_buf_, sizeof(char) * spelling_size_,
-            spelling_num_, fp) != spelling_num_)
+  if (fp->read((char *)spelling_buf_, spelling_size_ * spelling_num_) != spelling_size_ * spelling_num_)
     return false;
 
   return construct(spelling_buf_, spelling_size_, spelling_num_,
diff --git a/src/virtualkeyboard/doc/src/build.qdoc b/src/virtualkeyboard/doc/src/build.qdoc
index a858cdea..8753fdc6 100644
--- a/src/virtualkeyboard/doc/src/build.qdoc
+++ b/src/virtualkeyboard/doc/src/build.qdoc
@@ -189,6 +189,12 @@ the preferred input method to the configuration, e.g., \c CONFIG+="lang-zh_TW zh
         system, or if it is built using the local project file. Integration can be
         disabled using this option.
 \row
+    \li \e no-bundle-pinyin
+    \li \e n/a
+    \li Disables bundling of pinyin resources
+    \li This option excludes pinyin resources from the plugin binary. This option may be
+        used to reduce the plugin binary size.
+\row
     \li \e static
     \li Enables static builds of the virtual keyboard
     \li This option enables \l {Static builds}{static builds}.
diff --git a/src/virtualkeyboard/doc/src/deployment-guide.qdoc b/src/virtualkeyboard/doc/src/deployment-guide.qdoc
index 431dcf10..264f1557 100644
--- a/src/virtualkeyboard/doc/src/deployment-guide.qdoc
+++ b/src/virtualkeyboard/doc/src/deployment-guide.qdoc
@@ -168,8 +168,11 @@ There are several environment variables defined by the module that are listed be
         \li QT_VIRTUALKEYBOARD_PINYIN_DICTIONARY
         \li Overrides the location of the Pinyin dictionary.
 
-            The default location depends on the value of
-            \c {QLibraryInfo::location(QLibraryInfo::DataPath)}.
+            By default, the dictionary is bundled into the plugin's resources.
+
+            To disable resource bundling, add \c CONFIG+=no-bundle-pinyin in the
+            plugin's qmake command line. In this scenario, the default location
+            depends on the value of \c {QLibraryInfo::location(QLibraryInfo::DataPath)}.
             For example, for Qt libraries built from source,
             it could be \c {qtbase/qtvirtualkeyboard/pinyin/dict_pinyin.dat}.
     \row
diff --git a/src/virtualkeyboard/pinyindecoderservice.cpp b/src/virtualkeyboard/pinyindecoderservice.cpp
index d531560a..8416d2bc 100644
--- a/src/virtualkeyboard/pinyindecoderservice.cpp
+++ b/src/virtualkeyboard/pinyindecoderservice.cpp
@@ -76,8 +76,11 @@ bool PinyinDecoderService::init()
         return true;
 
     QString sysDict(QString::fromLatin1(qgetenv("QT_VIRTUALKEYBOARD_PINYIN_DICTIONARY").constData()));
-    if (sysDict.isEmpty())
-        sysDict = QLibraryInfo::location(QLibraryInfo::DataPath) + "/qtvirtualkeyboard/pinyin/dict_pinyin.dat";
+    if (!QFileInfo::exists(sysDict)) {
+        sysDict = QLatin1String(":///QtQuick/VirtualKeyboard/3rdparty/pinyin/data/dict_pinyin.dat");
+        if (!QFileInfo::exists(sysDict))
+            sysDict = QLibraryInfo::location(QLibraryInfo::DataPath) + "/qtvirtualkeyboard/pinyin/dict_pinyin.dat";
+    }
 
     QString usrDictPath = QStandardPaths::writableLocation(QStandardPaths::ConfigLocation);
     QFileInfo usrDictInfo(usrDictPath + "/qtvirtualkeyboard/pinyin/usr_dict.dat");
diff --git a/src/virtualkeyboard/virtualkeyboard.pro b/src/virtualkeyboard/virtualkeyboard.pro
index b8d0a8dc..ece00047 100644
--- a/src/virtualkeyboard/virtualkeyboard.pro
+++ b/src/virtualkeyboard/virtualkeyboard.pro
@@ -15,6 +15,7 @@ win32 {
 !no-pkg-config: CONFIG += link_pkgconfig
 
 include(../config.pri)
+include(generateresource.pri)
 
 SOURCES += platforminputcontext.cpp \
     inputcontext.cpp \
@@ -338,10 +339,15 @@ pinyin {
         pinyindecoderservice.h
     DEFINES += HAVE_PINYIN
     QMAKE_USE += pinyin
-    pinyin_data.files = $$PWD/3rdparty/pinyin/data/dict_pinyin.dat
-    pinyin_data.path = $$DATAPATH/pinyin
-    INSTALLS += pinyin_data
-    !prefix_build: COPIES += pinyin_data
+    !no-bundle-pinyin {
+        PINYIN_FILES += 3rdparty/pinyin/data/dict_pinyin.dat
+        RESOURCES += $$generate_resource(pinyin.qrc, $$PINYIN_FILES, /QtQuick/VirtualKeyboard)
+    } else {
+        pinyin_data.files = $$PWD/3rdparty/pinyin/data/dict_pinyin.dat
+        pinyin_data.path = $$DATAPATH/pinyin
+        INSTALLS += pinyin_data
+        !prefix_build: COPIES += pinyin_data
+    }
 }
 
 tcime {
@@ -459,8 +465,6 @@ record-trace-input {
 
 arrow-key-navigation: DEFINES += QT_VIRTUALKEYBOARD_ARROW_KEY_NAVIGATION
 
-include(generateresource.pri)
-
 !disable-layouts {
     RESOURCES += $$generate_resource(layouts.qrc, $$LAYOUT_FILES, /QtQuick/VirtualKeyboard)
     DEFINES += HAVE_LAYOUTS