Add cipd handling to init-repository and take_snapshot

Add parsing deps for packages deployed by cipd.
Keep changes to minimum as we so far need just one
androidx package. All the package files are filtered by the
chromium black list.

Whitelist pydeps and info files for android dependencies,
as this minimizes required changes to gn build files,
add icu file for android.

Update the version to 108.0.5359.181 as it was outdated.

Cipd package parsing is done only for androidx package and
the logic runs for

* init-repository --baseline-upstream
  > cipd downloads and extracts androidx package to
    upstream repository
* take-snapshot
  > cipd returns content so it can got through blacklist lists
    (files are not in git so git ls-file does not help here)

Task-number: QTBUG-83459
Change-Id: I7b8203bfb273ba90113a449725ba9ea5c9a6692c
Reviewed-by: Michael Brüning <michael.bruning@qt.io>

5 files changed, 207 insertions, 35 deletions

diff --git a/tools/scripts/cipd_package.py b/tools/scripts/cipd_package.py
new file mode 100644
index 000000000..a9dfce91e
--- /dev/null
+++ b/tools/scripts/cipd_package.py
@@ -0,0 +1,103 @@
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+import glob
+import os
+import re
+import subprocess
+import sys
+import version_resolver as resolver
+
+androidx_package_name = 'chromium/third_party/androidx'
+
+def subprocessCall(args):
+    print(args)
+    return subprocess.call(args)
+
+def subprocessCheckOutput(args):
+    print(args)
+    return subprocess.check_output(args).decode()
+
+class PackageDEPSParser(resolver.DEPSParser):
+    def __init__(self):
+        super().__init__()
+
+    def createEntitiesFromScope(self, scope):
+        entities = []
+        for dep in scope:
+            if (type(scope[dep]) == dict and 'packages' in scope[dep] \
+            and 'dep_type' in scope[dep] and scope[dep]['dep_type'] == 'cipd'):
+                subdir = self.subdir(dep)
+                if subdir is None:
+                    continue
+                entity = CIPDEntity(subdir, sp=self.topmost_supermodule_path_prefix)
+                path = entity.pathRelativeToTopMostSupermodule()
+                for pkg in scope[dep]['packages']:
+                   p = Package(pkg['package'], pkg['version'], path)
+                   entity.packages.append(p)
+                entities.append(entity)
+        return entities
+
+    def parse(self, deps_content):
+        exec(deps_content, self.global_scope, self.local_scope)
+        entities = []
+        entities.extend(self.createEntitiesFromScope(self.local_scope['deps']))
+        return entities
+
+class Package:
+    def __init__(self, name, version, path):
+        self.name = name
+        self.version = version
+        self.path = path
+        self.fileName = name.replace('/','.') + '.pkg'
+
+    def fetchAndDeploy(self):
+        if os.path.isdir(self.path):
+            currentDir = os.getcwd()
+            os.chdir(self.path)
+            subprocessCall(['cipd', 'pkg-fetch', self.name ,'-out' , self.fileName, '-version', self.version ])
+            subprocessCall(['cipd', 'pkg-deploy', self.fileName , '-root', '.' ])
+            os.chdir(currentDir)
+        else:
+            print('-- missing directory' + self.path + ' skipping')
+
+    def listFiles(self):
+        if os.path.isdir(self.path):
+            currentDir = os.getcwd()
+            os.chdir(self.path)
+            files = []
+            if os.path.isfile(self.fileName):
+                files = subprocessCheckOutput(['cipd', 'pkg-inspect', self.fileName]).splitlines()
+                files = map( lambda x: x.replace( ' F ', self.path + '/'), files)
+            else:
+                print('-- missing package file ' + self.path + ' skipping')
+            os.chdir(currentDir)
+            return files
+        else:
+            print('-- missing directory' + self.path + ' skipping')
+        return []
+
+class CIPDEntity:
+    def __init__(self, path='', packages=[], os=[], sp=''):
+        self.path = path
+        self.packages = []
+        self.topmost_supermodule_path_prefix = sp
+
+    def pathRelativeToTopMostSupermodule(self):
+        return os.path.normpath(os.path.join(self.topmost_supermodule_path_prefix, self.path))
+
+    def findPackage(self, package):
+        pkg = [p for p in self.packages if p.name == package]
+        if len(pkg) > 1:
+            raise Exception(package + " is ambiguous package name for" + self.path)
+        return pkg[0] if pkg else None
+
+    def readEntities(self):
+        cipd_packages = []
+        cipd_packages = resolver.read(PackageDEPSParser)
+        print('DEPS file provides the following packages:')
+        for cipd_package in cipd_packages:
+            print(cipd_package.pathRelativeToTopMostSupermodule() + ':')
+            for package in cipd_package.packages:
+                print(' * {:<80}'.format(package.name) + package.version)
+        return cipd_packages
diff --git a/tools/scripts/git_submodule.py b/tools/scripts/git_submodule.py
index 07b5eb756..b2c21de27 100644
--- a/tools/scripts/git_submodule.py
+++ b/tools/scripts/git_submodule.py
@@ -18,15 +18,9 @@ def subprocessCheckOutput(args):
     print(args)
     return subprocess.check_output(args).decode()
 
-class DEPSParser:
+class SubmoduleDEPSParser(resolver.DEPSParser):
     def __init__(self):
-        self.global_scope = {
-          'Var': lambda var_name: '{%s}' % var_name,
-          'Str': str,
-          'deps_os': {},
-        }
-        self.local_scope = {}
-        self.topmost_supermodule_path_prefix = ''
+        super().__init__()
 
     def get_vars(self):
         """Returns a dictionary of effective variable values
@@ -37,9 +31,6 @@ class DEPSParser:
         #result.update(self.custom_vars or {})
         return result
 
-    def get_recursedeps(self):
-        return self.local_scope["recursedeps"]
-
     def createSubmodulesFromScope(self, scope, os):
         submodules = []
         for dep in scope:
@@ -48,25 +39,18 @@ class DEPSParser:
                 url = scope[dep]
             elif (type(scope[dep]) == dict and 'url' in scope[dep]):
                 url = scope[dep]['url']
-
-                if ('condition' in scope[dep]) and (not 'checkout_linux' in scope[dep]['condition']):
+                if ('condition' in scope[dep]) and \
+                (not 'checkout_linux' in scope[dep]['condition']) and \
+                (not 'checkout_android_native_support' in scope[dep]['condition']):
                     url = ''
-
             if url:
                 url = url.format(**self.get_vars())
                 repo_rev = url.split('@')
                 repo = repo_rev[0]
                 rev = repo_rev[1]
-                subdir = dep
-                if subdir.startswith('src/'):
-                    subdir = subdir[4:]
-                # Don't skip submodules that have a supermodule path prefix set (at the moment these
-                # are 2nd level deep submodules).
-                elif not self.topmost_supermodule_path_prefix:
-                    # Ignore the information about chromium itself since we get that from git,
-                    # also ignore anything outside src/ (e.g. depot_tools)
+                subdir = self.subdir(dep)
+                if subdir is None:
                     continue
-
                 submodule = Submodule(subdir, repo, sp=self.topmost_supermodule_path_prefix)
                 submodule.os = os
 
@@ -294,7 +278,7 @@ class Submodule:
     def readSubmodules(self, use_deps=False):
         submodules = []
         if use_deps:
-            submodules = resolver.readSubmodules()
+            submodules = resolver.read(SubmoduleDEPSParser)
             print('DEPS file provides the following submodules:')
             for submodule in submodules:
                 print('{:<80}'.format(submodule.pathRelativeToTopMostSupermodule()) + '{:<120}'.format(submodule.url) + submodule.ref)
diff --git a/tools/scripts/init-repository.py b/tools/scripts/init-repository.py
index 4dc5ca98a..e6ec0c440 100755
--- a/tools/scripts/init-repository.py
+++ b/tools/scripts/init-repository.py
@@ -12,6 +12,7 @@ import argparse
 qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
 
 import git_submodule as GitSubmodule
+import cipd_package as CIPDPackage
 import version_resolver as resolver
 
 chromium_src = os.environ.get('CHROMIUM_SRC_DIR')
@@ -21,13 +22,14 @@ use_external_chromium = False
 
 parser = argparse.ArgumentParser(description='Initialize QtWebEngine repository.')
 parser.add_argument('--baseline-upstream', action='store_true', help='initialize using upstream Chromium submodule w/o applying patches (for maintenance purposes only)')
+parser.add_argument('--use_cipd', action='store_true', help='fetch and deploy packages with cipd')
 group = parser.add_mutually_exclusive_group()
 group.add_argument('-u', '--upstream', action='store_true', help='initialize using upstream Chromium submodule')
 group.add_argument('-s', '--snapshot', action='store_true', help='initialize using flat Chromium snapshot submodule (default)')
 args = parser.parse_args()
-
 if args.baseline_upstream:
     args.upstream = True
+    args.use_cipd = True
 
 if chromium_src:
     chromium_src = os.path.abspath(chromium_src)
@@ -110,6 +112,20 @@ def initSnapshot():
     snapshot.os = 'all'
     snapshot.initialize()
 
+def initPackages():
+    # 'androidx' it is the only so far cipd package we need
+    third_party_upstream_chromium = os.path.join(qtwebengine_root, 'src/3rdparty_upstream/chromium')
+    currentDir = os.getcwd()
+    os.chdir(third_party_upstream_chromium)
+    cipd = CIPDPackage.CIPDEntity(third_party_upstream_chromium)
+    cipd_entites = cipd.readEntities()
+    for e in cipd_entites:
+        pkg = e.findPackage(CIPDPackage.androidx_package_name)
+        if pkg:
+            print('-- fetching and deploying ' + CIPDPackage.androidx_package_name)
+            pkg.fetchAndDeploy()
+    os.chdir(currentDir)
+
 os.chdir(qtwebengine_root)
 
 if args.upstream:
@@ -119,3 +135,8 @@ if args.upstream:
         subprocess.call(['python3', os.path.join(qtwebengine_root, 'tools', 'scripts', 'patch_upstream.py')])
 if args.snapshot:
     initSnapshot()
+if args.use_cipd:
+    cipdNotFound = subprocess.call(['which', 'cipd'])
+    if cipdNotFound:
+        raise Exception("You need cipd from depo tools.Try setting ./src/3rdparty_upstream/chromium/third_party/depot_tools/cipd in your PATH.")
+    initPackages()
diff --git a/tools/scripts/take_snapshot.py b/tools/scripts/take_snapshot.py
index e2e454060..41cd46b3c 100755
--- a/tools/scripts/take_snapshot.py
+++ b/tools/scripts/take_snapshot.py
@@ -12,6 +12,7 @@ import shutil
 
 from distutils.version import StrictVersion
 import git_submodule as GitSubmodule
+import cipd_package as CIPDPackage
 
 qtwebengine_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
 os.chdir(qtwebengine_root)
@@ -32,6 +33,11 @@ def isInChromiumBlacklist(file_path):
     if file_path.endswith('.gn') or file_path.endswith('.gni') or file_path.endswith('.typemap') or \
     file_path.endswith('.mojom'):
         return False
+    # Add android dependencies info so gn build tree can be parsed
+    if file_path.endswith('.info') or file_path.endswith('.pydeps'):
+        return False
+    if file_path.endswith('.mailmap') or file_path.endswith('.tat.gz.sha1'):
+        return True
     if (file_path.startswith('android_webview')
         or file_path.startswith('apps/')
         or file_path.startswith('ash/')
@@ -121,6 +127,7 @@ def isInChromiumBlacklist(file_path):
             or file_path.startswith('third_party/accessibility')
             or file_path.startswith('third_party/afl')
             or file_path.startswith('third_party/android_')
+            or file_path.startswith('third_party/androidx')
             or file_path.startswith('third_party/angle/third_party/deqp')
             or file_path.startswith('third_party/angle/third_party/glmark2')
             or file_path.startswith('third_party/angle/third_party/VK-GL-CTS')
@@ -201,7 +208,6 @@ def isInChromiumBlacklist(file_path):
             or file_path.startswith('third_party/google_')
             or file_path.startswith('third_party/grpc/')
             or file_path.startswith('third_party/hunspell_dictionaries')
-            or file_path.startswith('third_party/icu/android')
             or file_path.startswith('third_party/icu/cast')
             or file_path.startswith('third_party/icu/chromeos')
             or file_path.startswith('third_party/instrumented_libraries')
@@ -213,6 +219,7 @@ def isInChromiumBlacklist(file_path):
             or file_path.startswith('third_party/libFuzzer')
             or file_path.startswith('third_party/liblouis')
             or file_path.startswith('third_party/libphonenumber')
+            or file_path.startswith('third_party/libunwindstack')
             or file_path.startswith('third_party/logilab')
             or file_path.startswith('third_party/markdown')
             or file_path.startswith('third_party/openh264/src/res')
@@ -292,7 +299,7 @@ def printProgress(current, total):
     sys.stdout.write("\r{} of {}".format(current, total))
     sys.stdout.flush()
 
-def copyFile(src, dst):
+def copyFile(src, dst, use_link = True, force_remove = False):
     src = os.path.abspath(src)
     dst = os.path.abspath(dst)
     dst_dir = os.path.dirname(dst)
@@ -300,11 +307,14 @@ def copyFile(src, dst):
     if not os.path.isdir(dst_dir):
         os.makedirs(dst_dir)
 
-    if os.path.exists(dst):
+    if force_remove or os.path.exists(dst):
         os.remove(dst)
 
     try:
-        os.link(src, dst)
+        if use_link:
+            os.link(src, dst)
+        else:
+            shutil.copy(src,dst)
         # Qt uses LF-only but Chromium isn't.
         subprocess.call(['dos2unix', '--keep-bom', '--quiet', dst])
     except OSError as exception:
@@ -336,6 +346,16 @@ def listFilesInCurrentRepository(use_deps=False):
             files.append(os.path.join(submodule.pathRelativeToTopMostSupermodule(), submodule_file))
     return files
 
+def listPackageFilesInCurrentRepositoryForPackage(packageName):
+    cipd = CIPDPackage.CIPDEntity(os.getcwd())
+    cipd_entities = cipd.readEntities()
+    files = []
+    for e in cipd_entities:
+        pkg = e.findPackage(CIPDPackage.androidx_package_name)
+        if pkg:
+            files.extend(pkg.listFiles())
+    return files
+
 def exportGn():
     third_party_upstream_gn = os.path.join(third_party_upstream, 'gn')
     third_party_gn = os.path.join(third_party, 'gn')
@@ -378,11 +398,15 @@ def exportChromium():
     files.append(b'build/util/LASTCHANGE.committime')
     files.append(b'skia/ext/skia_commit_hash.h')
     files.append(b'gpu/config/gpu_lists_version.h')
+
+    files.extend(listPackageFilesInCurrentRepositoryForPackage(CIPDPackage.androidx_package_name))
+
     for root, directories, local_files in os.walk(third_party_upstream_chromium + '/third_party/node/node_modules'):
         for name in local_files:
             f = os.path.relpath(os.path.join(root, name))
             files.append(f)
 
+    symlinks = []
     print('copying files to ' + third_party_chromium)
     for i in range(len(files)):
         printProgress(i+1, len(files))
@@ -391,7 +415,16 @@ def exportChromium():
         else:
             f = files[i]
         if not isInChromiumBlacklist(f) and not isInGitBlacklist(f):
-            copyFile(f, os.path.join(third_party_chromium, f))
+            d = os.path.join(third_party_chromium, f)
+            copyFile(f,d)
+            # make sure we did not make a hardlink of symlink which is broken afterwards
+            if os.path.islink(f):
+              symlinks.append((f,d))
+    # this is mostly used for files coming from cipd packages
+    for s in symlinks:
+        if not os.path.exists(s[1]):
+            print('fixing ivalid link ' + s[1])
+            copyFile(s[0],s[1], use_link = False, force_remove = True)
 
     # We need to gzip transport_security_state_static.json since it is otherwise too big for our git configuration:
     subprocess.call(['gzip', '-n',  third_party_chromium + '/net/http/transport_security_state_static.json'])
diff --git a/tools/scripts/version_resolver.py b/tools/scripts/version_resolver.py
index c8e37a6de..99b4d2869 100644
--- a/tools/scripts/version_resolver.py
+++ b/tools/scripts/version_resolver.py
@@ -11,8 +11,39 @@ import sys
 import json
 import urllib3
 import git_submodule as GitSubmodule
-
-chromium_version = '108.0.5359.109'
+from abc import ABC, abstractmethod
+
+class DEPSParser(ABC):
+    def __init__(self):
+        self.global_scope = {
+          'Var': lambda var_name: '{%s}' % var_name,
+          'Str': str,
+          'deps_os': {},
+        }
+        self.local_scope = {}
+        self.topmost_supermodule_path_prefix = ''
+
+    def subdir(self, dep):
+        if dep.startswith('src/'):
+            return dep[4:]
+        # Don't skip submodules that have a supermodule path prefix set (at the moment these
+        # are 2nd level deep submodules).
+        elif not self.topmost_supermodule_path_prefix:
+        # Ignore the information about chromium itself since we get that from git,
+        # also ignore anything outside src/ (e.g. depot_tools)
+           return None
+        else:
+           return dep
+
+    @abstractmethod
+    def parse(self):
+        pass
+
+    def get_recursedeps(self):
+        return self.local_scope["recursedeps"]
+
+
+chromium_version = '108.0.5359.220'
 chromium_branch = '5359'
 ninja_version = 'v1.8.2'
 
@@ -59,10 +90,10 @@ def readReleaseChannels():
             channels[os].append({ 'channel': ver['channel'], 'version': ver['version'], 'branch': ver['true_branch'] })
     return channels
 
-def readSubmodules():
+def read(parserCls):
     git_deps = subprocess.check_output(['git', 'show', chromium_version +':DEPS'])
 
-    parser = GitSubmodule.DEPSParser()
+    parser = parserCls()
     git_submodules = parser.parse(git_deps)
 
     submodule_dict = {}
@@ -80,7 +111,7 @@ def readSubmodules():
             with open(extra_deps_file_path, 'r') as extra_deps_file:
                 extra_deps = extra_deps_file.read()
                 if extra_deps:
-                    extradeps_parser = GitSubmodule.DEPSParser()
+                    extradeps_parser = parserCls()
                     extradeps_parser.topmost_supermodule_path_prefix = extradeps_dir
                     extradeps_submodules = extradeps_parser.parse(extra_deps)
                     for sub in extradeps_submodules: