diff options
author | Patrik Teivonen <patrik.teivonen@qt.io> | 2022-03-30 14:36:31 +0300 |
---|---|---|
committer | Patrik Teivonen <patrik.teivonen@qt.io> | 2022-04-01 06:29:20 +0000 |
commit | c817546ffad6f61b748e389837aa072918d1e5a1 (patch) | |
tree | e076e12401ac2e9538326cc3109d62348d2df597 /packaging-tools | |
parent | d459d116f5b3a1d95192d4d0c4758e16daf29598 (diff) |
Remove unused scripts
These scripts are no longer used by Qt release team so remove those.
Change-Id: I517e5404903bedf0310fcafe6a1f300bffad7e0f
Reviewed-by: Iikka Eklund <iikka.eklund@qt.io>
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
Diffstat (limited to 'packaging-tools')
-rw-r--r-- | packaging-tools/batch_process_installer_bld.py | 612 | ||||
-rwxr-xr-x | packaging-tools/extract_examples.sh | 103 | ||||
-rwxr-xr-x | packaging-tools/mksrc.sh | 524 | ||||
-rw-r--r-- | packaging-tools/online_production_updater.py | 731 | ||||
-rwxr-xr-x | packaging-tools/split_qtlocation.py | 160 | ||||
-rw-r--r-- | packaging-tools/swap_repository.py | 196 | ||||
-rw-r--r-- | packaging-tools/update_repository.py | 358 | ||||
-rwxr-xr-x | packaging-tools/winzipdir.sh | 41 |
8 files changed, 0 insertions, 2725 deletions
diff --git a/packaging-tools/batch_process_installer_bld.py b/packaging-tools/batch_process_installer_bld.py deleted file mode 100644 index 43c9948cf..000000000 --- a/packaging-tools/batch_process_installer_bld.py +++ /dev/null @@ -1,612 +0,0 @@ -#!/usr/bin/env python -############################################################################# -## -## Copyright (C) 2019 The Qt Company Ltd. -## Contact: https://www.qt.io/licensing/ -## -## This file is part of the release tools of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:GPL-EXCEPT$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see https://www.qt.io/terms-conditions. For further -## information use the contact form at https://www.qt.io/contact-us. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3 as published by the Free Software -## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT -## included in the packaging of this file. Please review the following -## information to ensure the GNU General Public License requirements will -## be met: https://www.gnu.org/licenses/gpl-3.0.html. -## -## $QT_END_LICENSE$ -## -############################################################################# - -# import the print function which is used in python 3.x -from __future__ import print_function -import sys -import os -import bldinstallercommon -import pkg_constants -import ConfigParser -import urlparse - -SCRIPT_ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) - -# search/replace tags in configuration files -GLOBAL_VERSION = '%GLOBAL_VERSION%' -GLOBAL_VERSION_AND_TAG = '%GLOBAL_VERSION_AND_TAG%' - - -# container for online repository build job parameters -class BuildJob: - def __init__(self, - is_repo_job, - license_type, - node_name, - architecture, - version_number, - version_number_tag, - configurations_dir, - configurations_file, - ifw_tools, - substitution_list, - repo_content_type, - repo_components_to_update, - repo_url_specifier, - installer_name, - rta_keys - ): - self.is_repo_job = is_repo_job - self.license = license_type - self.node_name = node_name - self.architecture = architecture - self.version_number = version_number - self.version_number_tag = version_number_tag - self.configurations_dir = configurations_dir - self.configurations_file = configurations_file - self.ifw_tools = ifw_tools - self.substitution_arg_list = [] - self.repo_content_type = repo_content_type - self.repo_components_to_update = repo_components_to_update - self.repo_url_specifier = repo_url_specifier - self.installer_name = preformat_global_version_and_tag(installer_name, version_number, version_number_tag) - self.rta_key_list = rta_keys.replace(' ', '') - self.format_substitution_list(substitution_list) - - # format substitution list so that it can be used by create_installer.py - def format_substitution_list(self, substitution_list): - item_list = substitution_list.split(',') - for item in item_list: - temp = item.strip() - if temp: - if bldinstallercommon.is_win_platform(): - # On Windows we must escape the '%' so that the subprocess shell will - # not attempt to replace the environment variables - temp = temp.replace('%', '^%') - self.substitution_arg_list.append('--add-substitution=' + temp) - - # print some verbose - def print_data(self): - print('==============================') - print('[{0}]'.format(self.node_name)) - print(self.license) - print(self.architecture) - print(self.version_number) - print(self.version_number_tag) - print(self.configurations_dir) - print(self.configurations_file) - print(self.ifw_tools) - print(self.substitution_arg_list) - if self.is_repo_job: - print(self.repo_content_type) - print(self.repo_components_to_update) - print(self.repo_url_specifier) - print('') - - # validate content - def validate(self): - if not self.node_name: - raise RuntimeError('*** Fatal error! <node_name> not defined for build job?') - if not self.license: - raise RuntimeError('*** Fatal error! <license> not defined for: %s' % self.node_name) - if not self.configurations_dir: - raise IOError('*** Fatal error! <configurations_dir> not defined for: %s' % self.node_name) - if not self.configurations_file: - raise IOError('*** Fatal error! <configurations_file> not defined for: %s' % self.node_name) - if not self.ifw_tools: - raise RuntimeError('*** Fatal error! <ifw_tools> not defined for: %s' % self.node_name) - if self.is_repo_job: - if not self.repo_content_type: - raise RuntimeError('*** Fatal error! <repo_content_type> not defined for: %s' % self.node_name) - if not self.repo_components_to_update: - raise RuntimeError('*** Fatal error! <repo_components_to_update> not defined for: %s' % self.node_name) - if not self.repo_url_specifier: - raise RuntimeError('*** Fatal error! <repo_url_specifier> not defined for: %s' % self.node_name) - # all ok - return True - - def get_package_type(self): - items = self.node_name.split('.') - if len(items) >= 6: - return items[5] - - -def preformat_global_version_and_tag(arg_substitution_list, global_version, global_version_tag): - version = global_version - if global_version_tag: - version = version + '-' + global_version_tag - temp = arg_substitution_list.replace(GLOBAL_VERSION_AND_TAG, version).replace(GLOBAL_VERSION, global_version) - return temp - - -def is_valid_job_type(job_type_specifier): - if (job_type_specifier == 'repository'): - return True - if (job_type_specifier == 'offline'): - return True - if (job_type_specifier == 'online'): - return True - return False - - -# parse online repository build jobs for given platform and architecture -#<branch>.<qt_version>.<offline/repository>.<host_os>.<architecture>.<package_type> -def get_repo_job_list(optionDict, branch, arch, global_version, global_version_tag): - return get_job_list(optionDict, 'repository', branch, arch, global_version, global_version_tag) - - -# parse build jobs for given platform and architecture -#<branch>.<qt_version>.<offline/repository>.<host_os>.<architecture>.<package_type> -def get_job_list(optionDict, job_type_specifier, branch, arch, global_version, global_version_tag): - conf_file = optionDict['RELEASE_DESCRIPTION_FILE'] - license_type = optionDict['LICENSE'] - platform = optionDict['HOST_PLATFORM'] - conf_file_base_dir = optionDict['CONFIGURATIONS_FILE_BASE_DIR'] - ifw_base_url = optionDict.get('IFW_TOOLS_BASE_URL', None) - print('Get [{0}] build job list for: {1}'.format(job_type_specifier, platform + '-' + arch)) - if not os.path.isfile(conf_file): - raise IOError('*** Fatal error! Given file does not exist: %s' % conf_file) - # ensure the string ends with '/' - if ifw_base_url and not ifw_base_url.endswith('/'): - ifw_base_url += '/' - parser = ConfigParser.ConfigParser() - parser.readfp(open(conf_file)) - # validate job type - if not is_valid_job_type(job_type_specifier): - raise RuntimeError('*** Fatal error! Unsupported job type specifier given: %s' % job_type_specifier) - # determine which ifw tools to use (which platform) - ifw_tools = bldinstallercommon.safe_config_key_fetch(parser, 'ifwtools', platform + '-' + arch) - if not ifw_tools: - raise RuntimeError('*** Fatal error! Unable to find ifw tools for_ %s' % (platform + '-' + arch)) - ifw_tools_url = urlparse.urljoin(ifw_base_url, ifw_tools) - # check if repository build job - is_repo_job = False - if job_type_specifier == 'repository': - is_repo_job = True - # first read global arg substitution list applicable for all build jobs in this file - global_arg_substitution_list = bldinstallercommon.safe_config_key_fetch(parser, 'release.global', 'arg_substitution_list') - # parse - job_list = [] - for s in parser.sections(): - section_parts = s.split('.') - if (len(section_parts) < 5): - continue - if (section_parts[0] == branch and section_parts[2] == job_type_specifier): - # Check first if this job was assigned to dedicated machine label - machine_label = bldinstallercommon.safe_config_key_fetch(parser, s, 'assign_to_machine_label') - if machine_label: - if not machine_label == optionDict['TARGET_ENV']: - continue - else: # If not then check against the platform and arch - if not ((section_parts[3] == platform) and (section_parts[4] == arch)): - continue - - # parse from conf file - version_number = bldinstallercommon.safe_config_key_fetch(parser, s, 'version_number') - if not version_number: - version_number = global_version - version_number_tag = bldinstallercommon.safe_config_key_fetch(parser, s, 'version_number_tag') - if not version_number_tag: - version_number_tag = global_version_tag - arg_configurations_file = bldinstallercommon.safe_config_key_fetch(parser, s, 'arg_configurations_file') - if not arg_configurations_file: - raise RuntimeError('*** Fatal error! Configuration file not defined for: %s' % s) - - # for triggering rta later on if specified - rta_key_list = bldinstallercommon.safe_config_key_fetch(parser, s, 'rta_key_list') - # preferred installer name - installer_name = bldinstallercommon.safe_config_key_fetch(parser, s, 'installer_name') - arg_substitution_list = bldinstallercommon.safe_config_key_fetch(parser, s, 'arg_substitution_list') - arg_substitution_list += ',' + global_arg_substitution_list - arg_substitution_list += ',' + "%BUILD_NUMBER%=" + optionDict['BUILD_NUMBER'] - arg_substitution_list = arg_substitution_list.replace('\n', '') - arg_substitution_list = preformat_global_version_and_tag(arg_substitution_list, global_version, global_version_tag) - repo_content_type = '' - repo_components_to_update = '' - repo_url_specifier = '' - # if online repo job - if job_type_specifier == 'repository': - repo_content_type = bldinstallercommon.safe_config_key_fetch(parser, s, 'repo_content_type') - if not repo_content_type: - repo_content_type = arg_configurations_file.split("/")[-1] # if the 'repo_content_type' (for temp dir name) is not defined then parse it from the conf file - repo_components_to_update = bldinstallercommon.safe_config_key_fetch(parser, s, 'repo_components_to_update') - if not repo_components_to_update: - raise RuntimeError('*** Fatal error! <repo_components_to_update> not defined for: %s' % s) - repo_url_specifier = bldinstallercommon.safe_config_key_fetch(parser, s, 'repo_url_specifier') - if not repo_url_specifier: - raise RuntimeError('*** Fatal error! <repo_url_specifier> not defined for: %s' % s) - # determine full path for the conf file - full_conf_file_path = os.path.join(conf_file_base_dir, arg_configurations_file) - # create build job - job = BuildJob(is_repo_job, license_type, s, arch, version_number, version_number_tag, conf_file_base_dir, full_conf_file_path, ifw_tools_url, arg_substitution_list, repo_content_type, repo_components_to_update, repo_url_specifier, installer_name, rta_key_list) - if (job.validate()): - job_list.append(job) - return job_list - - -# execute -# - online installer build(s) -def handle_installer_build(optionDict, installer_type, branch, arch): - conf_file = optionDict['RELEASE_DESCRIPTION_FILE'] - print('Parsing [{0}] installer build jobs from: {0}'.format(conf_file)) - if not os.path.isfile(conf_file): - raise IOError('*** Fatal error! Given file does not exist: {0}'.format(conf_file)) - init_env(optionDict) - license_type = optionDict['LICENSE'] - platform = optionDict['HOST_PLATFORM'] - packages_base_url = optionDict['PACKAGE_STORAGE_SERVER_PATH_HTTP'] - # parse conf file - parser = ConfigParser.ConfigParser() - parser.readfp(open(conf_file)) - section_name = branch + '.' + 'global' - global_version = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version') - global_version_tag = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version_tag') - if not global_version: - raise RuntimeError('*** Fatal error! Invalid values in {0} -> {1}'.format(conf_file, section_name)) - # parse build jobs - job_list = get_job_list(optionDict, installer_type, branch, arch, global_version, global_version_tag) - if (len(job_list) == 0): - raise RuntimeError('*** Fatal error! No [{0}] installer build jobs found from: {1}. Probably an error?'.format(installer_type, conf_file)) - installer_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.INSTALLER_OUTPUT_DIR_NAME) - rta_descr_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME) - bldinstallercommon.create_dirs(rta_descr_output_dir) - # create rta description file - architecture = bldinstallercommon.get_architecture() - plat_suffix = bldinstallercommon.get_platform_suffix() - rta_description_file_name = os.path.join(rta_descr_output_dir, pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE + '-' + plat_suffix + '-' + architecture + '.txt') - # handle build jobs - for job in job_list: - # create installer - type_arg = '--online' if 'online' in installer_type.lower() else '--offline' - creation_ok = create_installer(job, packages_base_url, type_arg) - # write the rta description file only if installer creation was ok - if (creation_ok): - rta_description_file = open(rta_description_file_name, 'a') - rta_description_file.write(job.installer_name + ' ' + job.rta_key_list + '\n') - rta_description_file.close() - # if "/installer_output" directory is empty -> error - if not os.listdir(installer_output_dir): - raise RuntimeError('*** Fatal error! No installers generated into: %s' % installer_output_dir) - - -# helper function/wrapper to create online installer -def create_installer(job, packages_base_url, installer_type): - # ensure the string ends with '/' - if not packages_base_url.endswith('/'): - packages_base_url += '/' - job.print_data() - cmd_args = ['python', '-u','create_installer.py'] - cmd_args = cmd_args + ['-c', job.configurations_dir] - cmd_args = cmd_args + ['-f', job.configurations_file] - cmd_args = cmd_args + [installer_type] - cmd_args = cmd_args + ['-l', job.license] - cmd_args = cmd_args + ['-u', packages_base_url] - cmd_args = cmd_args + ['--ifw-tools=' + job.ifw_tools] - cmd_args = cmd_args + ['--preferred-installer-name=' + job.installer_name] - cmd_args = cmd_args + ['--max-cpu-count=' + str(6 if bldinstallercommon.is_win_platform() else 8)] - if (len(job.substitution_arg_list) > 0): - for item in job.substitution_arg_list: - cmd_args = cmd_args + [item] - # execute, do not bail out if installer job fails - subprocess_exec_stataus = False - try: - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - subprocess_exec_stataus = True - except: - # catch any interrupt into installer creation, we assume the operation failed - subprocess_exec_stataus = False - return subprocess_exec_stataus - -# execute: -# - online reposiory build -# - upload repository into test server -# - update existing repository at test server with new content -def handle_repo_build(optionDict, branch, arch, update_staging_repo, update_production_repo): - conf_file = optionDict['RELEASE_DESCRIPTION_FILE'] - if not os.path.isfile(conf_file): - raise IOError('*** Fatal error! Given file does not exist: %s' % conf_file) - init_env(optionDict) - packages_base_url = optionDict['PACKAGE_STORAGE_SERVER_PATH_HTTP'] - ifw_base_url = optionDict.get('IFW_TOOLS_BASE_URL', None) - release_tools_dir = SCRIPT_ROOT_DIR - # parse conf file - parser = ConfigParser.ConfigParser() - parser.readfp(open(conf_file)) - section_name = branch + '.' + 'global' - global_version = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version') - global_version_tag = bldinstallercommon.safe_config_key_fetch(parser, section_name, 'version_tag') - # parse build jobs - repo_job_list = get_repo_job_list(optionDict, branch, arch, global_version, global_version_tag) - if (len(repo_job_list) == 0): - raise RuntimeError('*** Fatal error! No repository build jobs found. Probably an error? %s' % conf_file) - - if (update_staging_repo or update_production_repo): - # init repo dirs - init_repositories(optionDict, repo_job_list) - # is this snapshot build? Then enable component version number forced update - forced_version_number_bump = False - if update_staging_repo and not update_production_repo: - forced_version_number_bump = True - if optionDict.get('FORCE_VERSION_NUMBER_INCREASE') in ['yes', 'true', '1']: - forced_version_number_bump = True - - # create rta description file - rta_descr_output_dir = os.path.join(SCRIPT_ROOT_DIR, pkg_constants.RTA_DESCRIPTION_FILE_DIR_NAME) - bldinstallercommon.create_dirs(rta_descr_output_dir) - architecture = bldinstallercommon.get_architecture() - plat_suffix = bldinstallercommon.get_platform_suffix() - rta_description_file_name = os.path.join( rta_descr_output_dir, pkg_constants.RTA_DESCRIPTION_FILE_NAME_BASE + '-' + plat_suffix + '-' + architecture + '-repo.txt') - - # handle repo build jobs - for job in repo_job_list: - create_online_repository(job, packages_base_url, forced_version_number_bump) - if (update_staging_repo or update_production_repo): - # determine testination path on test server - dest_path_repository, dest_path_pkg = generate_repo_dest_path_pending(optionDict, job) - # copy repo content to test server - source_path_repository = os.path.join(release_tools_dir, 'online_repository') - source_path_pkg = os.path.join(release_tools_dir, 'pkg') - push_online_repository(optionDict, optionDict['PKG_STAGING_SERVER'], optionDict['PKG_STAGING_SERVER_UNAME'], source_path_repository, dest_path_repository) - push_online_repository(optionDict, optionDict['PKG_STAGING_SERVER'], optionDict['PKG_STAGING_SERVER_UNAME'], source_path_pkg, dest_path_pkg) - # remove local repository and pkg directories - bldinstallercommon.remove_tree(source_path_repository) - bldinstallercommon.remove_tree(source_path_pkg) - # update repo in testing area - staging_repo_updated, production_repo_updated = update_online_repo(optionDict, job, update_staging_repo, update_production_repo) - # write the rta description file only if staging repository creation was ok - # remove also temp staging repositories from 'staging_pending' and 'production_dist_update_work' directories - server_addr = optionDict['PKG_STAGING_SERVER_UNAME'] + '@' + optionDict['PKG_STAGING_SERVER'] - if (staging_repo_updated): - rta_description_file = open(rta_description_file_name, 'a') - rta_description_file.write(job.repo_url_specifier + ' ' + job.rta_key_list + '\n') - rta_description_file.close() - staging_temp_content_to_be_deleted = generate_repo_path_for_pending_area(optionDict, job) - delete_online_repo_paths(optionDict, server_addr, staging_temp_content_to_be_deleted) - if (production_repo_updated): - production_temp_content_to_be_deleted = optionDict['REPO_STAGING_SERVER_TEST_REPO_DIST_WORK'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier - delete_online_repo_paths(optionDict, server_addr, production_temp_content_to_be_deleted) - - -# helper function to create online repository -def create_online_repository(build_job, packages_base_url, forced_version_number_bump): - # ensure the string ends with '/' - if not packages_base_url.endswith('/'): - packages_base_url += '/' - build_job.print_data() - cmd_args = ['python', '-u', 'create_installer.py', \ - '-c', build_job.configurations_dir, \ - '-f', build_job.configurations_file, \ - '--create-repo', \ - '-l', build_job.license, \ - '-u', packages_base_url, \ - '--ifw-tools='+ build_job.ifw_tools] - if forced_version_number_bump: - cmd_args += ['--force-version-number-increase'] - if (len(build_job.substitution_arg_list) > 0): - for item in build_job.substitution_arg_list: - cmd_args = cmd_args + [item] - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - - -# push online repository into remote server -def push_online_repository(optionDict, server_addr, username, directory_to_copy, where_to_copy): - print('Preparing to copy: {0}'.format(directory_to_copy)) - where_to_copy = ensure_unix_paths(where_to_copy) - destination = username + '@' + server_addr + ':' + where_to_copy + '/' - - parts = directory_to_copy.strip(os.path.sep).split(os.path.sep) - parentDir = os.path.sep + os.path.sep.join(parts[:-1]) - dirToCopy = parts[-1] - if bldinstallercommon.is_win_platform(): - parentDir = os.path.sep.join(parts[:-1]) - cmd_args = [optionDict['SCP_COMMAND'], '-r', dirToCopy, destination] - bldinstallercommon.do_execute_sub_process(cmd_args, parentDir) - - -# init online repository directory structure at remote server -def init_repositories(optionDict, job_list): - print('Initializing repositories ...') - init_env(optionDict) - server_addr = optionDict['PKG_STAGING_SERVER_UNAME'] + '@' + optionDict['PKG_STAGING_SERVER'] - for item in job_list: - # create test area paths - test_area_path = generate_repo_path_for_test_area(optionDict, item) - create_remote_dirs(optionDict, server_addr, test_area_path) - # pending (merge) area paths - pending_area_base_path_repo, pending_area_base_path_pkg = generate_repo_dest_path_pending(optionDict, item) - # delete old pending material first from pending area - content_to_be_deleted = generate_repo_path_for_pending_area(optionDict, item) + '/*' - delete_online_repo_paths(optionDict, server_addr, content_to_be_deleted) - # create clean dir structure in pending area - create_remote_dirs(optionDict, server_addr, pending_area_base_path_pkg) - create_remote_dirs(optionDict, server_addr, pending_area_base_path_repo) - - -# generate the common part for the repo path -def generate_repo_path(optionDict, build_job): - path = optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + build_job.repo_url_specifier - return ensure_unix_paths(path) - - -# generate full path for test area repository -def generate_repo_path_for_test_area(optionDict, build_job): - path = optionDict['REPO_STAGING_SERVER_TEST_REPO'] + '/' - path += generate_repo_path(optionDict, build_job) - return ensure_unix_paths(path) - - -# generate pending area (base) path -def generate_repo_path_for_pending_area(optionDict, build_job): - path = optionDict['REPO_STAGING_SERVER_TEST_REPO_PENDING'] + '/' - path += build_job.repo_content_type + '/' - path += generate_repo_path(optionDict, build_job) - return ensure_unix_paths(path) - - -# generate temporary 'pkg' and 'online_repository' pending area paths for repo update work -def generate_repo_dest_path_pending(optionDict, repo_job): - base_path_pending = generate_repo_path_for_pending_area(optionDict, repo_job) - dest_path_repository = ensure_unix_paths(os.path.join(base_path_pending)) - dest_path_pkg = ensure_unix_paths(os.path.join(base_path_pending)) - return dest_path_repository, dest_path_pkg - - -# helper function to create remote directories -def create_remote_dirs(optionDict, server, dir_path): - temp_path = ensure_unix_paths(dir_path) - cmd_args = [optionDict['SSH_COMMAND'], '-t', '-t', server, 'mkdir -p', temp_path] - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - - -# helper function to delete online repository directories on remote server -def delete_online_repo_paths(optionDict, server_addr, path_to_be_deleted): - temp_path = ensure_unix_paths(path_to_be_deleted) - cmd_args = [optionDict['SSH_COMMAND'], '-t', '-t', server_addr, 'rm -rf', temp_path] - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - - -# ensure unix style paths -def ensure_unix_paths(path): - temp_path = path.replace('\\', '/') - temp_path = temp_path.replace('//', '/') - return temp_path - - -# execute online repository update -def update_online_repo(optionDict, job, update_staging_repo, update_production_repo): - staging_server_addr = optionDict['PKG_STAGING_SERVER_UNAME'] + '@' + optionDict['PKG_STAGING_SERVER'] - staging_server_ifw_tools = 'installer-framework-build-linux-x64.7z' - script = optionDict['REPO_STAGING_SERVER_HOME_TOOLS'] + '/' + 'update_repository.py' - if optionDict.get('IFW_TOOLS_BASE_URL', None): - repogen_tools = optionDict['IFW_TOOLS_BASE_URL'] + '/' + staging_server_ifw_tools - else: - repogen_tools = staging_server_ifw_tools - # determine paths on test server - staging_source_repo, staging_source_pkg = generate_repo_dest_path_pending(optionDict, job) - repo_components_to_update = job.repo_components_to_update - staging_repo_updated = False - production_pending_repo_updated = False - - # do we update the staging repository - if update_staging_repo: - # determine target repo - staging_target_repo = optionDict['REPO_STAGING_SERVER_TEST_REPO'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier - cmd_args = [optionDict['SSH_COMMAND'], '-t', '-t', staging_server_addr] - cmd_args = cmd_args + ['python', script] - cmd_args = cmd_args + ['--repogen_tools=' + repogen_tools] - cmd_args = cmd_args + ['--source_pkg=' + staging_source_pkg] - cmd_args = cmd_args + ['--source_repo=' + staging_source_repo] - cmd_args = cmd_args + ['--target_repo=' + staging_target_repo] - cmd_args = cmd_args + ['--components_to_update=' + repo_components_to_update] - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - staging_repo_updated = True - - # do we also update the production repository? - if update_production_repo: - # (1) pull repo from production into staging server 'temp' location - production_repo = optionDict['PROD_USER'] + '@' + optionDict['PROD_ADDR'] - production_repo_path = optionDict['PROD_SRV_REPO_BASE_PATH'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier - production_repo_path = ensure_unix_paths(production_repo_path) - prod_url = production_repo + ":" + production_repo_path - staging_prod_repo_temp_path = optionDict['REPO_STAGING_SERVER_TEST_REPO_DIST_WORK'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier - # delete old existing 'temp' paths - delete_online_repo_paths(optionDict, staging_server_addr, staging_prod_repo_temp_path) - # create 'temp' location where to pull the repo from production - create_remote_dirs(optionDict, staging_server_addr, staging_prod_repo_temp_path) - # chop out the last path component for remote copy as it would result in duplicate nested subdirectory - remote_copy_path = staging_prod_repo_temp_path - if (remote_copy_path.endswith('/')): - remote_copy_path = remote_copy_path[:len(remote_copy_path) - 1] - remote_copy_path = os.path.dirname(remote_copy_path) - remote_copy_path = ensure_unix_paths(remote_copy_path) - # test if production repository exists - cmd_args = [optionDict['SSH_COMMAND'], '-t', '-t', staging_server_addr, 'ssh', '-t', '-t', production_repo, 'test', '-d', production_repo_path] - return_code, dummy = bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR, abort_on_fail=False) - if return_code == -1: - print('Pulling production repository: [{0}] into [{1}]'.format(prod_url, remote_copy_path)) - cmd_args = [optionDict['SSH_COMMAND'], '-t', '-t', staging_server_addr, 'rsync', '-rk', prod_url, remote_copy_path] - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - # (2) update - cmd_args = [optionDict['SSH_COMMAND'], '-t', '-t', staging_server_addr] - cmd_args = cmd_args + ['python', script] - cmd_args = cmd_args + ['--repogen_tools=' + repogen_tools] - cmd_args = cmd_args + ['--source_pkg=' + staging_source_pkg] - cmd_args = cmd_args + ['--source_repo=' + staging_source_repo] - cmd_args = cmd_args + ['--target_repo=' + staging_prod_repo_temp_path] - cmd_args = cmd_args + ['--components_to_update=' + repo_components_to_update] - cmd_args = cmd_args + ['--update_new_components_only'] # for production repos we update only those with version number increase - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - # (3) push updated repo back to production - prod_server_pending_area_dir = optionDict['PROD_SRV_REPO_PENDING_AREA_DIR'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier - prod_server_pending_area_dir = ensure_unix_paths(prod_server_pending_area_dir) - cmd_args_log_to_staging = [optionDict['SSH_COMMAND'], '-t', '-t', staging_server_addr] - cmd_args_log_to_prod = cmd_args_log_to_staging + ['ssh', '-t', '-t', optionDict['PROD_USER'] + '@' + optionDict['PROD_ADDR'] ] - # delete old stuff from pending area, but do sanity check first! - if (os.path.normpath(optionDict['PROD_SRV_REPO_PENDING_AREA_DIR']) in (os.path.normpath(optionDict['PROD_SRV_REPO_BASE_PATH'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME']))): - raise RuntimeError('*** Fatal error!!! You are trying to delete production repository: %s' % prod_server_pending_area_dir) - cmd_args_rm_old = cmd_args_log_to_prod + ['rm', '-rf', prod_server_pending_area_dir] - bldinstallercommon.do_execute_sub_process(cmd_args_rm_old, SCRIPT_ROOT_DIR) - # create pending dirs into production server - cmd_args_mkdirp = cmd_args_log_to_prod + ['mkdir', '-p', prod_server_pending_area_dir] - bldinstallercommon.do_execute_sub_process(cmd_args_mkdirp, SCRIPT_ROOT_DIR) - # chop out the last path component for remote copy as it would result in duplicate nested subdirectory - prod_dest_path = production_repo + ':' + optionDict['PROD_SRV_REPO_PENDING_AREA_DIR'] + '/' + optionDict['ONLINE_REPOSITORY_BASE_NAME'] + '/' + job.repo_url_specifier - if (prod_dest_path.endswith('/')): - prod_dest_path = prod_dest_path[:len(prod_dest_path) - 1] - prod_dest_path = os.path.dirname(prod_dest_path) - # copy updated repo into 'waiting' area on production server - print('Pushing [{0}] into [{1}]'.format(staging_prod_repo_temp_path, prod_dest_path)) - cmd_args = cmd_args_log_to_staging + ['rsync', '-r', staging_prod_repo_temp_path, prod_dest_path] - bldinstallercommon.do_execute_sub_process(cmd_args, SCRIPT_ROOT_DIR) - print('Repository [{0}] updated and pushed into production server pending area successfully.'.format(job.repo_url_specifier)) - production_pending_repo_updated = True - - return staging_repo_updated, production_pending_repo_updated - - -# init environment -def init_env(optionDict): - # Define the base directory under which the rest of directories exist - # Staging specific - optionDict['REPO_STAGING_SERVER_TEST_REPO'] = optionDict['STAGING_SRV_ONLINE_REPO_BASE_PATH'] + '/' + optionDict['LICENSE'] + '/staging' - optionDict['REPO_STAGING_SERVER_TEST_REPO_PENDING'] = optionDict['STAGING_SRV_ONLINE_REPO_BASE_PATH'] + '/' + optionDict['LICENSE'] + '/staging_pending' - optionDict['REPO_STAGING_SERVER_HOME_TOOLS'] = optionDict['REPO_STAGING_SERVER_HOME'] + '/qtsdk/packaging-tools' - # Production specific directories are set directly from environment - - onlineRepositoryBaseName = os.environ.get('ONLINE_REPOSITORY_BASE_NAME') - optionDict['ONLINE_REPOSITORY_BASE_NAME'] = onlineRepositoryBaseName if onlineRepositoryBaseName else 'qtsdkrepository' - - print('Staging server base directory: {0}'.format(optionDict['REPO_STAGING_SERVER_TEST_REPO'])) - print('Staging server base directory pending: {0}'.format(optionDict['REPO_STAGING_SERVER_TEST_REPO_PENDING'])) - print('Staging server tools dir: {0}'.format(optionDict['REPO_STAGING_SERVER_HOME_TOOLS'])) - print('Production server base directory: {0}'.format(optionDict['PROD_SRV_REPO_BASE_PATH'])) - print('Production server base directory pending: {0}'.format(optionDict['PROD_SRV_REPO_PENDING_AREA_DIR'])) - print('Production server base directory dist update work: {0}'.format(optionDict['REPO_STAGING_SERVER_TEST_REPO_DIST_WORK'])) - print('Configurations file base dir: {0}'.format(optionDict['CONFIGURATIONS_FILE_BASE_DIR'])) - bldinstallercommon.init_common_module(SCRIPT_ROOT_DIR) diff --git a/packaging-tools/extract_examples.sh b/packaging-tools/extract_examples.sh deleted file mode 100755 index 690ae1eb9..000000000 --- a/packaging-tools/extract_examples.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/sh -############################################################################# -## -## Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). -## Contact: http://www.qt-project.org/legal -## -## This file is part of the release tools of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and Digia. For licensing terms and -## conditions see http://qt.digia.com/licensing. For further information -## use the contact form at http://qt.digia.com/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 2.1 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 2.1 requirements -## will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -## -## In addition, as a special exception, Digia gives you certain additional -## rights. These rights are described in the Digia Qt LGPL Exception -## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3.0 as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU General Public License version 3.0 requirements will be -## met: http://www.gnu.org/copyleft/gpl.html. -## -## -## $QT_END_LICENSE$ -## -############################################################################# - -APPLICATION_NAME='' -LICENSE='' -APPLICATION_VERSION='' -PACKAGE_STORAGE_SERVER_USER='' -PACKAGE_STORAGE_SERVER='' -PACKAGE_STORAGE_SERVER_BASE_DIR='' -BUILD_NUMBER='' - -if [ $# -lt 14 ]; then - echo "Missing argument" - exit 1 -fi - -# read the arguments -while test $# -gt 0; do - case "$1" in - -n|--appname) - shift - APPLICATION_NAME=$1 - shift - ;; - -l|--license) - shift - LICENSE=$1 - shift - ;; - -v|--version) - shift - APPLICATION_VERSION=$1 - shift - ;; - -u|--user) - shift - PACKAGE_STORAGE_SERVER_USER=$1 - shift - ;; - -s|--server) - shift - PACKAGE_STORAGE_SERVER=$1 - shift - ;; - -d|--directory) - shift - PACKAGE_STORAGE_SERVER_BASE_DIR=$1 - shift - ;; - -b|--build) - shift - BUILD_NUMBER=$1 - shift - ;; - esac -done - -7z x $APPLICATION_NAME-$LICENSE-src-$APPLICATION_VERSION.7z -otemp -cd temp/$APPLICATION_NAME-$LICENSE-src-$APPLICATION_VERSION -rm examples/examples.pro -7z a examples_$APPLICATION_NAME.7z examples/* -tar -cvzf examples_$APPLICATION_NAME.tar.gz examples/* -scp examples_$APPLICATION_NAME.* $PACKAGE_STORAGE_SERVER_USER@$PACKAGE_STORAGE_SERVER:$PACKAGE_STORAGE_SERVER_BASE_DIR/$LICENSE/$APPLICATION_NAME/$APPLICATION_VERSION/latest/examples || exit 1 -cd .. diff --git a/packaging-tools/mksrc.sh b/packaging-tools/mksrc.sh deleted file mode 100755 index 5a14c8799..000000000 --- a/packaging-tools/mksrc.sh +++ /dev/null @@ -1,524 +0,0 @@ -#!/bin/bash -# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). -# Contact: http://www.qt-project.org/legal -# -# You may use this file under the terms of the 3-clause BSD license. -# See the file LICENSE from this package for details. -# -# -# -# Script for archiving qt5 repositories -# -# Usage: -# ./mksrc.sh -u <file url to local git clone> -v <version> -# - Currently supporting only local clones, not direct git:// url's -# After running the script, one will get qt-everywhere-opensource-src-<version>.tar.gz -# and qt-everywhere-opensource-src-<version>.zip -# - -# Gives an unbound variable error message at each attempt -# to use an undeclared parameter. -# -# $ packaging-tools/mksrc.sh -v -# packaging-tools/mksrc.sh: line 161: $1: unbound variable -set -u - -# Exit immediately if a command exits with a non-zero status. -set -e - - -CUR_DIR=$PWD -SCRIPT=$(readlink -f $0) -SCRIPT_DIR=$(dirname $SCRIPT) -DO_FETCH=true -DO_TAG=false -IGNORE_LIST= -LICENSE=opensource -MULTIPACK=no -PACK_TIME=`date '+%Y-%m-%d'` -PACK_FILE=.release-timestamp -PATCH_FILE='' -QTGITTAG=.sha1s -QTSHORTVER=0.0 -QTSYNCQTVER=0.0.0 -QTVER=0.0.0 -REPO_DIR=$CUR_DIR -REPO_NAME='' -REPO_TAG=HEAD -SINGLEMODULE=no -SKIPSYNCQT=no -STRICT=1 -NESTED_SUBMODULE_SKIP_LIST=("qtwebengine/src/3rdparty") -SUBMODULES_WITH_NESTED_SUBMODULES_LIST=("qtwebengine") -PRODUCT_NAME='' - -function usage() -{ - echo "Usage:" - echo "./mksrc.sh -u <file_url_to_git_repo> -v <version> [-m][-N][--tag][-i sub][-l lic][-p patch][-r revision][--single-module][--skip-syncqt][-S]" - echo "where -u is path to git repo and -v is version" - echo "Optional parameters:" - echo "-m one is able to tar each sub module separately" - echo "-N don't use git fetch to update submodules" - echo "--tag also tag the repository" - echo "-i submodule will exclude the submodule from final package " - echo "-l license license type, will default to 'opensource', if set to 'enterprise' all the necessary patches will be applied for enterprise build" - echo "-p patch file patch file (.sh) to execute, example: change_licenses.sh" - echo "-r revision committish to pack (tag name, branch name or SHA-1)" - echo "--single-module tar any single git repository (that might live outside the supermodule)" - echo "--skip-syncqt do not run syncqt by default" - echo "-S don't run in strict mode" - echo "--product-name Additional product name for src package" -} - -function cleanup() -{ - echo "Cleaning all tmp artifacts" - rm -f _txtfiles - rm -f __files_to_zip - rm -f _tmp_mod - rm -f _tmp_shas - rm -rf $PACKAGE_NAME -} - -function is_skippable_nested_submodule() { - module_name=$1 - for i in "${NESTED_SUBMODULE_SKIP_LIST[@]}" - do - if [[ "$i" == "$module_name" ]]; then - echo "1" - return - fi - done - - echo "0" -} - -function has_nested_submodules() { - module_name=$1 - for i in "${SUBMODULES_WITH_NESTED_SUBMODULES_LIST[@]}" - do - if [[ "$i" == "$module_name" ]]; then - echo "1" - return - fi - done - - echo "0" -} - -function create_main_file() -{ - echo " - Creating tarballs -" - # The trick below uses tee to feed the output of tar to - # two additional files, which are actually bash/zsh - # "Process substitution" - # This will cause it to compress simultaneously, though - # at the rate of the slowest of the processes (e.g., - # with xz at 100% CPU, and gzip 15%) -( tar cf - $PACKAGE_NAME/ | \ - tee \ - >(xz -9 > $PACKAGE_NAME.tar.xz) | \ - gzip -9 > $PACKAGE_NAME.tar.gz - - echo " - Created tar.xz and tar.gz - " -) & -( 7z a $PACKAGE_NAME.7z $PACKAGE_NAME/ > /dev/null - echo " - Created 7z - " -) & -( - $SCRIPT_DIR/winzipdir.sh $PACKAGE_NAME.zip $PACKAGE_NAME - echo " - Created single win zip - " -) & -wait -echo " - Done creating archives - " -} - -function create_and_delete_submodule() -{ - mkdir submodules_tar - mkdir submodules_zip - cd $PACKAGE_NAME - while read submodule submodule_sha1; do - # Check if this submodule was marked to be skipped - if [ "$(is_skippable_nested_submodule $submodule)" = "1" ] ; then - continue - fi - _file=$submodule-$LICENSE-src-$QTVER - if [ $PRODUCT_NAME ]; then - _file=$submodule-$PRODUCT_NAME-$LICENSE-src-$QTVER - fi - mv $submodule $_file - echo " - Creating archives - " - ( tar c $_file | tee \ - >(xz -9 > ../submodules_tar/$_file.tar.xz) | \ - gzip -9 > ../submodules_tar/$_file.tar.gz - echo " - Done tarring $_file -" - ) & - ( 7z a ../submodules_zip/$_file.7z $_file/ > /dev/null - echo " - Done 7zipping $_file - " - ) & - ( $SCRIPT_DIR/winzipdir.sh ../submodules_zip/$_file.zip $_file - echo " - Done zipping $_file -" - ) & - wait - rm -rf $_file - done < $MODULES - cd .. -} - -# read the arguments -while test $# -gt 0; do - case "$1" in - -h|--help) - usage - exit 0 - ;; - -m|--modules) - shift - MULTIPACK=yes - ;; - --make-args) - shift - echo "************************************************" - echo " --make-args switch has been depracated. " - echo " Doc creation was removed from mksrc.sh " - echo "************************************************" - shift - ;; - --no-docs) - shift - echo "****************************************" - echo " --no-docs switch has been depracated. " - echo " Doc creation was removed from mksrc.sh " - echo "****************************************" - ;; - -t|--tag) - shift - DO_TAG=true - ;; - -N|--no-fetch) - shift - DO_FETCH=false - ;; - -i|--ignore) - shift - IGNORE_LIST=$IGNORE_LIST" "$1 - shift - ;; - -u|--url) - shift - REPO_DIR=/$1 - shift - ;; - -v|--version) - shift - QTVER=$1 - QTSHORTVER=$(echo $QTVER | cut -d. -f1-2) - QTSYNCQTVER=$(echo $QTVER | cut -d- -f1) - shift - ;; - -l|--license) - shift - LICENSE=$1 - shift - ;; - -p|--patch_file) - shift - PATCH_FILE=$1 - shift - ;; - -r|--revision) - shift - REPO_TAG=$1 - shift - ;; - --exit-after-docs) - shift - echo "************************************************" - echo " --exit-after-docs switch has been depracated. " - echo " Doc creation was removed from mksrc.sh " - echo "************************************************" - ;; - --skip-syncqt) - shift - SKIPSYNCQT=yes - ;; - --single-module) - shift - SINGLEMODULE=yes - ;; - -S|--no-strict) - shift - STRICT=0 - ;; - --product-name) - shift - PRODUCT_NAME=$1 - shift - ;; - *) - echo "Error: Unknown option $1" - usage - exit 0 - ;; - esac -done - -# Check if the DIR is valid git repository -cd $REPO_DIR -if ! git rev-parse --git-dir >/dev/null 2>/dev/null; then - echo "$REPO_DIR is not a valid git repo" - exit 2 -fi -REPO_NAME=$(basename $REPO_DIR) - -if [ $SINGLEMODULE = no ]; then - if [ $PRODUCT_NAME ]; then - PACKAGE_NAME=qt-everywhere-$PRODUCT_NAME-$LICENSE-src-$QTVER - else - PACKAGE_NAME=qt-everywhere-$LICENSE-src-$QTVER - fi -else - if [ $PRODUCT_NAME ]; then - PACKAGE_NAME=$REPO_NAME-$PRODUCT_NAME-$LICENSE-src-$QTVER - else - PACKAGE_NAME=$REPO_NAME-$LICENSE-src-$QTVER - fi -fi -MODULES=$CUR_DIR/submodules.txt -_TMP_DIR=$CUR_DIR/$PACKAGE_NAME - -#------------------------------------------------------------------ -# Step 1, Find all submodules from main repo and archive them -#------------------------------------------------------------------ - -if [ $SINGLEMODULE = no ]; then - echo " -- Finding submodules from $REPO_DIR -- " - - rm -f $MODULES - rm -rf $_TMP_DIR - mkdir $_TMP_DIR - - # detect the submodules to be archived - git ls-tree $REPO_TAG | while read mode type sha1 name; do - test "$type" = "commit" || continue - test -d "$name" || { - echo >&2 "Warning: submodule '$name' is not present" - continue - } - case " $IGNORE_LIST " in - *" $name "*) - # Ignored module, skip - continue - ;; - esac - echo $name $sha1 - # Check if this submodule has nested submodules that need to handled too - if [ "$(has_nested_submodules $name)" = "1" ] ; then - cd $name - git ls-tree -r $sha1 | while read sub_mode sub_type sub_sha1 sub_name; do - test "$sub_type" = "commit" || continue - test -d "$sub_name" || { - echo >&2 "Warning: submodule '$sub_name' is not present" - continue - } - echo $name/$sub_name $sub_sha1 - done - cd $REPO_DIR - fi - done >> $MODULES - - #tag the master repo, maybe - if $DO_TAG && test "v$QTVER" != "$REPO_TAG"; then - git tag -f -a -m "Qt $QTVER Release" v$QTVER $REPO_TAG || \ - { echo >&2 "Unable to tag master repository"; exit 1; } - REPO_TAG=v$QTVER - fi - - cd $REPO_DIR - - #archive the main repo - git archive --format=tar $REPO_TAG | tar -x -C $_TMP_DIR - _SHA=`git rev-parse $REPO_TAG` - MASTER_SHA=$_SHA - rm -f $_TMP_DIR/$QTGITTAG - echo "$REPO_NAME=$_SHA">$_TMP_DIR/$QTGITTAG - - echo " -- From dir $PWD, let's pack the master repo at $MASTER_SHA --" - - #archive all the submodules and generate file from sha1's - while read submodule _SHA; do - echo " -- From dir $PWD/$submodule, lets pack $submodule at $_SHA --" - cd $submodule - _file=$(echo "$submodule" | cut -d'/' -f1).tar.gz - #check that _SHA exists - if ! git cat-file -e $_SHA; then - $DO_FETCH && git fetch >/dev/null - if ! git cat-file -e $_SHA; then - echo >&2 "Commit $_SHA does not exist in submodule $submodule" - echo >&2 "and could not be fetched. Cannot continue." - exit 1 - fi - fi - #tag me, maybe - if $DO_TAG; then - git tag -f -a -m "Qt $QTVER Release" v$QTVER $_SHA || \ - { echo >&2 "Unable to tag submodule $submodule"; exit 1; } - _SHA=v$QTVER - fi - #export the repository contents - git archive --format=tar --prefix=$submodule/ $_SHA | \ - tar -x -C $_TMP_DIR - #store the sha1 - echo "$(echo $(echo $submodule|sed 's/-/_/g') | cut -d/ -f1)=$_SHA" >>$_TMP_DIR/$QTGITTAG - #add QT_PACKAGEDATE_STR for enterprise license key check - if [ $LICENSE = enterprise -a $submodule = qtbase ]; then - rm -f $_TMP_DIR/$submodule/$PACK_FILE - echo "QT_PACKAGEDATE_STR=$PACK_TIME">$_TMP_DIR/$submodule/$PACK_FILE - fi - cd $REPO_DIR - done < $MODULES - #mv $MODULES $CUR_DIR - - cd $CUR_DIR/$PACKAGE_NAME - __skip_sub=no - rm -f _tmp_mod - rm -f _tmp_shas - - # read the shas - echo "$REPO_NAME was archived from $MASTER_SHA" >$CUR_DIR/_tmp_shas - echo "------------------------------------------------------------------------">>$CUR_DIR/_tmp_shas - echo "Fixing shas" - while read submodule submodule_sha1; do - echo $submodule >>$CUR_DIR/_tmp_mod - echo "$submodule was archived from $submodule_sha1" - echo "------------------------------------------------------------------------" - done < $MODULES >>$CUR_DIR/_tmp_shas - cat $CUR_DIR/_tmp_mod > $MODULES - rm -f $CUR_DIR/$PACKAGE_NAME/$QTGITTAG - cat $CUR_DIR/_tmp_shas > $CUR_DIR/$PACKAGE_NAME/$QTGITTAG - - # remove possible empty directories in case of some submodules ignored - for IDIR in $IGNORE_LIST ; do - rm -rf $CUR_DIR/$PACKAGE_NAME/$IDIR - done -else - rm -rf $_TMP_DIR - mkdir $_TMP_DIR - - cd $REPO_DIR - - #archive the single repo - git archive --format=tar $REPO_TAG | tar -x -C $_TMP_DIR - if [ $LICENSE = enterprise ]; then - git submodule update --init - git submodule foreach "git archive HEAD | tar -x -C $_TMP_DIR/\$path" - fi - _SHA=`git rev-parse $REPO_TAG` - SINGLEMODULE_SHA=$_SHA - - echo " -- From dir $PWD, let's pack the $REPO_NAME repo at $SINGLEMODULE_SHA --" -fi # SINGLEMODULE - - -#------------------------------------------------------------------ -# Step 2, run syncqt -#------------------------------------------------------------------ -if [ $SKIPSYNCQT = no ]; then - PACKAGE_DIR=$CUR_DIR/$PACKAGE_NAME - echo "Running syncqt.pl" - if [ $SINGLEMODULE = no ]; then - while read submodule; do - # Check if this submodule was marked to be skipped - if [ "$(is_skippable_nested_submodule $submodule)" = "1" ] ; then - continue - fi - if [ $submodule != qtbase ]; then - RESULT=$(grep "MODULE_VERSION" $PACKAGE_DIR/$submodule/.qmake.conf) - QTSYNCQTVER=$(echo $RESULT | sed 's/.[^=]*=\(.[^ \t]*\)[ \t]*/\1/') - fi - echo " - Running syncqt.pl for $submodule with -version $QTSYNCQTVER" - if [ $submodule = qtwebkit ]; then - SYNC_PROFILE_DIR=$PACKAGE_DIR/$submodule/Source - else - SYNC_PROFILE_DIR=$PACKAGE_DIR/$submodule - fi - $PACKAGE_DIR/qtbase/bin/syncqt.pl -version $QTSYNCQTVER -outdir $PACKAGE_DIR/$submodule $SYNC_PROFILE_DIR - done < $MODULES - else - if [ -f $PACKAGE_DIR/".qmake.conf" ]; then - RESULT=$(grep "MODULE_VERSION" $PACKAGE_DIR/.qmake.conf || true) - QTSYNCQTVER=$(echo $RESULT | sed 's/.[^=]*=\(.[^ \t]*\)[ \t]*/\1/') - else - echo "*** WARNING .qmake.conf not found *** - not running syncqt.pl ***" - fi - if [[ -f $PACKAGE_DIR/"sync.profile" && $QTSYNCQTVER ]]; then - echo " - Running syncqt.pl for $REPO_NAME with -version $QTSYNCQTVER" - $CUR_DIR/../qtbase/bin/syncqt.pl -version $QTSYNCQTVER -outdir $PACKAGE_DIR $PACKAGE_DIR - else - echo "*** WARNING sync.profile or MODULE_VERSION not found - not running syncqt.pl ***" - fi - fi -fi - -#------------------------------------------------------------------ -# Step 3, replace version strings with correct version, and -# patch Qt_PACKAGE_TAG and QT_PACKAGEDATE_STR defines -#------------------------------------------------------------------ -echo " -- Patching %VERSION% etc. defines --" -cd $CUR_DIR/$PACKAGE_NAME/ -find . -type f -print0 | xargs -0 sed -i -e "s/%VERSION%/$QTVER/g" -e "s/%SHORTVERSION%/$QTSHORTVER/g" - -#------------------------------------------------------------------ -# Step 4, check which license type is selected, and run patches -# if needed -#------------------------------------------------------------------ -if [ $PATCH_FILE ]; then - if [ $LICENSE = enterprise]; then - # when doing enterprise build, patch file needs src folder and qt version no as parameters - $PATCH_FILE $CUR_DIR/$PACKAGE_NAME/ $QTVER - else - $PATCH_FILE - fi -fi - -#------------------------------------------------------------------ -# Step 5, create zip file and tar files -#------------------------------------------------------------------ - -cd $CUR_DIR - -echo " -- Create B I G archives -- " -create_main_file - -# Create tar/submodule -if [ $MULTIPACK = yes -a $SINGLEMODULE = no ]; then - mkdir single - mv $PACKAGE_NAME.* single/ - echo " -- Creating archives per submodule -- " - create_and_delete_submodule - echo " -- Creating archive from super repository" - create_main_file - for POSTFIX in "7z" "zip" "tar.gz" "tar.xz"; do - if [ -f $PACKAGE_NAME.$POSTFIX ]; then - if [[ $POSTFIX == *"tar"* ]]; then - if [ $PRODUCT_NAME ]; then - mv $PACKAGE_NAME.$POSTFIX submodules_tar/$REPO_NAME-$PRODUCT_NAME-$LICENSE-src-$QTVER.$POSTFIX - else - mv $PACKAGE_NAME.$POSTFIX submodules_tar/$REPO_NAME-$LICENSE-src-$QTVER.$POSTFIX - fi - else - if [ $PRODUCT_NAME ]; then - mv $PACKAGE_NAME.$POSTFIX submodules_zip/$REPO_NAME-$PRODUCT_NAME-$LICENSE-src-$QTVER.$POSTFIX - else - mv $PACKAGE_NAME.$POSTFIX submodules_zip/$REPO_NAME-$LICENSE-src-$QTVER.$POSTFIX - fi - fi - fi - done -fi -cleanup - -echo "Done!" - diff --git a/packaging-tools/online_production_updater.py b/packaging-tools/online_production_updater.py deleted file mode 100644 index 5c91330b8..000000000 --- a/packaging-tools/online_production_updater.py +++ /dev/null @@ -1,731 +0,0 @@ -#!/usr/bin/env python -############################################################################# -## -## Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). -## Contact: http://www.qt-project.org/legal -## -## This file is part of the release tools of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and Digia. For licensing terms and -## conditions see http://qt.digia.com/licensing. For further information -## use the contact form at http://qt.digia.com/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 2.1 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 2.1 requirements -## will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -## -## In addition, as a special exception, Digia gives you certain additional -## rights. These rights are described in the Digia Qt LGPL Exception -## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3.0 as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU General Public License version 3.0 requirements will be -## met: http://www.gnu.org/copyleft/gpl.html. -## -## -## $QT_END_LICENSE$ -## -############################################################################# - -import os -import sys -import fnmatch -import hashlib -import ConfigParser -import argparse -from time import gmtime, strftime -import bldinstallercommon - -DEFAULT_ONLINE_UPDATE_JOB_FILE_NAME = 'online_update_jobs.conf' -ONLINE_UPDATE_JOB_CONFIG = '' -ONLINE_REPOSITORY_ROOT_NAME = 'qtsdkrepository' -ONLINE_REPOSITORY_META_FILE_NAME = 'Updates.xml' - - -############################### -# Define possible update states -############################### -class UpdateState: - initial = 'initial' - ongoing = 'ongoing' - done = 'done' - -ONLINE_UPDATE_JOB_STATES = [UpdateState.initial, UpdateState.ongoing, UpdateState.done] - - -############################################### -# Container for single update job attributes -############################################### -class UpdateJob: - - def __init__(self, update_job_section, updates_xml, repo_platform, repo_specifier, repo_update_state): - self.update_job_section = update_job_section - self.source_updates_xml = updates_xml - self.source_repo_platform = repo_platform - self.source_repo_specifier = repo_specifier - self.repo_update_state = repo_update_state - self.target_updates_xml = '' - self.temp_id = 0 - self.flush = False - - def __eq__(self, other): - if self.source_updates_xml == other.source_updates_xml: - if is_fresh_update_package(os.path.dirname(self.source_updates_xml)): - return False - else: - return True - return False - - def __hash__(self): - return hash(('source_updates_xml', self.source_updates_xml)) - - def validate(self): - if self.repo_update_state != UpdateState.done: - if not os.path.isfile(self.source_updates_xml): - sys.exit('Given section [{0}] contains invalid Updates.xml path: {1}'.format(self.update_job_section, self.source_updates_xml)) - if self.repo_update_state not in ONLINE_UPDATE_JOB_STATES: - sys.exit('Given section [{0}] contains unsupported repo_update_state: {1}'.format(self.update_job_section, self.repo_update_state)) - if not ONLINE_REPOSITORY_ROOT_NAME in self.source_updates_xml: - sys.exit('*** Given Updates.xml file ignored, does not contain repository name specifier ({0}): {1}'.format(ONLINE_REPOSITORY_ROOT_NAME, self.source_updates_xml)) - - def print_data(self): - print('') - print('[{0}]'.format(self.update_job_section)) - print('source_updates_xml: {0}'.format(self.source_updates_xml)) - print('source_repo_platform: {0}'.format(self.source_repo_platform)) - print('source_repo_specifier: {0}'.format(self.source_repo_specifier)) - print('target_updates_xml: {0}'.format(self.target_updates_xml)) - print('repo_update_state: {0}'.format(self.repo_update_state)) - print('flush status: {0}'.format(self.flush)) - - - -########################################################### -# Read existing update jobs from given configuration file -########################################################### -def read_update_job_info(file_name): - update_jobs = [] - if not os.path.isfile(file_name): - print('*** Warning - Unable to read existing update jobs as given file does not exist: {0}'.format(file_name)) - return update_jobs - print('') - print('') - print('Parsing: {0}'.format(file_name)) - parser = ConfigParser.ConfigParser() - parser.readfp(open(file_name)) - # parse - for section in parser.sections(): - if section.startswith('UpdateJob.'): - updates_xml = bldinstallercommon.safe_config_key_fetch(parser, section, 'updates_xml') - repo_platform = bldinstallercommon.safe_config_key_fetch(parser, section, 'repo_platform') - repo_specifier = bldinstallercommon.safe_config_key_fetch(parser, section, 'repo_specifier') - repo_update_state = bldinstallercommon.safe_config_key_fetch(parser, section, 'repo_update_state') - update_job = UpdateJob(section, updates_xml, repo_platform, repo_specifier, repo_update_state) - update_jobs.append(update_job) - # validate - for item in update_jobs: - item.validate() - return update_jobs - - -########################################### -# Store update jobs into configuration file -########################################### -def store_update_job_info(file_name, update_jobs): - print('Writing to: {0}'.format(file_name)) - cfgfile = open(file_name,'w') - for update_job in update_jobs: - section_name = update_job.update_job_section - config = ConfigParser.ConfigParser() - config.add_section(section_name) - config.set(section_name, 'updates_xml', update_job.source_updates_xml) - config.set(section_name, 'repo_platform', update_job.source_repo_platform) - config.set(section_name, 'repo_specifier', update_job.source_repo_specifier) - config.set(section_name, 'repo_update_state', update_job.repo_update_state) - config.set(section_name, 'target_updates_xml', update_job.target_updates_xml) - config.write(cfgfile) - cfgfile.close() - - -############################### -# Collect Updates.xml files -############################### -def collect_files(directory, file_name): - file_list = [] - for root, dummy, files in os.walk(directory): - for basename in files: - if fnmatch.fnmatch(basename, file_name): - filename = os.path.join(root, basename) - file_list.append(filename) - return file_list - - -############################### -# Function -############################### -def locate_directory(base_dir, dir_name): - for root, dirs, dummy in os.walk(base_dir): - for basename in dirs: - if fnmatch.fnmatch(basename, dir_name): - fulldirname = os.path.join(root, basename) - # return the first match - return fulldirname - return '' - - -############################### -# Function -############################### -def locate_file(directory, file_name): - for root, dummy, files in os.walk(directory): - for basename in files: - if fnmatch.fnmatch(basename, file_name): - filename = os.path.join(root, basename) - # return the first match - return filename - return '' - - -############################### -# Generate new update jobs from -# found Updates.xml files -############################### -def parse_new_update_jobs(update_xml_files): - update_jobs = [] - for update_xml_file in update_xml_files: - path_remainder = update_xml_file.split(ONLINE_REPOSITORY_ROOT_NAME)[1] - path_remainder = path_remainder.split(ONLINE_REPOSITORY_META_FILE_NAME)[0] - repo_platform = path_remainder.split('/')[1] - repo_specifier = path_remainder.split(repo_platform)[1] - # create new update job - section_name_remainder = update_xml_file.replace(os.sep, '').replace('.', '') # unique string - update_job = UpdateJob('UpdateJob.' + section_name_remainder, update_xml_file, repo_platform, repo_specifier, UpdateState.initial) - update_jobs.append(update_job) - # validate - for item in update_jobs: - item.validate() - return update_jobs - - -############################### -# Function -############################### -def is_fresh_update_package(path): - # check if contains only Updates.xml file, nothing else - file_list = os.listdir(path) - updates_xml_found = False - file_count = 0 - for name in file_list: - file_count += 1 - if name == ONLINE_REPOSITORY_META_FILE_NAME: - updates_xml_found = True - if updates_xml_found and file_count > 1: - return True - return False - - -############################### -# Function -############################### -def merge_update_jobs(pending_update_jobs, loaded_update_jobs): - merged_jobs = [] - for pending_job in pending_update_jobs: - for loaded_job in loaded_update_jobs: - if loaded_job.source_updates_xml == pending_job.source_updates_xml: - # filter out those jobs loaded from conf file which are superseded - # by newer jobs in pending area as pending area always has the latest - # update jobs - if is_fresh_update_package(os.path.dirname(pending_job.source_updates_xml)): - loaded_job.flush = True - else: - pending_job.flush = True - combined_list_with_duplicates = pending_update_jobs + loaded_update_jobs - # ignore duplicates - for job in combined_list_with_duplicates: - if not job.flush: - merged_jobs.append(job) - # generate unique id for interactive menus - for i, _ in enumerate(merged_jobs): - merged_jobs[i].temp_id = i - # sanity check for duplicates - fail_if_duplicate_found(merged_jobs) - return merged_jobs - - -############################### -# Function -############################### -def fail_if_duplicate_found(update_job_list): - for item in update_job_list: - count = 0 - for item2 in update_job_list: - if item.source_updates_xml == item2.source_updates_xml: - count += 1 - if count > 1: - sys.exit('*** Internal Error - Duplicate update job found for: {0}'.format(item.source_updates_xml)) - - -############################### -# Function -############################### -def match_update_jobs_with_target_repository(update_jobs, target_repo_root): - temp = [] - for update_job in update_jobs: - # locate matching target platform first (linux_x64, linux_x86, mac_x64, windows_x86) - match = platform_base_dir = locate_directory(target_repo_root, update_job.source_repo_platform) - if not match: - sys.exit('*** Internal Error - Unable to locate matching target platform for: {0}'.format(update_job.source_updates_xml)) - repo_specifiers = filter(None, update_job.source_repo_specifier.split('/')) - for item in repo_specifiers: - match = locate_directory(match, item) - target_updates_xml = '' - # given repsitory (directory) exists i.e. we update existing repository - if match: - target_updates_xml = locate_file(match, ONLINE_REPOSITORY_META_FILE_NAME) - # if the target repo did not exist or was in 'ongoing' state (i.e. no Updates.xml present yet) - # then try to figure out the Updates.xml path - if not target_updates_xml: - target_updates_xml = platform_base_dir + update_job.source_updates_xml.split(update_job.source_repo_platform)[1] - update_job.target_updates_xml = target_updates_xml - temp.append(update_job) - return temp - - -############################### -# Function -############################### -def collect_update_info(update_info_conf_file, update_source_root_dir, target_repo_root): - # (1) find all Updates.xml files under "update_source_root_dir" first i.e. possible updates - update_xml_files = collect_files(update_source_root_dir, ONLINE_REPOSITORY_META_FILE_NAME) - # (2) generate new update jobs from found Updates.xml files - new_update_jobs = parse_new_update_jobs(update_xml_files) - # (3) check from previous "update_info_conf_file" (if exists) if we had - # previous pending updates ongoing - old_update_jobs = [] - if update_info_conf_file: - old_update_jobs = read_update_job_info(update_info_conf_file) - # (4) "merge" new update jobs into ongoing update jobs - merged_update_jobs = merge_update_jobs(new_update_jobs, old_update_jobs) - # (5) match update jobs with target repositories - final_update_jobs = match_update_jobs_with_target_repository(merged_update_jobs, target_repo_root) - return final_update_jobs - - -############################### -# Function -############################### -def count_update_jobs_by_state(update_jobs, repo_update_state): - count = 0 - for item in update_jobs: - if item.repo_update_state == repo_update_state: - count += 1 - return count - - -############################### -# Function -############################### -def list_updates_with_status(update_jobs, status): - print('') - print('------------------------------------------------') - print('Available repository updates with status: {0}'.format(status)) - print('') - for update_job in update_jobs: - if update_job.repo_update_state == status: - print(' * {0}'.format(os.path.dirname(update_job.source_updates_xml))) - print('') - - -############################### -# Function -############################### -def print_selection_details(update_jobs, id_number): - for i, _ in enumerate(update_jobs): - if update_jobs[i].temp_id == id_number: - print('') - print('--------------------------------') - print('Platform: {0}'.format(update_jobs[i].source_repo_platform)) - print('Repo specifier: {0}'.format(update_jobs[i].source_repo_specifier)) - print('Target: {0}'.format(os.path.dirname(update_jobs[i].target_updates_xml))) - print('State: {0}'.format(update_jobs[i].repo_update_state)) - print('') - - -############################### -# Function -############################### -def do_execute_updates(update_jobs, update_state, update_function): - while (True): - count = count_update_jobs_by_state(update_jobs, update_state) - if not count: - return - print('') - print('Available update jobs with state [{0}]: {1}'.format(update_state, count)) - print('') - for i, _ in enumerate(update_jobs): - if update_jobs[i].repo_update_state == update_state: - print('================================') - print('{0}:'.format(update_jobs[i].temp_id)) - print('Source: {0}'.format(os.path.dirname(update_jobs[i].source_updates_xml))) - print('Target: {0}'.format(os.path.dirname(update_jobs[i].target_updates_xml))) - print('') - - answer = ask_number_input('Enter number which item to update:') - print('') - if (answer < 0): - return - print_selection_details(update_jobs, answer) - - questions = [] - questions.append([1,'Confirm update the selected item?']) - questions.append([5,'Cancel']) - result = ask_enumerated_user_input(questions) - print('') - if (result == 1): - update_function(answer, update_jobs) - if (result == 5): - return - - -############################### -# Function -############################### -def flush_finished_updates(update_jobs): - while (True): - print('') - print('----------------------------') - print('Old finished update jobs:') - print('') - for i, _ in enumerate(update_jobs): - if update_jobs[i].repo_update_state == UpdateState.done: - print('================================') - print('{0}:'.format(update_jobs[i].temp_id)) - print('Source: {0}'.format(os.path.dirname(update_jobs[i].source_updates_xml))) - print('Target: {0}'.format(os.path.dirname(update_jobs[i].target_updates_xml))) - print('') - print('Remove the listed update jobs from used configuration file?') - questions = [] - questions.append([1,'Yes']) - questions.append([9,'No']) - result = ask_enumerated_user_input(questions) - print('') - if (result == 1): - to_remove = [i for i, job in enumerate(update_jobs) if job.repo_update_state == UpdateState.done] - for index in reversed(to_remove): # start at the end to avoid recomputing offsets - del update_jobs[index] - # Save changed update job status to config file - store_update_job_info(ONLINE_UPDATE_JOB_CONFIG, update_jobs) - return - if (result == 9): - return - - -############################### -# Move repository data files -############################### -def do_update_repository_data_files(id_num, update_jobs, update_all=False): - for i, _ in enumerate(update_jobs): - if update_jobs[i].repo_update_state != UpdateState.initial: - continue - if update_all or update_jobs[i].temp_id == id_num: - print('Updating data files') - # initial update i.e. only data files should be updated i.e. copied - # (1) sanity check - source_repo_path = os.path.dirname(update_jobs[i].source_updates_xml).split(ONLINE_REPOSITORY_ROOT_NAME)[1] - target_repo_path = os.path.dirname(update_jobs[i].target_updates_xml).split(ONLINE_REPOSITORY_ROOT_NAME)[1] - if not source_repo_path == target_repo_path: - print('*** Fatal error! Source and target paths do not seem to match correctly for the update?') - print('Source: {0}'.format(source_repo_path)) - print('Target: {0}'.format(target_repo_path)) - sys.exit(-1) - # (2) does destination directory exist? - target_dir = os.path.dirname(update_jobs[i].target_updates_xml) - if not os.path.exists(target_dir): - print('Destination directory does not exist, creating: {0}'.format(target_dir)) - os.makedirs(target_dir) - # (3) copy data files (only) - source_dir = os.path.dirname(update_jobs[i].source_updates_xml) - safe_move_data_files_recursive(source_dir, target_dir) - # (4) update state - update_jobs[i].repo_update_state = UpdateState.ongoing - # (5) Save changed update job status to config file - store_update_job_info(ONLINE_UPDATE_JOB_CONFIG, update_jobs) - print('Updating data files done') - - -############################### -# Move repository meta data files -############################### -def do_update_repository_meta_files(id_num, update_jobs, update_all=False): - for i, _ in enumerate(update_jobs): - if update_jobs[i].repo_update_state != UpdateState.ongoing: - continue - if update_all or update_jobs[i].temp_id == id_num: - print('Updating meta files') - # initial update i.e. only data files should be updated i.e. copied - source_updates_xml = update_jobs[i].source_updates_xml - target_updates_xml = update_jobs[i].target_updates_xml - # (1) sanity check - if not os.path.isfile(source_updates_xml): - sys.exit('*** Fatal error! Source Updates.xml file not existing: {0}'.format(source_updates_xml)) - target_dir = os.path.dirname(update_jobs[i].target_updates_xml) - if not os.path.exists(target_dir): - sys.exit('*** Fatal error! Destination directory does not exist: {0}'.format(target_dir)) - # (2) Backup old Updates.xml file if exists - is_existing_repository = os.path.isfile(target_updates_xml) - if is_existing_repository: - backup_filename = target_updates_xml + '_backup_' + strftime('%Y%m%d%H%M%S', gmtime()) - os.rename(target_updates_xml, backup_filename) - # (3) Move new Updates.xml in place - os.rename(source_updates_xml, target_updates_xml) - if is_existing_repository: - # (4) Save backup - os.rename(backup_filename, target_updates_xml + '_backup_official_' + strftime('%Y%m%d%H%M%S', gmtime())) - # (5) Remove empty source directory - delete_directory_if_empty(os.path.dirname(update_jobs[i].source_updates_xml)) - # (6) update state - update_jobs[i].repo_update_state = UpdateState.done - # (7) Save changed update job status to config file - store_update_job_info(ONLINE_UPDATE_JOB_CONFIG, update_jobs) - print('Updating meta files done') - - -############################### -# Function -############################### -def safe_move_data_files_recursive(srcdir, dstdir): - srcnames = os.listdir(srcdir) - for name in srcnames: - srcfname = os.path.join(srcdir, name) - dstfname = os.path.join(dstdir, name) - if os.path.isdir(srcfname) and not os.path.islink(srcfname): - if not os.path.exists(dstfname): - os.mkdir(dstfname) - safe_move_data_files_recursive(srcfname, dstfname) - delete_directory_if_empty(srcfname) - elif name != ONLINE_REPOSITORY_META_FILE_NAME: - if os.path.basename(srcfname) == os.path.basename(dstfname): - if os.path.isfile(dstfname): - # we may be copying over the same files. E.g. "/tools_mingw" where we may update - # only one component but the rest are the same. As in practice we swap repo - # directories so we may end up copying over the existing files. Then we must ensure that - # the files we override are the same i.e. hash checksums match - mdsum_source = hashlib.sha256(open(srcfname, 'rb').read()).digest() - mdsum_dest = hashlib.sha256(open(dstfname, 'rb').read()).digest() - if mdsum_source != mdsum_dest: - print('*** Fatal! Source: {0} Destination {1}'.format(srcdir, dstdir)) - print('*** Preventing accidental data file override. Can not override a file with same filename (and version) but different content. Hash check sums do not match. Aborting...') - sys.exit(-1) - os.rename(srcfname, dstfname) - - -############################### -# Function -############################### -def delete_directory_if_empty(dir_name): - try: - os.rmdir(dir_name) - except OSError as ex: - if ex.errno != errno.ENOTEMPTY: - print('*** Error while trying to delete directory, not empty: {0}, {1}'.format(ex.filename, ex.strerror)) - - -############################### -# Function -############################### -def chunk_reader(fobj, chunk_size=1024): - """Generator that reads a file in chunks of bytes""" - while True: - chunk = fobj.read(chunk_size) - if not chunk: - return - yield chunk - - -############################################################################################# -# Interactive menu stuff -############################################################################################# -def show_main_menu(update_jobs): - print('--------------------------------------------------------') - print('Online Repository update tool. Available updates: {0}'.format(len(update_jobs))) - while (True): - print('') - if len(update_jobs) <= 0: - print('Nothing to do...') - return - - initial_update_job_count = count_update_jobs_by_state(update_jobs, UpdateState.initial) - ongoing_update_job_count = count_update_jobs_by_state(update_jobs, UpdateState.ongoing) - finished_update_job_count = count_update_jobs_by_state(update_jobs, UpdateState.done) - print('New updates found: {0}'.format(initial_update_job_count)) - print('Ongoing updates found: {0}'.format(ongoing_update_job_count)) - print('Finished updates found: {0}'.format(finished_update_job_count)) - - questions = [] - if initial_update_job_count: - questions.append([1,'List NEW updates']) - questions.append([2,'Perform NEW updates']) - if ongoing_update_job_count: - questions.append([3,'List ONGOING updates']) - questions.append([4,'Perform ONGOING updates']) - if finished_update_job_count: - questions.append([5,'List FINISHED updates']) - questions.append([6,'Clear FINISHED updates']) - questions.append([9,'Exit']) - - result = ask_enumerated_user_input(questions) - print('') - # exit - if (result == 9): - return - # list - if (result == 1): - list_updates_with_status(update_jobs, UpdateState.initial) - if (result == 3): - list_updates_with_status(update_jobs, UpdateState.ongoing) - if (result == 5): - list_updates_with_status(update_jobs, UpdateState.done) - # perform updates - if (result == 2): - do_execute_updates(update_jobs, UpdateState.initial, do_update_repository_data_files) - if (result == 4): - do_execute_updates(update_jobs, UpdateState.ongoing, do_update_repository_meta_files) - # clear old finished updated from config file - if (result == 6): - flush_finished_updates(update_jobs) - - -############################### -# Function -############################### -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - - -############################### -# Aks user input -############################### -def ask_enumerated_user_input(question_list): - while (True): - print('') - allowed_answers = [] - for item in question_list: - print('{0}: {1}'.format(item[0], item[1])) - allowed_answers.append(item[0]) - var = raw_input("Selection: ") - if is_number(var) and int(var) in allowed_answers: - return int(var) - - -############################### -# Aks user input -############################### -def ask_number_input(question): - print('') - while(True): - print(question) - print('Cancel: c') - var = raw_input("Selection: ") - if var in ['c', 'C']: - return -1 - if is_number(var) and int(var) >= 0: - return int(var) - - - -############################### -# Function -############################### -def generate_test_update_job_info(): - update_jobs = [] - config = ConfigParser.ConfigParser() - section_name = 'UpdateJob.' + 'test_1' - config.add_section(section_name) - config.set(section_name, 'updates_xml', '/base_dir/foo/bar/qtsdkrepository/linux_x64/desktop/qt5/Updates.xml') - config.set(section_name, 'repo_platform', 'linux_x64') - config.set(section_name, 'repo_specifier', '/desktop/qt5') - config.set(section_name, 'repo_update_state', UpdateState.initial) - update_jobs.append(UpdateJob(config, section_name)) - - section_name = 'UpdateJob.' + 'test_2' - config.add_section(section_name) - config.set(section_name, 'updates_xml', '/base_dir/foo/bar/qtsdkrepository/linux_x64/desktop/tools_maintenance/Updates.xml') - config.set(section_name, 'repo_platform', 'linux_x64') - config.set(section_name, 'repo_specifier', '/desktop/tools_maintenance') - config.set(section_name, 'repo_update_state', UpdateState.initial) - update_jobs.append(UpdateJob(config, section_name)) - - section_name = 'UpdateJob.' + 'test_3' - config.add_section(section_name) - config.set(section_name, 'updates_xml', '/base_dir/foo/bar/qtsdkrepository/linux_x64/android/qt5/Updates.xml') - config.set(section_name, 'repo_platform', 'linux_x64') - config.set(section_name, 'repo_specifier', '/android/qt5') - config.set(section_name, 'repo_update_state', UpdateState.initial) - update_jobs.append(UpdateJob(config, section_name)) - - return update_jobs - - -############################### -# Set and get cmd line parser -############################### -def get_cmd_line_parser(): - parser = argparse.ArgumentParser(prog = os.path.basename(sys.argv[0]), - add_help=True, description="Update Qt Installer-Framework based online repositories", formatter_class=argparse.RawTextHelpFormatter) - parser.epilog = "This script can be used to update online repositories generated with Qt Installer-Framework."\ - "The actual purpose for this tool is to ease out online updates in mirrored production servers where you have no control" \ - "in which order files get mirrored. For Qt Installer-Framework we must ensure the data files (*.7z) are in sync first before" \ - "meta files (Updates.xml) can be put in place as the meta files refer to the data files." \ - "The script allows the user to perform updates in phases. In first phase the user can update only the data files." \ - "In the second phase the user can update the meta files for those components that had the data files updated previously." \ - "The script saves component status information into configuration file so it can continue from previous time." \ - "".format(os.path.basename(sys.argv[0])) - - parser.add_argument('--pending_repo_root', help="Source root dir for new online repository components", required=True) - parser.add_argument('--target_repo_root', help="Destination root dir where to perform the update", required=True) - parser.add_argument('--update_info_conf', help="Config file defining existing/ongoing update jobs", required=False, default=DEFAULT_ONLINE_UPDATE_JOB_FILE_NAME) - parser.add_argument('--force_initial_updates', help="Perform all available initial update jobs automatically", action='store_true', default=False) - parser.add_argument('--force_ongoing_updates', help="Perform all available ongoing update jobs automatically", action='store_true', default=False) - return parser.parse_args() - - -############################### -# Main -############################### -if __name__ == "__main__": - cmd_line_parser = get_cmd_line_parser() - update_jobs = collect_update_info(cmd_line_parser.update_info_conf, cmd_line_parser.pending_repo_root, cmd_line_parser.target_repo_root) - ONLINE_UPDATE_JOB_CONFIG = cmd_line_parser.update_info_conf - show_menu = True - # show main menu and start interactive mode - if cmd_line_parser.force_initial_updates: - do_update_repository_data_files(None, update_jobs, True) - show_menu = False - if cmd_line_parser.force_ongoing_updates: - do_update_repository_meta_files(None, update_jobs, True) - show_menu = False - if show_menu: - show_main_menu(update_jobs) - # store changed update job states to config file - store_update_job_info(cmd_line_parser.update_info_conf, update_jobs) - - - - diff --git a/packaging-tools/split_qtlocation.py b/packaging-tools/split_qtlocation.py deleted file mode 100755 index 68ce2494c..000000000 --- a/packaging-tools/split_qtlocation.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env - -############################################################################ -## -## Copyright (C) 2015 The Qt Company Ltd. -## Contact: http://www.qt.io/licensing/ -## -## -## $QT_BEGIN_LICENSE:LGPL21$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and The Qt Company. For licensing terms -## and conditions see http://www.qt.io/terms-conditions. For further -## information use the contact form at http://www.qt.io/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 2.1 or version 3 as published by the Free -## Software Foundation and appearing in the file LICENSE.LGPLv21 and -## LICENSE.LGPLv3 included in the packaging of this file. Please review the -## following information to ensure the GNU Lesser General Public License -## requirements will be met: https://www.gnu.org/licenses/lgpl.html and -## http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -## -## As a special exception, The Qt Company gives you certain additional -## rights. These rights are described in The Qt Company LGPL Exception -## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -## -## $QT_END_LICENSE$ -## -############################################################################ - -from __future__ import print_function -import os -import re -import sys -import shutil -import urllib -import bldinstallercommon - - -locationDir = "qt5_qtlocation" -positioningDir = "qt5_qtpositioning" -geoservicesDir = "geoservices" -geoLocationHeaders = ['QGeoLocation', - 'qgeolocation.h', - 'qdeclarativegeolocation_p.h', - 'qlocationutils_p.h', - 'qgeolocation_p.h'] - - -############################### -# function -############################### -def do_content_comparison(pkgLocation, fileCount): - pkgFileList = [] - for root, dirs, files in os.walk(pkgLocation): - for fileName in files: - pkgFileList.append(fileName) - # exit if files added to qt5_qtlocation.7z - if fileCount < len(pkgFileList): - diff = len(pkgFileList) - fileCount - # sys.exit('Exiting, file count difference found: %s ' % diff) - -############################### -# function -############################### -def do_cleanup(regExp, pkgLocation): - print("Cleanup: ", pkgLocation) - os.chdir(pkgLocation) - for root, dirs, files in os.walk(pkgLocation): - for dirName in dirs: - if regExp.findall(dirName): - print("Removing dir: ", os.path.join(root, dirName)) - shutil.rmtree(os.path.join(root, dirName)) - if geoservicesDir in dirName and positioningDir in pkgLocation: - shutil.rmtree(os.path.join(root, dirName)) - for fileName in files: - if regExp.findall(fileName): - if os.path.islink(os.path.join(root, fileName)): - print("Unlink: ", os.path.join(root, fileName)) - os.unlink(os.path.join(root, fileName)) - if os.path.isfile(os.path.join(root, fileName)): - if fileName in geoLocationHeaders and positioningDir in pkgLocation: - print("Geolocation header, not removing: ", os.path.join(root, fileName)) - else: - print("Removing file: ", os.path.join(root, fileName)) - os.remove(os.path.join(root, fileName)) - -############################### -# function -############################### -def getFileCount(package): - filecount = 0 - if "android" in package: - filecount = 225 - elif "linux_gcc_64_rhel66" in package: - filecount = 227 - elif "linux_gcc_32_rhel66" in package: - filecount = 225 - elif "ios" in package: - filecount = 301 - elif "windows_mingw492_x86" in package: - filecount = 231 - elif "windows_vs2013_winrt_x64" in package: - filecount = 241 - elif "windows_vs" in package: - filecount = 237 - elif "winphone" in package: - filecount = 241 - else: # mac_x64 - filecount = 243 - return filecount - -############################### -# function -############################### -def do_split(location_archive, archives_dir, tmp_archive_dir): - - archivePath = location_archive - tmpDir = tmp_archive_dir - - # create temp directories for location & positioning - if not os.path.exists(os.path.join(tmpDir, locationDir)): - os.makedirs(os.path.join(tmpDir, locationDir)) - if not os.path.exists(os.path.join(tmpDir, positioningDir)): - os.makedirs(os.path.join(tmpDir, positioningDir)) - - locationDest = os.path.join(tmpDir, locationDir) - positioningDest = os.path.join(tmpDir, positioningDir) - bldinstallercommon.extract_file(archivePath, locationDest) - bldinstallercommon.extract_file(archivePath, positioningDest) - - # check for new files in the qtlocation archive - do_content_comparison(locationDest, getFileCount(archivePath)) - - # remove *qtpositioning* from qt5_qtlocation.7z - locRegex = re.compile('\.*osition\.*') - do_cleanup(locRegex, locationDest) - # remove *qtlocation* from qt5_qtpositionign.7z - posRegex = re.compile('\.*ocation\.*') - do_cleanup(posRegex, positioningDest) - - # remove original qtlocation - os.remove(archivePath) - - # archive qt5_qtlocation.7z - os.chdir(locationDest) - cmd_args_archive = ['7z', 'a', locationDir, '*'] - bldinstallercommon.do_execute_sub_process(cmd_args_archive, locationDest) - shutil.copy2(os.path.join(locationDest, locationDir) + '.7z', archives_dir) - # archive qt5_qtpositioning.7z - os.chdir(positioningDest) - cmd_args_archive = ['7z', 'a', positioningDir, '*'] - bldinstallercommon.do_execute_sub_process(cmd_args_archive, positioningDest) - shutil.copy2(os.path.join(positioningDest, positioningDir) + '.7z', archives_dir) - - diff --git a/packaging-tools/swap_repository.py b/packaging-tools/swap_repository.py deleted file mode 100644 index e190d2302..000000000 --- a/packaging-tools/swap_repository.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python -############################################################################# -## -## Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). -## Contact: http://www.qt-project.org/legal -## -## This file is part of the release tools of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and Digia. For licensing terms and -## conditions see http://qt.digia.com/licensing. For further information -## use the contact form at http://qt.digia.com/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 2.1 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 2.1 requirements -## will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -## -## In addition, as a special exception, Digia gives you certain additional -## rights. These rights are described in the Digia Qt LGPL Exception -## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3.0 as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU General Public License version 3.0 requirements will be -## met: http://www.gnu.org/copyleft/gpl.html. -## -## -## $QT_END_LICENSE$ -## -############################################################################# - -# import the print function which is used in python 3.x -from __future__ import print_function -import sys -import os -import argparse -import difflib -import binascii -import shutil -import bldinstallercommon - -QT_SDK_REPOSITORY_NAME = 'qtsdkrepository' - -############################### -# Helper class -############################### -class SwapOperation: - def __init__(self): - self.componen_name = '' - self.source_dir = '' - self.destination_dir = '' - - -############################### -# Helper class -############################### -def generate_random_string(): - return binascii.b2a_hex(os.urandom(15)) - - -############################### -# function -############################### -def parse_cmd_line_args(): - parser = argparse.ArgumentParser(prog = os.path.basename(sys.argv[0]), - add_help=True, description="Swap online repositories", formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--component', help="What directory (repository) to swap", required=True) - parser.add_argument('--source', help="Source base directory", required=True) - parser.add_argument('--destination', help="Destination base directory", required=True) - return parser.parse_args() - - -############################### -# function -############################### -def get_directory_list(base_dir, search_match): - matches = [] - for root, dirs, dummy in os.walk(base_dir): - for basename in dirs: - if basename == search_match: - fulldirname = os.path.join(root, basename) - matches.append(fulldirname) - return matches - - -############################### -# function -############################### -def determine_dest_dir(item, destination_base_dir): - start_index = item.index(QT_SDK_REPOSITORY_NAME) - remainder = item[start_index:] - dest_start_index = destination_base_dir.index(QT_SDK_REPOSITORY_NAME) - dest_path_start = destination_base_dir[:dest_start_index] - return dest_path_start + remainder - - -############################### -# function -############################### -def is_platform_match(source_item, dest_item): - temp_index = source_item.index(QT_SDK_REPOSITORY_NAME) - temp_index += len(QT_SDK_REPOSITORY_NAME) + 1 - chopped_source_item = source_item[temp_index:] - delimeter_index = chopped_source_item.index(os.path.sep) - platform = chopped_source_item[:delimeter_index] - if platform in dest_item: - return True - return False - - -############################### -# function -############################### -def generate_match_list(source_match_list, dest_match_list, component_name, destination_base_dir): - match_list = [] - for item in source_match_list: - matches = difflib.get_close_matches(item, dest_match_list) - dest_dir = '' - if not matches or \ - not os.path.isdir(matches[0]) or \ - not matches[0].endswith(component_name) or \ - not is_platform_match(item, matches[0]): - print('*** No match found? Is this the first time the repo is being copied?') - dest_dir = determine_dest_dir(item, destination_base_dir) - else: - dest_dir = matches[0] - swap_operation = SwapOperation() - swap_operation.componen_name = component_name - swap_operation.source_dir = item - swap_operation.destination_dir = dest_dir - match_list.append(swap_operation) - return match_list - - -############################### -# function -############################### -def swap_repository(parser_args): - if not parser_args: - raise RuntimeError('*** No options available to swap online reposities') - - source_match_list = get_directory_list(parser_args.source, parser_args.component) - if not source_match_list: - raise RuntimeError('*** Nothing to update? Did not find any component named: %s' % parser_args.component) - dest_match_list = get_directory_list(parser_args.destination, parser_args.component) - - match_list = generate_match_list(source_match_list, dest_match_list, parser_args.component, parser_args.destination) - for swap_option in match_list: - print() - print('###################################################') - print('Replacing: {0}'.format(swap_option.destination_dir)) - print('With: {0}'.format(swap_option.source_dir)) - print() - print('y: Yes') - print('n: No (abort)') - print() - keep_asking = True - while (keep_asking): - var = raw_input("Proceed? ") - if var in ['n', 'N']: - keep_asking = False - break - if var in ['y', 'Y']: - keep_asking = False - print() - # if the repo exists, take backup - if os.path.exists(swap_option.destination_dir): - backup_dir = swap_option.destination_dir + '_old_' + generate_random_string() - bldinstallercommon.create_dirs(backup_dir) - bldinstallercommon.copy_tree(swap_option.destination_dir, backup_dir) - print('Backup taken into: {0}'.format(backup_dir)) - shutil.rmtree(swap_option.destination_dir) - bldinstallercommon.create_dirs(swap_option.destination_dir) - bldinstallercommon.copy_tree(swap_option.source_dir, swap_option.destination_dir) - print('Repository updated: {0}'.format(swap_option.destination_dir)) - - -############################### -# function -############################### -if __name__ == "__main__": - parser_args = parse_cmd_line_args() - swap_repository(parser_args) - - diff --git a/packaging-tools/update_repository.py b/packaging-tools/update_repository.py deleted file mode 100644 index d1d423d54..000000000 --- a/packaging-tools/update_repository.py +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env python -############################################################################# -## -## Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). -## Contact: http://www.qt-project.org/legal -## -## This file is part of the release tools of the Qt Toolkit. -## -## $QT_BEGIN_LICENSE:LGPL$ -## Commercial License Usage -## Licensees holding valid commercial Qt licenses may use this file in -## accordance with the commercial license agreement provided with the -## Software or, alternatively, in accordance with the terms contained in -## a written agreement between you and Digia. For licensing terms and -## conditions see http://qt.digia.com/licensing. For further information -## use the contact form at http://qt.digia.com/contact-us. -## -## GNU Lesser General Public License Usage -## Alternatively, this file may be used under the terms of the GNU Lesser -## General Public License version 2.1 as published by the Free Software -## Foundation and appearing in the file LICENSE.LGPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU Lesser General Public License version 2.1 requirements -## will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. -## -## In addition, as a special exception, Digia gives you certain additional -## rights. These rights are described in the Digia Qt LGPL Exception -## version 1.1, included in the file LGPL_EXCEPTION.txt in this package. -## -## GNU General Public License Usage -## Alternatively, this file may be used under the terms of the GNU -## General Public License version 3.0 as published by the Free Software -## Foundation and appearing in the file LICENSE.GPL included in the -## packaging of this file. Please review the following information to -## ensure the GNU General Public License version 3.0 requirements will be -## met: http://www.gnu.org/copyleft/gpl.html. -## -## -## $QT_END_LICENSE$ -## -############################################################################# - -# import the print function which is used in python 3.x -from __future__ import print_function -import sys -import os -import argparse -from time import gmtime, strftime -import bldinstallercommon - -TIMESTAMP = strftime('%Y-%m-%d-%H-%M', gmtime()) -ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) -REPOGEN_TOOL = 'repogen' -REPOGEN_TOOLS_DIR = os.path.join(ROOT_DIR, 'repogen_tools') - -UPDATE_NEW_COMPONENTS_ONLY = False # default to update all (given) components - - -############################### -# Setup argument parser -############################### -def setup_argument_parser(): - parser = argparse.ArgumentParser(prog=os.path.basename(sys.argv[0]), - add_help=True, description="Update online repository", - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('--repogen_tools', help="Where to fetch repogen tools (.7z, .zip, .tar.gz)", required=True, default="") - parser.add_argument('--target_repo', help="Repository to be updated", required=True, default="") - parser.add_argument('--source_repo', help="New repository source. Used only if target repository does not exits i.e. first time usage", required=False, default="") - parser.add_argument('--source_pkg', help="pkg folder containing updates", required=True, default="") - parser.add_argument('--source_config_xml', help="Path to config.xml file", required=False, default="") - parser.add_argument('--components_to_update', help="Comma separated list of component to update", required=False, default="") - parser.add_argument('--backup_base_dir', help="Backup directory, if given a backup will be taken from the repo when making updates", required=False, default="") - parser.add_argument('--update_new_components_only', help="Update component(s) only if version number increase", required=False, action='store_true', default=False) - return parser - - -############################### -# Function -############################### -def fetch_repogen_tools(tools_uri): - global REPOGEN_TOOL - executable_suffix = bldinstallercommon.get_executable_suffix() - # first check if we have existing copy of the tool - if os.path.exists(REPOGEN_TOOLS_DIR): - tool = bldinstallercommon.locate_executable(REPOGEN_TOOLS_DIR, REPOGEN_TOOL + executable_suffix) - if os.path.isfile(tool): - REPOGEN_TOOL = tool - print('Found existing repogen tool: {0}'.format(REPOGEN_TOOL)) - return - else: - # remove the bogus directory - bldinstallercommon.remove_tree(REPOGEN_TOOLS_DIR) - - # create dirs - bldinstallercommon.create_dirs(REPOGEN_TOOLS_DIR) - # fetch - print('Fetch repogen tools') - if bldinstallercommon.is_content_url_valid(tools_uri): - package_save_as_temp = os.path.normpath(os.path.join(ROOT_DIR, os.path.basename(tools_uri))) - bldinstallercommon.retrieve_url(tools_uri, package_save_as_temp) - bldinstallercommon.extract_file(package_save_as_temp, REPOGEN_TOOLS_DIR) - print('Trying to locate repogen tool: {0}'.format(REPOGEN_TOOL + executable_suffix)) - tool = bldinstallercommon.locate_executable(REPOGEN_TOOLS_DIR, REPOGEN_TOOL + executable_suffix) - if not os.path.isfile(tool): - raise IOError('Unable to locate repogen tool [%s] under directory: %s' % (REPOGEN_TOOL + executable_suffix, REPOGEN_TOOLS_DIR)) - else: - REPOGEN_TOOL = tool - else: - raise IOError('Invalid url: %s' % tools_uri) - - # found the tool - print('Using repogen tool: {0}'.format(REPOGEN_TOOL)) - - -############################### -# Function -############################### -def update_repository(source_pkg, target_repo, components_to_update): - print('Updating repository') - print(' Target repository: {0}'.format(target_repo)) - print(' Source pkg: {0}'.format(source_pkg)) - print(' Components: {0}'.format(components_to_update)) - print() - if not len(components_to_update): - raise RuntimeError('*** You asked me to update nothing?') - if not os.path.exists(source_pkg): - raise IOError('*** Source pkg does not exist: %s' % source_pkg) - if not os.path.exists(target_repo): - raise IOError('*** Target repository does not exist: %s' % target_repo) - # do we update new components only or all given components no matter - # what the version numbers are - repogen_update_cmd = '--update' - if UPDATE_NEW_COMPONENTS_ONLY: - repogen_update_cmd = '--update-new-components' - cmd_args = [REPOGEN_TOOL, repogen_update_cmd, '-p', source_pkg] - if components_to_update[0] and components_to_update[0] == '*': - cmd_args += [target_repo] - else: - cmd_args += ['--include', ','.join(components_to_update), target_repo] - bldinstallercommon.do_execute_sub_process(cmd_args, source_pkg, True) - - -############################### -# Function -############################### -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - - -############################### -# Function -############################### -def sanity_check(component_list, source_pkg): - source_packages = [] - source_pkg_path = os.path.join(source_pkg, 'online_repository') - for name in os.listdir(source_pkg_path): - temp = os.path.join(source_pkg_path, name) - if os.path.isdir(temp): - source_packages.append(name) - for item in component_list: - orig_item = item - if item == '*': - break - if '*' in item: - left, dummy = item.split('*') - item = left.rstrip('.') - if item not in source_packages: - print('*** Sanity check fail!') - print('*** Can not update component: [{0}] as it does not exist under: {1}'.format(orig_item, source_pkg_path)) - raise RuntimeError() - - -############################### -# Function -############################### -def expand_wildcard(selection, component_list): - expanded_list = [] - left, dummy = selection.split('*') - component = left.rstrip('.') - for dummy, item in enumerate(component_list): - # e.g. if 'qt.502.*' found in 'qt.502.gcc' - if component in item: - expanded_list.append(item) - return expanded_list - - -############################### -# Function -############################### -def ask_for_components(source_pkg): - components = [] - for name in os.listdir(source_pkg): - temp = os.path.join(source_pkg, name) - if os.path.isdir(temp): - components.append(name) - - component_list = [] - selected_items = [] - var = '' - count = len(components) - while True: - print() - print('Which component you wish to update? Provide comma separated list:') - for counter, item in enumerate(components): - marker = '-' - if counter in selected_items: - marker = '+' - print('{0} {1} {2}'.format(counter, marker, item)) - print() - print('a: Select all') - print('c: Continue') - print() - var = raw_input("Enter item number: ") - if var in ['c', 'C']: - break - if var in ['a', 'A']: - for counter, item in enumerate(components): - selected_items.append(counter) - component_list.append(item) - break - if is_number(var) and int(var) not in selected_items and (0 <= int(var) < count): - selected_items.append(int(var)) - component_list.append(components[int(var)]) - if '*' in var: - expanded_components = expand_wildcard(var, components) - component_list += expanded_components - break - print() - print('You are about to update the following components:') - print() - for item in component_list: - print(' {0}'.format(item)) - print() - var = raw_input("Is the above selection correct? y/n ") - if var not in ['y', 'Y']: - print('*** Aborting...') - raise RuntimeError() - - # return the components to be updated - return component_list - - -############################### -# Function -############################### -def backup_repo(backup_base_dir, directory_to_be_backed_up): - backup_full_path = os.path.join(backup_base_dir, TIMESTAMP) - # create dirs - bldinstallercommon.create_dirs(backup_full_path) - # backup - bldinstallercommon.copy_tree(directory_to_be_backed_up, backup_full_path) - print('Created backup of repository:') - print('Source: {0}'.format(directory_to_be_backed_up)) - print('Destination: {0}'.format(backup_full_path)) - - -############################### -# Function -############################### -def parse_components_from_argument(caller_arguments): - global UPDATE_NEW_COMPONENTS_ONLY - if caller_arguments.update_new_components_only: - UPDATE_NEW_COMPONENTS_ONLY = True - - components_to_update_list = caller_arguments.components_to_update - components_to_update_list = components_to_update_list.replace(" ", "") - split_components = caller_arguments.components_to_update.split(',') - return_list = [] - # parse all possible components in source repo - full_component_listing = [] - source_pkg_path = os.path.join(caller_arguments.source_pkg, 'pkg') - for name in os.listdir(source_pkg_path): - temp = os.path.join(source_pkg_path, name) - if os.path.isdir(temp): - full_component_listing.append(name) - # figure out all components that should be updated - for item in split_components: - if '*' in item: - expanded_items = expand_wildcard(item, full_component_listing) - return_list += expanded_items - else: - return_list.append(item) - # return list should contain all single items and items with wild mark - return return_list - - -############################### -# Main -############################### -if __name__ == "__main__": - # init things - bldinstallercommon.init_common_module(ROOT_DIR) - PARSER = setup_argument_parser() - # parse args - CALLER_ARGUMENTS = PARSER.parse_args() - # check if first time usage! - # 1) target repository directory must be empty i.e. we initialize things for the first time - # 2) copy the source repository as target repository 1:1 and nothing else - if CALLER_ARGUMENTS.source_repo: - source_repo = os.path.join(CALLER_ARGUMENTS.source_repo, 'online_repository') - if not os.path.isdir(source_repo) or not os.path.isfile(os.path.join(source_repo, 'Updates.xml')): - print('*** The given source directory does not seem to be proper repository? Abort!') - print('Given source repository: {0}'.format(source_repo)) - raise RuntimeError() - if os.path.isfile(os.path.join(CALLER_ARGUMENTS.target_repo, 'Updates.xml')): - print('The given destination directory already contains a repository.') - print('We just update the existing repository:') - print('Given target repository: {0}'.format(CALLER_ARGUMENTS.target_repo)) - else: - print('Initializing the repository for the first time!') - # create dirs - bldinstallercommon.create_dirs(CALLER_ARGUMENTS.target_repo) - # copy repository - bldinstallercommon.copy_tree(source_repo, CALLER_ARGUMENTS.target_repo) - # everything done now! - print('Repository initialized:') - print('Source: {0}'.format(source_repo)) - print('Destination: {0}'.format(CALLER_ARGUMENTS.target_repo)) - sys.exit() - # fetch tools - fetch_repogen_tools(CALLER_ARGUMENTS.repogen_tools) - # components to update - COMPONENTS_TO_UPDATE = [] - if not CALLER_ARGUMENTS.components_to_update or CALLER_ARGUMENTS.components_to_update == '': - # ask user which components to update - COMPONENTS_TO_UPDATE = ask_for_components(CALLER_ARGUMENTS.source_pkg) - else: - COMPONENTS_TO_UPDATE = parse_components_from_argument(CALLER_ARGUMENTS) - # sanity check - source_pkg_path = os.path.join(CALLER_ARGUMENTS.source_pkg, 'pkg') - sanity_check(COMPONENTS_TO_UPDATE, CALLER_ARGUMENTS.source_pkg) - # backup current repo - if CALLER_ARGUMENTS.backup_base_dir: - backup_repo(CALLER_ARGUMENTS.backup_base_dir, CALLER_ARGUMENTS.target_repo) - # update repo - update_repository(source_pkg_path, CALLER_ARGUMENTS.target_repo, COMPONENTS_TO_UPDATE) - print('\nRepository updated successfully!') - - - - - - - - - - - - - - - - - diff --git a/packaging-tools/winzipdir.sh b/packaging-tools/winzipdir.sh deleted file mode 100755 index 38cdd5604..000000000 --- a/packaging-tools/winzipdir.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). -# Contact: http://www.qt-project.org/legal -# -# You may use this file under the terms of the 3-clause BSD license. -# See the file LICENSE from this package for details. -# -# -# -# Script for generating a zip file with windows line endings for text file -# - -USAGE="Usage: `basename $0` zipfile dir" - -if [ $# -ne 2 ]; then - echo $USAGE >&2 - exit 1 -fi - -zipfile=$1 -dir=$2 - -tmp_txtpattern=$(mktemp) -tmp_filetypes=$(mktemp) - -# Make sure intermediate files are removed on exit -trap "rm -f $tmp_txtpattern $tmp_filetypes >/dev/null 2>&1" 0 -trap "exit 2" 1 2 3 13 15 - -echo ".*:.*ASCII -.*:.*directory -.*:.*empty -.*:.*POSIX -.*:.*html -.*:.*text" > $tmp_txtpattern || exit 1 - -# list all files and record file types -find $dir -exec file {} \; > $tmp_filetypes || exit 1 -# zip text files and binary files separately -cat $tmp_filetypes | grep -f $tmp_txtpattern -v | cut -d: -f1 | zip -9 -q $zipfile -@ -cat $tmp_filetypes | grep -f $tmp_txtpattern | cut -d: -f1 | zip -9 -q --to-crlf $zipfile -@ || exit 1 |