From 26404dd4b6c61083bf5de1c1c2208bf2350aba0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simo=20F=C3=A4lt?= Date: Tue, 25 Jun 2019 09:47:54 +0300 Subject: Take new Qt CI config into use Change-Id: I32fcdaa2df6a4bed4f3263875ab1ad272ad8ff8c Reviewed-by: Qt CI Bot Reviewed-by: Cristian Maureira-Fredes --- coin/module_config.yaml | 546 +++++++++++++++++++++++++++++++++++++++++++++ coin_build_instructions.py | 9 +- coin_test_instructions.py | 6 +- 3 files changed, 551 insertions(+), 10 deletions(-) create mode 100644 coin/module_config.yaml diff --git a/coin/module_config.yaml b/coin/module_config.yaml new file mode 100644 index 000000000..71d4613b0 --- /dev/null +++ b/coin/module_config.yaml @@ -0,0 +1,546 @@ +version: 2 +accept_configuration: + condition: or + conditions: + - condition: and + conditions: + - condition: property # No cross compilation + property: host.os + equals_property: target.os + - condition: property # qtnamespace is not supported + property: configureArgs + not_contains_value: -qtnamespace + - condition: property # no-widgets is notsupported + property: configureArgs + not_contains_value: -no-widgets + - condition: property # no-gui is not supported + property: configureArgs + not_contains_value: -no-gui + - condition: property # Following configs are not supported + property: target.osVersion + not_in_values: [OPENSUSE_13_01, QEMU, WebAssembly, Ubuntu_18_04, SLES_12, SLES_15] + - condition: property # MibnGW and msvc2015 are not supported + property: target.compiler + not_in_values: [Mingw73, MSVC2015] + - condition: and + conditions: + - condition: property + property: target.osVersion + equals_value: Ubuntu_18_04 + - condition: property + property: features + contains_value: LicenseCheck + +set_environment: &set_environment + type: Group + instructions: + - type: EnvironmentVariable + variableName: TARGET_ARCHITECTURE + variableValue: amd64 + enable_if: + condition: and + conditions: + - condition: property + property: target.arch + equals_value: X86_64 + - condition: property + property: host.os + equals_value: Windows + - type: EnvironmentVariable + variableName: TARGET_ARCHITECTURE + variableValue: amd64_x86 + enable_if: + condition: and + conditions: + - condition: property + property: target.arch + equals_value: X86 + - condition: property + property: host.os + equals_value: Windows + - type: EnvironmentVariable + variableName: MAKEFLAGS + variableValue: "" + enable_if: + condition: property + property: host.os + equals_value: Windows + - type: AppendToEnvironmentVariable + variableName: PATH + variableValue: ;%CI_JOM_PATH% + enable_if: + condition: property + property: host.os + equals_value: Windows + - type: PrependToEnvironmentVariable + variableName: PATH + variableValue: "\\Users\\qt\\work\\install\\bin;" + enable_if: + condition: property + property: host.os + equals_value: Windows + - type: EnvironmentVariable + variableName: ICC64_18_LDLP + variableValue: /opt/intel/lib/intel64 + enable_if: + condition: property + property: target.compiler + equals_value: ICC_18 + - type: EnvironmentVariable + variableName: ICC64_18_PATH # Seems a bit hard to maintain + variableValue: /opt/intel/compilers_and_libraries_2018.1.163/linux/bin/intel64:/opt/intel/bin + enable_if: + condition: property + property: target.compiler + equals_value: ICC_18 + - type: AppendToEnvironmentVariable + variableName: LD_LIBRARY_PATH + variableValue: :/opt/intel/lib/intel64 + enable_if: + condition: property + property: target.compiler + equals_value: ICC_18 + - type: AppendToEnvironmentVariable + variableName: PATH + variableValue: ":{{.Env.ICC64_18_PATH}}" + enable_if: + condition: property + property: target.compiler + equals_value: ICC_18 + - type: EnvironmentVariable + variableName: QT_MODULE_TO_TEST + variableValue: "pyside/pyside-setup" + enable_if: + condition: property + property: features + contains_value: LicenseCheck + +find_path_to_msvc_compiler: &find_path_to_msvc_compiler + type: Group + enable_if: + condition: property + property: host.os + equals_value: Windows + instructions: + - type: EnvironmentVariable + variableName: VC_SCRIPT + variableValue: "%ProgramFiles(x86)%\\Microsoft Visual Studio\\2017\\Professional\\VC\\Auxiliary\\Build\\vcvarsall.bat" + enable_if: + condition: property + property: host.compiler + equals_value: MSVC2017 + - type: EnvironmentVariable + variableName: VC_SCRIPT + variableValue: "%ProgramFiles(x86)%\\Microsoft Visual Studio\\2019\\Professional\\VC\\Auxiliary\\Build\\vcvarsall.bat" + enable_if: + condition: property + property: host.compiler + equals_value: MSVC2019 + - type: WriteFile + fileContents: "call \"{{.Env.VC_SCRIPT}}\" {{.Env.TARGET_ARCHITECTURE}} \r\ncmd /c %*" + filename: "c:\\users\\qt\\MSVC.bat" + fileMode: 420 + enable_if: + condition: property + property: host.os + equals_value: Windows + +run_instructions: &run_instructions + type: Group + instructions: + - type: ExecuteCommand + command: python -u coin_build_instructions.py --os=MacOS --packaging --instdir=/Users/qt/work/install --targetOs=MacOS --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: MacOS + - condition: property + property: features + contains_value: Packaging + userMessageOnFailure: > + Failed to execute build instructions on osx + - type: ExecuteCommand + command: python -u coin_build_instructions.py --os=MacOS --instdir=/Users/qt/work/install --targetOs=MacOS --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: MacOS + - condition: property + property: features + not_contains_value: Packaging + userMessageOnFailure: > + Failed to execute build instructions on osx + - type: ExecuteCommand + command: python -u coin_build_instructions.py --os=Linux --packaging --instdir=/home/qt/work/install --targetOs=Linux --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Linux + - condition: property + property: features + contains_value: Packaging + userMessageOnFailure: > + Failed to execute build instructions on Linux + - type: ExecuteCommand + command: python -u coin_build_instructions.py --os=Linux --instdir=/home/qt/work/install --targetOs=Linux --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Linux + - condition: property + property: features + not_contains_value: Packaging + userMessageOnFailure: > + Failed to execute build instructions on Linux + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_build_instructions.py --os=Windows --packaging --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: features + contains_value: Packaging + - condition: property + property: target.arch + equals_property: host.arch + userMessageOnFailure: > + Failed to execute build instructions on 64 bit Windows + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_build_instructions.py --os=Windows --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: features + not_contains_value: Packaging + - condition: property + property: target.arch + equals_property: host.arch + userMessageOnFailure: > + Failed to execute build instructions on 64 bit Windows + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_build_instructions.py --os=Windows --packaging --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: features + contains_value: Packaging + - condition: property + property: target.arch + equals_value: X86 + userMessageOnFailure: > + Failed to execute 32 bit build instructions on 64 bit Windows + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_build_instructions.py --os=Windows --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 300 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: features + not_contains_value: Packaging + - condition: property + property: target.arch + equals_value: X86 + userMessageOnFailure: > + Failed to execute 32 bit build instructions on 64 bit Windows +run_test_instructions: &run_test_instructions + type: Group + enable_if: + condition: property + property: features + not_contains_value: LicenseCheck + instructions: + - type: ExecuteCommand + command: python -u coin_test_instructions.py --os=MacOS --instdir=/Users/qt/work/install --targetOs=MacOS --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: MacOS + - condition: property + property: features + not_contains_value: Packaging + userMessageOnFailure: > + Failed to execute test instructions on osx + - type: ExecuteCommand + command: python -u coin_test_instructions.py --packaging --os=MacOS --instdir=/Users/qt/work/install --targetOs=MacOS --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: MacOS + - condition: property + property: features + contains_value: Packaging + userMessageOnFailure: > + Failed to execute test instructions on osx + - type: ExecuteCommand + command: python -u coin_test_instructions.py --os=Linux --instdir=/home/qt/work/install --targetOs=Linux --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Linux + - condition: property + property: features + not_contains_value: Packaging + userMessageOnFailure: > + Failed to execute test instructions on Linux + - type: ExecuteCommand + command: python -u coin_test_instructions.py --packaging --os=Linux --instdir=/home/qt/work/install --targetOs=Linux --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 14400 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Linux + - condition: property + property: features + contains_value: Packaging + userMessageOnFailure: > + Failed to execute test instructions on Linux + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_test_instructions.py --os=Windows --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: target.arch + equals_property: host.arch + - condition: property + property: features + not_contains_value: Packaging + userMessageOnFailure: > + Failed to execute test instructions on 64 bit Windows + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_test_instructions.py --os=Windows --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86 + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: target.arch + equals_value: X86 + - condition: property + property: features + not_contains_value: Packaging + userMessageOnFailure: > + Failed to execute test instructions on 64 bit Windows + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_test_instructions.py --packaging --os=Windows --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86_64 + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: target.arch + equals_property: host.arch + - condition: property + property: features + contains_value: Packaging + userMessageOnFailure: > + Failed to execute 32 bit test instructions on 64 bit Windows + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat python -u coin_test_instructions.py --packaging --os=Windows --instdir=\Users\qt\work\install --targetOs=Windows --hostArch=X86_64 --targetArch=X86 + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: and + conditions: + - condition: property + property: host.os + equals_value: Windows + - condition: property + property: target.arch + equals_value: X86 + - condition: property + property: features + contains_value: Packaging + userMessageOnFailure: > + Failed to execute 32 bit test instructions on 64 bit Windows + +relocate_pyside: &relocate_pyside + type: Group + enable_if: + condition: property + property: features + not_contains_value: LicenseCheck + instructions: + - type: ChangeDirectory + directory: "{{.AgentWorkingDir}}/pyside" + maxTimeInSeconds: 300 + maxTimeBetweenOutput: 120 + userMessageOnFailure: > + Failed to change to pyside dir + - type: ExecuteCommand + command: c:\\users\\qt\\MSVC.bat rd /s /q pyside-setup + maxTimeInSeconds: 300 + maxTimeBetweenOutput: 120 + enable_if: + condition: property + property: host.os + equals_value: Windows + userMessageOnFailure: > + Failed to remove pyside-setup dir + - type: ExecuteCommand + command: /bin/rm -rf pyside-setup + maxTimeInSeconds: 300 + maxTimeBetweenOutput: 120 + enable_if: + condition: property + property: host.os + not_equals_value: Windows + userMessageOnFailure: > + Failed to remove pyside-setup dir + - type: InstallBinaryArchive + relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/artifacts.tar.gz" + directory: "pyside" + maxTimeInSeconds: 1200 + maxTimeBetweenOutput: 1200 + - type: ChangeDirectory + directory: "{{.AgentWorkingDir}}\\pyside\\pyside-setup" + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: property + property: host.os + equals_value: Windows + userMessageOnFailure: > + Failed to install binary artifacts + - type: ChangeDirectory + directory: "{{.AgentWorkingDir}}/pyside/pyside-setup" + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + enable_if: + condition: property + property: host.os + not_equals_value: Windows + userMessageOnFailure: > + Failed to change to pyside-setup dir + +upload_pyside: &upload_pyside + type: Group + instructions: + - type: UploadArtifact + archiveDirectory: "{{.AgentWorkingDir}}\\pyside" + transferType: UploadModuleBuildArtifact + maxTimeInSeconds: 1200 + maxTimeBetweenOutput: 1200 + enable_if: + condition: property + property: host.os + equals_value: Windows + - type: UploadArtifact + archiveDirectory: "{{.AgentWorkingDir}}/pyside" + transferType: UploadModuleBuildArtifact + maxTimeInSeconds: 1200 + maxTimeBetweenOutput: 1200 + enable_if: + condition: property + property: host.os + not_equals_value: Windows + +instructions: + LicenseCheck: + - type: ChangeDirectory + directory: "{{.AgentWorkingDir}}" + - type: InstallSourceArchive + maxTimeInSeconds: 600 + maxTimeBetweenOutput: 600 + project: qt/qtbase + ref: 5.14 + directory: qt/qtbase + userMessageOnFailure: > + Could not install source archive. Please investigate why. + - type: InstallSourceArchive + maxTimeInSeconds: 600 + maxTimeBetweenOutput: 600 + project: qt/qtqa + ref: master + directory: qt/qtqa-latest + userMessageOnFailure: > + Could not install source archive. Please investigate why. + - type: EnvironmentVariable + variableName: QT_MODULE_TO_TEST + variableValue: pyside/pyside-setup + userMessageOnFailure: > + Failed to set environment variable. This should not happen. + - type: ExecuteCommand + command: perl qt/qtqa-latest/tests/prebuild/license/tst_licenses.pl + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 120 + userMessageOnFailure: > + Failed to run license check script. + + Build: + - *set_environment + - *find_path_to_msvc_compiler + - *run_instructions + - *upload_pyside + Test: + - *set_environment + - *find_path_to_msvc_compiler + - *relocate_pyside + - *run_test_instructions + diff --git a/coin_build_instructions.py b/coin_build_instructions.py index cd55b9bcd..8df12ee4c 100644 --- a/coin_build_instructions.py +++ b/coin_build_instructions.py @@ -43,9 +43,10 @@ from build_scripts.utils import get_qtci_virtualEnv from build_scripts.utils import run_instruction from build_scripts.utils import rmtree from build_scripts.utils import get_python_dict -from build_scripts.utils import acceptCITestConfiguration from build_scripts.utils import get_ci_qmake_path import os +import datetime +import calendar # Values must match COIN thrift CI_HOST_OS = option_value("os") @@ -56,7 +57,7 @@ CI_HOST_OS_VER = option_value("osVer") CI_ENV_INSTALL_DIR = option_value("instdir") CI_ENV_AGENT_DIR = option_value("agentdir") CI_COMPILER = option_value("compiler") -CI_INTEGRATION_ID = option_value("coinIntegrationId") +CI_INTEGRATION_ID = option_value("coinIntegrationId") or str(calendar.timegm(datetime.datetime.now().timetuple())) CI_FEATURES = [] _ci_features = option_value("features") if _ci_features is not None: @@ -140,7 +141,6 @@ def call_setup(python_ver, phase): env = os.environ run_instruction(cmd, "Failed to run setup.py for build", initial_env=env) - def run_build_instructions(phase): # Uses default python, hopefully we have python2 installed on all hosts @@ -148,12 +148,11 @@ def run_build_instructions(phase): if CI_HOST_OS != "Windows": call_setup("", phase) # In case of packaging build, we have to build also python3 wheel + if CI_RELEASE_CONF and CI_HOST_OS_VER not in ["RHEL_6_6"]: call_setup("3", phase) if __name__ == "__main__": - if not acceptCITestConfiguration(CI_HOST_OS, CI_HOST_OS_VER, CI_TARGET_ARCH, CI_COMPILER): - exit() # Remove some environment variables that impact cmake for env_var in ['CC', 'CXX']: diff --git a/coin_test_instructions.py b/coin_test_instructions.py index 8ba82c26f..5c20e9848 100644 --- a/coin_test_instructions.py +++ b/coin_test_instructions.py @@ -42,7 +42,6 @@ from build_scripts.utils import install_pip_dependencies from build_scripts.utils import get_qtci_virtualEnv from build_scripts.utils import run_instruction from build_scripts.utils import rmtree -from build_scripts.utils import acceptCITestConfiguration from build_scripts.utils import get_ci_qmake_path import os @@ -82,9 +81,6 @@ def call_testrunner(python_ver, buildnro): run_instruction(cmd, "Error while running wheel_tester.py") def run_test_instructions(): - if not acceptCITestConfiguration(CI_HOST_OS, CI_HOST_OS_VER, CI_TARGET_ARCH, CI_COMPILER): - exit() - # Remove some environment variables that impact cmake for env_var in ['CC', 'CXX']: if os.environ.get(env_var): @@ -97,7 +93,7 @@ def run_test_instructions(): call_testrunner("", str(testRun)) testRun =+ 1 # We know that second build was with python3 - if CI_RELEASE_CONF and CI_HOST_OS_VER not in ["RHEL_6_6"]: + if CI_RELEASE_CONF: call_testrunner("3", str(testRun)) if __name__ == "__main__": -- cgit v1.2.3