From 0423814a2fc12f94e82b1f7d9508481dbb09189d Mon Sep 17 00:00:00 2001 From: Alexandru Croitor Date: Wed, 1 Jul 2020 18:14:41 +0200 Subject: CMake: Add support for enforcing passing CI tests per repository Create a new test instructions yaml template that does not ignore the exit code of running tests. This template can be included by repositories where tests pass in all configurations tested by the CI. The template should be included in module_config.yaml via Test: - !include "{{qt/qtbase}}/coin_module_test_template_v3.yaml" This allows us to opt into enforcing test runs per repository, rather than waiting to fix all tests in all qt5.git repositories. Implementation notes Try to extract the common parts of the instructions as much as possible to avoid duplication. Unfortunately some duplication still remains due to the restricted yaml language supported by Coin. Add a short README.md file that describes the differences between the existing templates. The v2 non-enforcing test template should probably be removed in the future. Task-number: QTBUG-84886 Task-number: QTBUG-85364 Change-Id: I718fc3cb44d6aefdbebc2fd2088a910a8095a375 Reviewed-by: Joerg Bornemann --- coin/instructions/README.md | 20 +++++++++++++++++ .../cmake_regular_test_instructions.yaml | 20 ++--------------- .../cmake_regular_test_instructions_common.yaml | 13 +++++++++++ .../cmake_regular_test_instructions_enforced.yaml | 8 +++++++ .../cmake_run_ctest_enforce_exit_code.yaml | 9 ++++++++ .../cmake_run_ctest_ignore_exit_code.yaml | 9 ++++++++ .../coin_module_test_template_common.yaml | 7 ++++++ .../instructions/coin_module_test_template_v2.yaml | 6 +----- .../instructions/coin_module_test_template_v3.yaml | 25 ++++++++++++++++++++++ 9 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 coin/instructions/README.md create mode 100644 coin/instructions/cmake_regular_test_instructions_common.yaml create mode 100644 coin/instructions/cmake_regular_test_instructions_enforced.yaml create mode 100644 coin/instructions/cmake_run_ctest_enforce_exit_code.yaml create mode 100644 coin/instructions/cmake_run_ctest_ignore_exit_code.yaml create mode 100644 coin/instructions/coin_module_test_template_common.yaml create mode 100644 coin/instructions/coin_module_test_template_v3.yaml (limited to 'coin/instructions') diff --git a/coin/instructions/README.md b/coin/instructions/README.md new file mode 100644 index 0000000000..aaefe3696f --- /dev/null +++ b/coin/instructions/README.md @@ -0,0 +1,20 @@ +# Information about Coin instruction templates + +## Build templates + +* ``coin_qtbase_build_template_v2.yaml`` did not exist. The build instructions were directly embedded into + ``module_config.yaml`` and did not support repos outside of qtbase, also no cross-compilation. +* ``coin_qtbase_build_template_v2`` introduced support for building other repos, and also enabled + build cross-compiling targets like ``Android`` and ``iOS``. + A bit later the template gained the ability to build ``qemu`` cross-compiling configurations. + The counterpart to qtbase to build other repositories is ``coin_module_build_template_v2`` + +## Test templates + +* ``coin_module_test_template_v1`` did not exist. The test instructions were directly embedded into + ``module_config.yaml`` and did not support repos outside of qtbase, also no cross-compilation. +* ``coin_module_test_template_v2`` introduced support for building tests for other repos, and made + sure not to build and run tests on cross-compiling configuration. A bit later the template gained + the ability to build and run tests for ``qemu`` cross-compiling configurations. +* ``coin_module_test_template_v3`` changed the run test instructions to not ignore the exit code + and thus enforce that tests pass in the CI. diff --git a/coin/instructions/cmake_regular_test_instructions.yaml b/coin/instructions/cmake_regular_test_instructions.yaml index 4a44924056..5d3571c5d8 100644 --- a/coin/instructions/cmake_regular_test_instructions.yaml +++ b/coin/instructions/cmake_regular_test_instructions.yaml @@ -1,23 +1,7 @@ type: Group instructions: - - type: MakeDirectory - directory: "{{.SourceDir}}_standalone_tests" - - type: InstallTestBinaryArchive - relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/tests.tar.gz" - directory: "{{.SourceDir}}_standalone_tests" - maxTimeInSeconds: 1200 - maxTimeBetweenOutput: 1200 - userMessageOnFailure: > - Failed to install tests archive. - - type: ChangeDirectory - directory: "{{.SourceDir}}_standalone_tests" - - type: ExecuteCommand - command: "{{.Env.TESTS_ENV_PREFIX}} ctest -V --rerun-failed --force-new-ctest-process" - ignoreExitCode: true - maxTimeInSeconds: 7200 - maxTimeBetweenOutput: 900 - userMessageOnFailure: > - Failed to run tests. + - !include "{{qt/qtbase}}/cmake_regular_test_instructions_common.yaml" + - !include "{{qt/qtbase}}/cmake_run_ctest_ignore_exit_code.yaml" disable_if: condition: property property: configureArgs diff --git a/coin/instructions/cmake_regular_test_instructions_common.yaml b/coin/instructions/cmake_regular_test_instructions_common.yaml new file mode 100644 index 0000000000..ec6f23a3f7 --- /dev/null +++ b/coin/instructions/cmake_regular_test_instructions_common.yaml @@ -0,0 +1,13 @@ +type: Group +instructions: + - type: MakeDirectory + directory: "{{.SourceDir}}_standalone_tests" + - type: InstallTestBinaryArchive + relativeStoragePath: "{{.Env.MODULE_ARTIFACTS_RELATIVE_STORAGE_PATH}}/tests.tar.gz" + directory: "{{.SourceDir}}_standalone_tests" + maxTimeInSeconds: 1200 + maxTimeBetweenOutput: 1200 + userMessageOnFailure: > + Failed to install tests archive. + - type: ChangeDirectory + directory: "{{.SourceDir}}_standalone_tests" diff --git a/coin/instructions/cmake_regular_test_instructions_enforced.yaml b/coin/instructions/cmake_regular_test_instructions_enforced.yaml new file mode 100644 index 0000000000..b7308a8f47 --- /dev/null +++ b/coin/instructions/cmake_regular_test_instructions_enforced.yaml @@ -0,0 +1,8 @@ +type: Group +instructions: + - !include "{{qt/qtbase}}/cmake_regular_test_instructions_common.yaml" + - !include "{{qt/qtbase}}/cmake_run_ctest_enforce_exit_code.yaml" +disable_if: + condition: property + property: configureArgs + contains_value: "-DBUILD_SHARED_LIBS=OFF" diff --git a/coin/instructions/cmake_run_ctest_enforce_exit_code.yaml b/coin/instructions/cmake_run_ctest_enforce_exit_code.yaml new file mode 100644 index 0000000000..13729c51f6 --- /dev/null +++ b/coin/instructions/cmake_run_ctest_enforce_exit_code.yaml @@ -0,0 +1,9 @@ +type: Group +instructions: + - type: ExecuteCommand + command: "{{.Env.TESTS_ENV_PREFIX}} ctest -V --rerun-failed --force-new-ctest-process" + ignoreExitCode: false + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 900 + userMessageOnFailure: > + Failed to run tests. diff --git a/coin/instructions/cmake_run_ctest_ignore_exit_code.yaml b/coin/instructions/cmake_run_ctest_ignore_exit_code.yaml new file mode 100644 index 0000000000..f2617ead68 --- /dev/null +++ b/coin/instructions/cmake_run_ctest_ignore_exit_code.yaml @@ -0,0 +1,9 @@ +type: Group +instructions: + - type: ExecuteCommand + command: "{{.Env.TESTS_ENV_PREFIX}} ctest -V --rerun-failed --force-new-ctest-process" + ignoreExitCode: true + maxTimeInSeconds: 7200 + maxTimeBetweenOutput: 900 + userMessageOnFailure: > + Failed to run tests. diff --git a/coin/instructions/coin_module_test_template_common.yaml b/coin/instructions/coin_module_test_template_common.yaml new file mode 100644 index 0000000000..716a4bb1e7 --- /dev/null +++ b/coin/instructions/coin_module_test_template_common.yaml @@ -0,0 +1,7 @@ +type: Group +instructions: + # The build env is needed on MSVC so that tst_qmake can properly build apps / libraries. + - !include "{{qt/qtbase}}/prepare_building_env.yaml" + # The test env vars are needed to pick the proper prefix.bat file. + - !include "{{qt/qtbase}}/cmake_setup_running_tests_env_vars.yaml" + - !include "{{qt/qtbase}}/coin_module_test_qemu_env_vars.yaml" diff --git a/coin/instructions/coin_module_test_template_v2.yaml b/coin/instructions/coin_module_test_template_v2.yaml index 786a59e03b..4691eff55a 100644 --- a/coin/instructions/coin_module_test_template_v2.yaml +++ b/coin/instructions/coin_module_test_template_v2.yaml @@ -2,11 +2,7 @@ type: Group instructions: - type: Group instructions: - # The build env is needed on MSVC so that tst_qmake can properly build apps / libraries. - - !include "{{qt/qtbase}}/prepare_building_env.yaml" - # The test env vars are needed to pick the proper prefix.bat file. - - !include "{{qt/qtbase}}/cmake_setup_running_tests_env_vars.yaml" - - !include "{{qt/qtbase}}/coin_module_test_qemu_env_vars.yaml" + - !include "{{qt/qtbase}}/coin_module_test_template_common.yaml" - type: Group instructions: - !include "{{qt/qtbase}}/cmake_regular_test_instructions.yaml" diff --git a/coin/instructions/coin_module_test_template_v3.yaml b/coin/instructions/coin_module_test_template_v3.yaml new file mode 100644 index 0000000000..9984305a03 --- /dev/null +++ b/coin/instructions/coin_module_test_template_v3.yaml @@ -0,0 +1,25 @@ +type: Group +instructions: + - type: Group + instructions: + - !include "{{qt/qtbase}}/coin_module_test_template_common.yaml" + - type: Group + instructions: + - !include "{{qt/qtbase}}/cmake_regular_test_instructions_enforced.yaml" + enable_if: + condition: or + conditions: + - condition: property + property: host.os + equals_property: target.os + - condition: property + property: target.osVersion + equals_value: QEMU + disable_if: + condition: property + property: features + contains_value: DisableTests + + - type: EnvironmentVariable + variableName: Dummy + variableValue: dummy -- cgit v1.2.3