CMake: Add handling of TESTRUNNER and TESTARGS by Qt tests

Wrap Qt tests to handle TESTRUNNER and TESTARGS environment variables.
Variables are handled according to qmake build procedure. All test
now are wrapped by generated CMake script.

Fixes: QTBUG-88053
Change-Id: I339977ce6ce11ddd38a4bf0bd26eb8f2ae463ba3
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
(cherry picked from commit f19266bd02a01d4b7b277ea769c4c17727b1e661)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

1 files changed, 137 insertions, 36 deletions

diff --git a/cmake/QtTestHelpers.cmake b/cmake/QtTestHelpers.cmake
index 8a11705d8e..3405eee83a 100644
--- a/cmake/QtTestHelpers.cmake
+++ b/cmake/QtTestHelpers.cmake
@@ -141,10 +141,18 @@ function(qt_internal_setup_docker_test_fixture name)
 endfunction()
 
 # This function creates a CMake test target with the specified name for use with CTest.
+#
+# All tests are wrapped with cmake script that supports TESTARGS and TESTRUNNER environment
+# variables handling. Endpoint wrapper may be used standalone as cmake script to run tests e.g.:
+# TESTARGS="-o result.xml,xunitxml" TESTRUNNER="testrunner --arg" ./tst_simpleTestWrapper.cmake
+# On non-UNIX machine you may need to use 'cmake -P' explicitly to execute wrapper.
+# You may avoid test wrapping by either passing NO_WRAPPER option or switching QT_NO_TEST_WRAPPERS
+# to ON. This is helpful if you want to use internal CMake tools within tests, like memory or
+# sanitizer checks. See https://cmake.org/cmake/help/v3.19/manual/ctest.1.html#ctest-memcheck-step
 function(qt_internal_add_test name)
     # EXCEPTIONS is a noop as they are enabled by default.
     qt_parse_all_arguments(arg "qt_add_test"
-        "RUN_SERIAL;EXCEPTIONS;NO_EXCEPTIONS;GUI;QMLTEST;CATCH;LOWDPI"
+        "RUN_SERIAL;EXCEPTIONS;NO_EXCEPTIONS;GUI;QMLTEST;CATCH;LOWDPI;NO_WRAPPER"
         "OUTPUT_DIRECTORY;WORKING_DIRECTORY;TIMEOUT;VERSION"
         "QML_IMPORTPATH;TESTDATA;QT_TEST_SERVER_LIST;${__default_private_args};${__default_public_args}" ${ARGN}
     )
@@ -240,6 +248,42 @@ function(qt_internal_add_test name)
         endif()
     endif()
 
+    # Get path to <qt_relocatable_install_prefix>/bin, as well as CMAKE_INSTALL_PREFIX/bin, then
+    # prepend them to the PATH environment variable.
+    # It's needed on Windows to find the shared libraries and plugins.
+    # qt_relocatable_install_prefix is dynamically computed from the location of where the Qt CMake
+    # package is found.
+    # The regular CMAKE_INSTALL_PREFIX can be different for example when building standalone tests.
+    # Any given CMAKE_INSTALL_PREFIX takes priority over qt_relocatable_install_prefix for the
+    # PATH environment variable.
+    set(install_prefixes "${CMAKE_INSTALL_PREFIX}")
+    if(QT_BUILD_INTERNALS_RELOCATABLE_INSTALL_PREFIX)
+        list(APPEND install_prefixes "${QT_BUILD_INTERNALS_RELOCATABLE_INSTALL_PREFIX}")
+    endif()
+
+    file(TO_NATIVE_PATH "${CMAKE_CURRENT_BINARY_DIR}" test_env_path)
+    foreach(install_prefix ${install_prefixes})
+        file(TO_NATIVE_PATH "${install_prefix}/${INSTALL_BINDIR}" install_prefix)
+        set(test_env_path "${test_env_path}${QT_PATH_SEPARATOR}${install_prefix}")
+    endforeach()
+    set(test_env_path "${test_env_path}${QT_PATH_SEPARATOR}$ENV{PATH}")
+    string(REPLACE ";" "\;" test_env_path "${test_env_path}")
+
+    # Add the install prefix to list of plugin paths when doing a prefix build
+    if(NOT QT_INSTALL_DIR)
+        foreach(install_prefix ${install_prefixes})
+            file(TO_NATIVE_PATH "${install_prefix}/${INSTALL_BINDIR}" install_prefix)
+            list(APPEND plugin_paths "${install_prefix}")
+        endforeach()
+    endif()
+
+    #TODO: Collect all paths from known repositories when performing a super
+    # build.
+    file(TO_NATIVE_PATH "${PROJECT_BINARY_DIR}/${INSTALL_PLUGINSDIR}" install_pluginsdir)
+    list(APPEND plugin_paths "${install_pluginsdir}")
+    list(JOIN plugin_paths "${QT_PATH_SEPARATOR}" plugin_paths_joined)
+    string(REPLACE ";" "\;" plugin_paths_joined "${plugin_paths_joined}")
+
     if (ANDROID)
         qt_internal_android_add_test("${name}")
     else()
@@ -258,15 +302,25 @@ function(qt_internal_add_test name)
         endif()
 
         if (NOT arg_CATCH)
-            list(APPEND test_outputs "-o" "${name}.xml,xml" "-o" "-,txt")
+            #TODO: Should we replace this by TESTARGS environment variable?
+            list(APPEND extra_test_args "-o" "${name}.xml,xml" "-o" "-,txt")
         endif()
 
-        add_test(NAME "${name}" COMMAND ${test_executable} ${extra_test_args} ${test_outputs} WORKING_DIRECTORY "${test_working_dir}")
+        if(arg_NO_WRAPPER OR QT_NO_TEST_WRAPPERS)
+            add_test(NAME "${name}" COMMAND ${test_executable} ${extra_test_args}
+                     WORKING_DIRECTORY "${test_working_dir}")
+        else()
+            _qt_internal_wrap_test("${test_executable}" "${extra_test_args}" "${test_working_dir}"
+                                   ENVIRONMENT "QT_TEST_RUNNING_IN_CTEST" 1
+                                               "PATH" "${test_env_path}"
+                                               "QT_PLUGIN_PATH" "${plugin_paths_joined}")
+        endif()
 
         if (arg_QT_TEST_SERVER_LIST)
             qt_internal_setup_docker_test_fixture(${name} ${arg_QT_TEST_SERVER_LIST})
         endif()
     endif()
+
     set_tests_properties("${name}" PROPERTIES RUN_SERIAL "${arg_RUN_SERIAL}" LABELS "${label}")
     if (arg_TIMEOUT)
         set_tests_properties(${name} PROPERTIES TIMEOUT ${arg_TIMEOUT})
@@ -284,41 +338,14 @@ function(qt_internal_add_test name)
         endif()
     endif()
 
-    # Get path to <qt_relocatable_install_prefix>/bin, as well as CMAKE_INSTALL_PREFIX/bin, then
-    # prepend them to the PATH environment variable.
-    # It's needed on Windows to find the shared libraries and plugins.
-    # qt_relocatable_install_prefix is dynamically computed from the location of where the Qt CMake
-    # package is found.
-    # The regular CMAKE_INSTALL_PREFIX can be different for example when building standalone tests.
-    # Any given CMAKE_INSTALL_PREFIX takes priority over qt_relocatable_install_prefix for the
-    # PATH environment variable.
-    set(install_prefixes "${CMAKE_INSTALL_PREFIX}")
-    if(QT_BUILD_INTERNALS_RELOCATABLE_INSTALL_PREFIX)
-        list(APPEND install_prefixes "${QT_BUILD_INTERNALS_RELOCATABLE_INSTALL_PREFIX}")
-    endif()
-
-    set(test_env_path "PATH=${CMAKE_CURRENT_BINARY_DIR}")
-    foreach(install_prefix ${install_prefixes})
-        set(test_env_path "${test_env_path}${QT_PATH_SEPARATOR}${install_prefix}/${INSTALL_BINDIR}")
-    endforeach()
-    set(test_env_path "${test_env_path}${QT_PATH_SEPARATOR}$ENV{PATH}")
-    string(REPLACE ";" "\;" test_env_path "${test_env_path}")
-    set_property(TEST "${name}" APPEND PROPERTY ENVIRONMENT "${test_env_path}")
-    set_property(TEST "${name}" APPEND PROPERTY ENVIRONMENT "QT_TEST_RUNNING_IN_CTEST=1")
-
-    # Add the install prefix to list of plugin paths when doing a prefix build
-    if(NOT QT_INSTALL_DIR)
-        foreach(install_prefix ${install_prefixes})
-            list(APPEND plugin_paths "${install_prefix}/${INSTALL_PLUGINSDIR}")
-        endforeach()
+    if(ANDROID OR arg_NO_WRAPPER OR QT_NO_TEST_WRAPPERS)
+        set_property(TEST "${name}" APPEND PROPERTY ENVIRONMENT
+                    "PATH=${test_env_path}"
+                    "QT_TEST_RUNNING_IN_CTEST=1"
+                    "QT_PLUGIN_PATH=${plugin_paths_joined}"
+                     )
     endif()
 
-    #TODO: Collect all paths from known repositories when performing a super
-    # build.
-    list(APPEND plugin_paths "${PROJECT_BINARY_DIR}/${INSTALL_PLUGINSDIR}")
-    list(JOIN plugin_paths "${QT_PATH_SEPARATOR}" plugin_paths_joined)
-    set_property(TEST "${name}" APPEND PROPERTY ENVIRONMENT "QT_PLUGIN_PATH=${plugin_paths_joined}")
-
     if(ANDROID OR IOS OR WINRT)
         set(builtin_testdata TRUE)
     endif()
@@ -376,6 +403,80 @@ function(qt_internal_add_test name)
 
 endfunction()
 
+# This function wraps test with cmake script, that makes possible standalone run with external
+# arguments.
+function(_qt_internal_wrap_test test_executable extra_test_args test_working_dir)
+    cmake_parse_arguments(PARSE_ARGV 3 arg "" "" "ENVIRONMENT")
+
+    set(environment_extras)
+    set(skipNext false)
+    if(arg_ENVIRONMENT)
+        list(LENGTH arg_ENVIRONMENT length)
+        math(EXPR length "${length} - 1")
+        foreach(envIdx RANGE ${length})
+            if(skipNext)
+                set(skipNext FALSE)
+                continue()
+            endif()
+
+            set(envVariable "")
+            set(envValue "")
+
+            list(GET arg_ENVIRONMENT ${envIdx} envVariable)
+            math(EXPR envIdx "${envIdx} + 1")
+            if (envIdx LESS_EQUAL ${length})
+                list(GET arg_ENVIRONMENT ${envIdx} envValue)
+            endif()
+
+            if(NOT "${envVariable}" STREQUAL "")
+                set(environment_extras "${environment_extras}\nset(ENV{${envVariable}} \
+\"${envValue}\")")
+            endif()
+            set(skipNext TRUE)
+        endforeach()
+    endif()
+
+    #Escaping environment variables before expand them by file GENERATE
+    string(REPLACE "\\" "\\\\" environment_extras "${environment_extras}")
+
+    set(test_wrapper_name "${CMAKE_CURRENT_BINARY_DIR}/${name}Wrapper$<CONFIG>.cmake")
+    add_test(NAME "${name}" COMMAND "${CMAKE_COMMAND}" "-P" "${test_wrapper_name}"
+        WORKING_DIRECTORY "${test_working_dir}")
+
+    if(TARGET ${test_executable})
+        set(test_executable_file "$<TARGET_FILE:${test_executable}>")
+    else()
+        set(test_executable_file "${test_executable}")
+    endif()
+
+    # If crosscompiling is enabled, we should avoid run cmake in emulator environment.
+    # Prepend emulator to test command in generated cmake script instead. Keep in mind that
+    # CROSSCOMPILING_EMULATOR don't check if actual cross compilation is configured,
+    # emulator is prepended independently.
+    if(CMAKE_CROSSCOMPILING)
+        get_test_property(${name} CROSSCOMPILING_EMULATOR crosscompiling_emulator)
+    endif()
+
+    if(WIN32)
+        # It's necessary to call actual test inside 'cmd.exe', because 'execute_process' uses
+        # SW_HIDE to avoid showing a console window, it affects other GUI as well.
+        # See https://gitlab.kitware.com/cmake/cmake/-/issues/17690 for details.
+        set(extra_test_runner "cmd /c")
+    endif()
+
+    file(GENERATE OUTPUT "${test_wrapper_name}" CONTENT
+"#!${CMAKE_COMMAND} -P
+# Qt generated test wrapper for ${name}
+
+${environment_extras}
+execute_process(COMMAND ${crosscompiling_emulator} ${extra_test_runner} \$ENV{TESTRUNNER} \"${test_executable_file}\" \
+\$ENV{TESTARGS} ${extra_test_args} WORKING_DIRECTORY \"${test_working_dir}\" \
+RESULT_VARIABLE result)
+if(NOT result EQUAL 0)
+    message(FATAL_ERROR)
+endif()"
+)
+endfunction()
 
 # This function creates an executable for use as a helper program with tests. Some
 # tests launch separate programs to test certain input/output behavior.