Implement dynamic resolve vaapi symbols

Let's get rid of the linking vaapi dependency, load and resolve
in runtime instead.

Change-Id: Icbc72470c935c1ae711c5457c8e85e4f96977a3b
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit ff2a0decb571e78d463d581eac5ab5d6f69ef381)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

5 files changed, 199 insertions, 9 deletions

diff --git a/cmake/FindFFmpeg.cmake b/cmake/FindFFmpeg.cmake
index 8a1d8d713..fb92a9f08 100644
--- a/cmake/FindFFmpeg.cmake
+++ b/cmake/FindFFmpeg.cmake
@@ -174,6 +174,27 @@ set(FFMPEG_LIBRARIES "")
 set(FFMPEG_DEFINITIONS "")
 set(FFMPEG_LIBRARY_DIRS "")
 
+# Apply dynamic symbols resolve for Linux build only. We might add Android and QNX as well.
+set(ENABLE_DYNAMIC_RESOLVE_OPENSSL_SYMBOLS ${LINUX} CACHE INTERNAL "")
+set(ENABLE_DYNAMIC_RESOLVE_VAAPI_SYMBOLS ${LINUX} CACHE INTERNAL "")
+
+function(__try_add_dynamic_resolve_dependency dep added)
+  set(added TRUE PARENT_SCOPE)
+
+  if(ENABLE_DYNAMIC_RESOLVE_OPENSSL_SYMBOLS AND
+      (${dep} STREQUAL "ssl" OR ${dep} STREQUAL "crypto"))
+    set(DYNAMIC_RESOLVE_OPENSSL_SYMBOLS TRUE CACHE INTERNAL "")
+  elseif(ENABLE_DYNAMIC_RESOLVE_VAAPI_SYMBOLS AND ${dep} STREQUAL "va")
+    set(DYNAMIC_RESOLVE_VAAPI_SYMBOLS TRUE CACHE INTERNAL "")
+  elseif(ENABLE_DYNAMIC_RESOLVE_VAAPI_SYMBOLS AND ${dep} STREQUAL "va-drm")
+    set(DYNAMIC_RESOLVE_VA_DRM_SYMBOLS TRUE CACHE INTERNAL "")
+  elseif(ENABLE_DYNAMIC_RESOLVE_VAAPI_SYMBOLS AND ${dep} STREQUAL "va-x11")
+    set(DYNAMIC_RESOLVE_VA_X11_SYMBOLS TRUE CACHE INTERNAL "")
+  else()
+    set(added FALSE PARENT_SCOPE)
+  endif()
+endfunction()
+
 # Function parses package config file to find the static library dependencies
 # and adds them to the target library.
 function(__ffmpeg_internal_set_dependencies lib)
@@ -196,11 +217,8 @@ function(__ffmpeg_internal_set_dependencies lib)
     foreach(dependency ${deps_no_suffix})
       string(REGEX REPLACE ${prefix_l} "" dependency ${dependency})
       if(NOT ${lib} STREQUAL ${dependency})
-        # Apply dynamic symbols resolve for Linux build only. We might add Android and QNX as well.
-        if(LINUX AND (${dependency} STREQUAL "ssl" OR ${dependency} STREQUAL "crypto"))
-          # TODO: implement OpenSsl headers check (or reuse WrapOpenSSLHeaders_FOUND)
-          set(DYNAMIC_RESOLVE_OPENSSL_SYMBOLS TRUE CACHE INTERNAL "")
-        else()
+        __try_add_dynamic_resolve_dependency(${dependency} added)
+        if(NOT added)
           target_link_libraries(FFmpeg::${lib} INTERFACE ${dependency})
         endif()
       endif()
diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
index 892361029..9a2cf8914 100644
--- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt
+++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
@@ -70,18 +70,58 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION DYNAMIC_RESOLVE_OPENSSL_S
         ${OPENSSL_INCLUDE_DIR}
 )
 
-if (DYNAMIC_RESOLVE_OPENSSL_SYMBOLS)
-    target_compile_definitions(QFFmpegMediaPlugin PRIVATE DYNAMIC_RESOLVE_OPENSSL_SYMBOLS)
+if (ENABLE_DYNAMIC_RESOLVE_VAAPI_SYMBOLS)
+    if (QT_FEATURE_vaapi AND NOT DYNAMIC_RESOLVE_VAAPI_SYMBOLS)
+        message(WARNING
+               "QT_FEATURE_vaapi is found but ffmpeg doesn't include vaapi,"
+               "however dynamic symbols resolve is possible.")
+
+        set(DYNAMIC_RESOLVE_OPENSSL_SYMBOLS TRUE CACHE INTERNAL "")
+    elseif (NOT QT_FEATURE_vaapi AND DYNAMIC_RESOLVE_VAAPI_SYMBOLS)
+
+        message(FATAL_ERROR
+               "QT_FEATURE_vaapi is not found "
+               "but ffmpeg includes VAAPI and dynamic symbols resolve is enabled.")
+    endif()
 endif()
 
-qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_ffmpeg AND QT_FEATURE_vaapi
+qt_internal_extend_target(QFFmpegMediaPlugin
+    CONDITION
+        DYNAMIC_RESOLVE_OPENSSL_SYMBOLS OR DYNAMIC_RESOLVE_VAAPI_SYMBOLS
+    SOURCES
+        qffmpegsymbolsresolveutils.cpp qffmpegsymbolsresolveutils_p.h
+)
+
+function (__propagate_to_compile_definitions VAR)
+    if (${VAR})
+        target_compile_definitions(QFFmpegMediaPlugin PRIVATE ${VAR})
+    endif()
+endfunction()
+
+__propagate_to_compile_definitions(DYNAMIC_RESOLVE_OPENSSL_SYMBOLS)
+__propagate_to_compile_definitions(DYNAMIC_RESOLVE_VAAPI_SYMBOLS)
+__propagate_to_compile_definitions(DYNAMIC_RESOLVE_VA_DRM_SYMBOLS)
+__propagate_to_compile_definitions(DYNAMIC_RESOLVE_VA_X11_SYMBOLS)
+
+qt_internal_extend_target(QFFmpegMediaPlugin CONDITION DYNAMIC_RESOLVE_VAAPI_SYMBOLS
+    SOURCES
+        qffmpegvaapisymbols.cpp
+    INCLUDE_DIRECTORIES
+        "$<TARGET_PROPERTY:VAAPI::VAAPI,INTERFACE_INCLUDE_DIRECTORIES>"
+)
+
+qt_internal_extend_target(QFFmpegMediaPlugin
+    CONDITION NOT DYNAMIC_RESOLVE_VAAPI_SYMBOLS AND QT_FEATURE_vaapi
+    LIBRARIES VAAPI::VAAPI
+)
+
+qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_vaapi
     SOURCES
         qffmpeghwaccel_vaapi.cpp qffmpeghwaccel_vaapi_p.h
     NO_UNITY_BUILD_SOURCES
         # Conflicts with macros defined in X11.h, and Xlib.h
         qffmpeghwaccel_vaapi.cpp
     LIBRARIES
-        VAAPI::VAAPI
         EGL::EGL
 )
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h
index 284bb7364..8064b8d85 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h
@@ -25,6 +25,11 @@ inline void resolveSymbols()
     extern bool resolveOpenSsl();
     resolveOpenSsl();
 #endif
+
+#ifdef DYNAMIC_RESOLVE_VAAPI_SYMBOLS
+    extern bool resolveVAAPI();
+    resolveVAAPI();
+#endif
 }
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h
index 780f1adc1..f7a2169d3 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h
@@ -110,6 +110,12 @@ struct FuncInfo<R(A...)>
 #define VARS3() VARS2(), VAR(2)
 #define VARS4() VARS3(), VAR(3)
 #define VARS5() VARS4(), VAR(4)
+#define VARS6() VARS5(), VAR(5)
+#define VARS7() VARS6(), VAR(6)
+#define VARS8() VARS7(), VAR(7)
+#define VARS9() VARS8(), VAR(8)
+#define VARS10() VARS9(), VAR(9)
+#define VARS11() VARS10(), VAR(10)
 
 #define TYPE_WITH_VAR(F, I) std::tuple_element_t<I, FuncInfo<decltype(F)>::Args> VAR(I)
 #define TYPES_WITH_VARS0(F)
@@ -118,6 +124,13 @@ struct FuncInfo<R(A...)>
 #define TYPES_WITH_VARS3(F) TYPES_WITH_VARS2(F), TYPE_WITH_VAR(F, 2)
 #define TYPES_WITH_VARS4(F) TYPES_WITH_VARS3(F), TYPE_WITH_VAR(F, 3)
 #define TYPES_WITH_VARS5(F) TYPES_WITH_VARS4(F), TYPE_WITH_VAR(F, 4)
+#define TYPES_WITH_VARS6(F) TYPES_WITH_VARS5(F), TYPE_WITH_VAR(F, 5)
+#define TYPES_WITH_VARS7(F) TYPES_WITH_VARS6(F), TYPE_WITH_VAR(F, 6)
+#define TYPES_WITH_VARS8(F) TYPES_WITH_VARS7(F), TYPE_WITH_VAR(F, 7)
+#define TYPES_WITH_VARS9(F) TYPES_WITH_VARS8(F), TYPE_WITH_VAR(F, 8)
+#define TYPES_WITH_VARS10(F) TYPES_WITH_VARS9(F), TYPE_WITH_VAR(F, 9)
+#define TYPES_WITH_VARS11(F) TYPES_WITH_VARS10(F), TYPE_WITH_VAR(F, 10)
+
 
 #define RET(F, ...) DefaultReturn<FuncInfo<decltype(F)>::Return>{__VA_ARGS__}
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvaapisymbols.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvaapisymbols.cpp
new file mode 100644
index 000000000..05588303e
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvaapisymbols.cpp
@@ -0,0 +1,114 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include <QtCore/qlibrary.h>
+
+#include "qffmpegsymbolsresolveutils_p.h"
+
+#include <QtCore/qglobal.h>
+#include <qstringliteral.h>
+
+#include <va/va.h>
+#include <va/va_drm.h>
+#include <va/va_x11.h>
+#include <va/va_str.h>
+
+QT_BEGIN_NAMESPACE
+
+static Libs loadLibs()
+{
+    Libs libs;
+    libs.push_back(std::make_unique<QLibrary>("va"));
+#ifdef DYNAMIC_RESOLVE_VA_DRM_SYMBOLS
+    libs.push_back(std::make_unique<QLibrary>("va-drm"));
+#endif
+
+#ifdef DYNAMIC_RESOLVE_VA_X11_SYMBOLS
+    libs.push_back(std::make_unique<QLibrary>("va-x11"));
+#endif
+
+    if (LibSymbolsResolver::tryLoad(libs))
+        return libs;
+
+    return {};
+}
+
+constexpr size_t symbolsCount = 39
+#ifdef DYNAMIC_RESOLVE_VA_DRM_SYMBOLS
+        + 1
+#endif
+#ifdef DYNAMIC_RESOLVE_VA_X11_SYMBOLS
+        + 1
+#endif
+        ;
+
+Q_GLOBAL_STATIC(LibSymbolsResolver, resolver, "VAAPI", symbolsCount, loadLibs);
+
+void resolveVAAPI()
+{
+    resolver()->resolve();
+}
+
+QT_END_NAMESPACE
+
+QT_USE_NAMESPACE
+
+DEFINE_FUNC(vaInitialize, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaTerminate, 1, VA_STATUS_ERROR_OPERATION_FAILED);
+
+constexpr auto errorStr = "VAAPI is not available";
+DEFINE_FUNC(vaErrorStr, 1, errorStr);
+DEFINE_FUNC(vaSetErrorCallback, 3);
+DEFINE_FUNC(vaSetInfoCallback, 3);
+
+DEFINE_FUNC(vaCreateImage, 5, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaGetImage, 7, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaPutImage, 11, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaDeriveImage, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaDestroyImage, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaQueryImageFormats, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+
+DEFINE_FUNC(vaBeginPicture, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaRenderPicture, 4, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaEndPicture, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+
+DEFINE_FUNC(vaCreateBuffer, 7, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaMapBuffer, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaUnmapBuffer, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaSyncBuffer, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaDestroyBuffer, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+
+DEFINE_FUNC(vaCreateSurfaces, 8, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaSyncSurface, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaExportSurfaceHandle, 5, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaDestroySurfaces, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+
+DEFINE_FUNC(vaCreateConfig, 6, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaGetConfigAttributes, 5, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaMaxNumProfiles, 1);
+DEFINE_FUNC(vaMaxNumImageFormats, 1);
+DEFINE_FUNC(vaMaxNumEntrypoints, 1);
+DEFINE_FUNC(vaQueryConfigProfiles, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaQueryConfigEntrypoints, 4, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaQuerySurfaceAttributes, 4, VA_STATUS_ERROR_OPERATION_FAILED);
+DEFINE_FUNC(vaDestroyConfig, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+
+DEFINE_FUNC(vaCreateContext, 8);
+DEFINE_FUNC(vaDestroyContext, 2);
+
+constexpr auto emptyString = "";
+DEFINE_FUNC(vaQueryVendorString, 1, emptyString);
+DEFINE_FUNC(vaProfileStr, 1, emptyString);
+DEFINE_FUNC(vaEntrypointStr, 1, emptyString);
+
+DEFINE_FUNC(vaGetDisplayAttributes, 3, VA_STATUS_ERROR_OPERATION_FAILED);
+
+DEFINE_FUNC(vaSetDriverName, 2, VA_STATUS_ERROR_OPERATION_FAILED);
+
+#ifdef DYNAMIC_RESOLVE_VA_DRM_SYMBOLS
+DEFINE_FUNC(vaGetDisplayDRM, 1); // va-drm
+#endif
+
+#ifdef DYNAMIC_RESOLVE_VA_X11_SYMBOLS
+DEFINE_FUNC(vaGetDisplay, 1); // va-x11
+#endif