Implement dynamic symbols resolve for openssl in ffmpeg

We should ship the networking QtMM functionality to Linux users.
Having openssl as a dependency is a bad idea since users might not
have openssl.

The patch implements dynamic symbols loading and resolving on QtMM
initialization.

We might reuse the approach to address other linking dependencies.

Change-Id: I0b21e44a66ebf892fb03f93844549e0877443481
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 00952848c0912a4f38ee6fced3e5aa12f8564aa1)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

7 files changed, 472 insertions, 2 deletions

diff --git a/cmake/FindFFmpeg.cmake b/cmake/FindFFmpeg.cmake
index ead6f7690..8a1d8d713 100644
--- a/cmake/FindFFmpeg.cmake
+++ b/cmake/FindFFmpeg.cmake
@@ -196,7 +196,13 @@ function(__ffmpeg_internal_set_dependencies lib)
     foreach(dependency ${deps_no_suffix})
       string(REGEX REPLACE ${prefix_l} "" dependency ${dependency})
       if(NOT ${lib} STREQUAL ${dependency})
-        target_link_libraries(FFmpeg::${lib} INTERFACE ${dependency})
+        # Apply dynamic symbols resolve for Linux build only. We might add Android and QNX as well.
+        if(LINUX AND (${dependency} STREQUAL "ssl" OR ${dependency} STREQUAL "crypto"))
+          # TODO: implement OpenSsl headers check (or reuse WrapOpenSSLHeaders_FOUND)
+          set(DYNAMIC_RESOLVE_OPENSSL_SYMBOLS TRUE CACHE INTERNAL "")
+        else()
+          target_link_libraries(FFmpeg::${lib} INTERFACE ${dependency})
+        endif()
       endif()
     endforeach()
 
diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
index ef7b1d8f1..892361029 100644
--- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt
+++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt
@@ -37,6 +37,7 @@ qt_internal_add_plugin(QFFmpegMediaPlugin
         qffmpegvideoencoderutils.cpp qffmpegvideoencoderutils_p.h
         qgrabwindowsurfacecapture.cpp qgrabwindowsurfacecapture_p.h
         qffmpegsurfacecapturethread.cpp qffmpegsurfacecapturethread_p.h
+        qffmpegsymbolsresolve_p.h
 
         qffmpegplaybackengine.cpp qffmpegplaybackengine_p.h
         playbackengine/qffmpegplaybackengineobject.cpp playbackengine/qffmpegplaybackengineobject_p.h
@@ -61,6 +62,18 @@ qt_internal_add_plugin(QFFmpegMediaPlugin
         FFmpeg::avformat FFmpeg::avcodec FFmpeg::swresample FFmpeg::swscale FFmpeg::avutil
 )
 
+qt_internal_extend_target(QFFmpegMediaPlugin CONDITION DYNAMIC_RESOLVE_OPENSSL_SYMBOLS
+    SOURCES
+        qffmpegsymbolsresolveutils.cpp qffmpegsymbolsresolveutils_p.h
+        qffmpegopensslsymbols.cpp
+    INCLUDE_DIRECTORIES
+        ${OPENSSL_INCLUDE_DIR}
+)
+
+if (DYNAMIC_RESOLVE_OPENSSL_SYMBOLS)
+    target_compile_definitions(QFFmpegMediaPlugin PRIVATE DYNAMIC_RESOLVE_OPENSSL_SYMBOLS)
+endif()
+
 qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_ffmpeg AND QT_FEATURE_vaapi
     SOURCES
         qffmpeghwaccel_vaapi.cpp qffmpeghwaccel_vaapi_p.h
@@ -72,7 +85,6 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_ffmpeg AND QT_
         EGL::EGL
 )
 
-
 qt_internal_extend_target(QFFmpegMediaPlugin CONDITION APPLE
     SOURCES
         ../darwin/qavfhelpers.mm ../darwin/qavfhelpers_p.h
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
index d4a1fb2aa..427e6743a 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
@@ -12,6 +12,7 @@
 #include "qffmpegimagecapture_p.h"
 #include "qffmpegaudioinput_p.h"
 #include "qffmpegaudiodecoder_p.h"
+#include "qffmpegsymbolsresolve_p.h"
 #include "qgrabwindowsurfacecapture_p.h"
 
 #ifdef Q_OS_MACOS
@@ -108,6 +109,8 @@ static void setupFFmpegLogger()
 
 QFFmpegMediaIntegration::QFFmpegMediaIntegration()
 {
+    resolveSymbols();
+
     setupFFmpegLogger();
 
     m_formatsInfo = new QFFmpegMediaFormatInfo();
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegopensslsymbols.cpp b/src/plugins/multimedia/ffmpeg/qffmpegopensslsymbols.cpp
new file mode 100644
index 000000000..76a2b49ae
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpegopensslsymbols.cpp
@@ -0,0 +1,185 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include <QtCore/qlibrary.h>
+
+#include "qffmpegsymbolsresolveutils_p.h"
+
+#include <QtCore/qglobal.h>
+#include <qstringliteral.h>
+
+#include <openssl/bio.h>
+#include <openssl/ssl.h>
+#include <openssl/bn.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+
+QT_BEGIN_NAMESPACE
+
+using namespace Qt::StringLiterals;
+
+static Libs loadLibs()
+{
+    Libs libs(2);
+    libs[0] = std::make_unique<QLibrary>();
+    libs[1] = std::make_unique<QLibrary>();
+
+    const auto majorVersion = OPENSSL_VERSION_NUMBER >> 28;
+
+    auto tryLoad = [&](QString sslName, QString cryptoName, auto version) {
+        libs[0]->setFileNameAndVersion(sslName, version);
+        libs[1]->setFileNameAndVersion(cryptoName, version);
+        return LibSymbolsResolver::tryLoad(libs);
+    };
+
+// Due to binary compatibility issues between 1.x.x openssl version, let's try taking exact version
+#if defined(SHLIB_VERSION_NUMBER)
+    if (majorVersion <= 1 && tryLoad("ssl"_L1, "crypto"_L1, SHLIB_VERSION_NUMBER ""_L1))
+        return libs;
+#endif
+
+// openssl on Android has specific suffixes
+#if defined(Q_OS_ANDROID)
+    {
+        auto suffix = qgetenv("ANDROID_OPENSSL_SUFFIX");
+        if (suffix.isEmpty())
+            suffix = QString("_"_L1) + QString::number(majorVersion);
+
+        if (tryLoad("ssl"_L1 + suffix, "crypto"_L1 + suffix, -1))
+            return libs;
+    }
+#endif
+
+    if (tryLoad("ssl"_L1, "crypto"_L1, majorVersion))
+        return libs;
+
+    return {};
+};
+
+Q_GLOBAL_STATIC(LibSymbolsResolver, resolver, "OpenSsl", 75, loadLibs);
+
+void resolveOpenSsl()
+{
+    resolver()->resolve();
+}
+
+QT_END_NAMESPACE
+
+QT_USE_NAMESPACE
+
+// BN functions
+
+DEFINE_FUNC(BN_value_one, 0);
+DEFINE_FUNC(BN_mod_word, 2);
+
+DEFINE_FUNC(BN_div_word, 2)
+DEFINE_FUNC(BN_mul_word, 2)
+DEFINE_FUNC(BN_add_word, 2)
+DEFINE_FUNC(BN_sub_word, 2)
+DEFINE_FUNC(BN_set_word, 2)
+DEFINE_FUNC(BN_new, 0)
+DEFINE_FUNC(BN_cmp, 2)
+
+DEFINE_FUNC(BN_free, 1);
+
+DEFINE_FUNC(BN_copy, 2);
+
+DEFINE_FUNC(BN_CTX_new, 0);
+
+DEFINE_FUNC(BN_CTX_free, 1);
+DEFINE_FUNC(BN_CTX_start, 1);
+
+DEFINE_FUNC(BN_CTX_get, 1);
+DEFINE_FUNC(BN_CTX_end, 1);
+
+DEFINE_FUNC(BN_rand, 4);
+DEFINE_FUNC(BN_mod_exp, 5);
+
+DEFINE_FUNC(BN_num_bits, 1);
+DEFINE_FUNC(BN_num_bits_word, 1);
+
+DEFINE_FUNC(BN_bn2hex, 1);
+DEFINE_FUNC(BN_bn2dec, 1);
+
+DEFINE_FUNC(BN_hex2bn, 2);
+DEFINE_FUNC(BN_dec2bn, 2);
+DEFINE_FUNC(BN_asc2bn, 2);
+
+DEFINE_FUNC(BN_bn2bin, 2);
+DEFINE_FUNC(BN_bin2bn, 3);
+
+// BIO-related functions
+
+DEFINE_FUNC(BIO_new, 1);
+DEFINE_FUNC(BIO_free, 1);
+
+DEFINE_FUNC(BIO_read, 3, -1);
+DEFINE_FUNC(BIO_write, 3, -1);
+DEFINE_FUNC(BIO_s_mem, 0);
+
+DEFINE_FUNC(BIO_set_data, 2);
+
+DEFINE_FUNC(BIO_get_data, 1);
+DEFINE_FUNC(BIO_set_init, 2);
+
+DEFINE_FUNC(BIO_set_flags, 2);
+DEFINE_FUNC(BIO_test_flags, 2);
+DEFINE_FUNC(BIO_clear_flags, 2);
+
+DEFINE_FUNC(BIO_meth_new, 2);
+DEFINE_FUNC(BIO_meth_free, 1);
+
+DEFINE_FUNC(BIO_meth_set_write, 2);
+DEFINE_FUNC(BIO_meth_set_read, 2);
+DEFINE_FUNC(BIO_meth_set_puts, 2);
+DEFINE_FUNC(BIO_meth_set_gets, 2);
+DEFINE_FUNC(BIO_meth_set_ctrl, 2);
+DEFINE_FUNC(BIO_meth_set_create, 2);
+DEFINE_FUNC(BIO_meth_set_destroy, 2);
+DEFINE_FUNC(BIO_meth_set_callback_ctrl, 2);
+
+// SSL functions
+
+DEFINE_FUNC(SSL_CTX_new, 1);
+DEFINE_FUNC(SSL_CTX_up_ref, 1);
+DEFINE_FUNC(SSL_CTX_free, 1);
+
+DEFINE_FUNC(SSL_new, 1);
+DEFINE_FUNC(SSL_up_ref, 1);
+DEFINE_FUNC(SSL_free, 1);
+
+DEFINE_FUNC(SSL_accept, 1);
+DEFINE_FUNC(SSL_stateless, 1);
+DEFINE_FUNC(SSL_connect, 1);
+DEFINE_FUNC(SSL_read, 3, -1);
+DEFINE_FUNC(SSL_peek, 3);
+DEFINE_FUNC(SSL_write, 3, -1);
+DEFINE_FUNC(SSL_ctrl, 4);
+DEFINE_FUNC(SSL_shutdown, 1);
+DEFINE_FUNC(SSL_set_bio, 3);
+
+// options are unsigned long in openssl 1.1.1, and uint64 in 3.x.x
+DEFINE_FUNC(SSL_CTX_set_options, 2);
+
+DEFINE_FUNC(SSL_get_error, 2);
+DEFINE_FUNC(SSL_CTX_load_verify_locations, 3, -1);
+
+DEFINE_FUNC(SSL_CTX_set_verify, 3);
+DEFINE_FUNC(SSL_CTX_use_PrivateKey, 2);
+
+DEFINE_FUNC(SSL_CTX_use_PrivateKey_file, 3);
+DEFINE_FUNC(SSL_CTX_use_certificate_chain_file, 2);
+
+DEFINE_FUNC(ERR_get_error, 0);
+
+static char ErrorString[] = "Ssl not found";
+DEFINE_FUNC(ERR_error_string, 2, ErrorString);
+
+// TLS functions
+
+DEFINE_FUNC(TLS_client_method, 0);
+DEFINE_FUNC(TLS_server_method, 0);
+
+// RAND functions
+
+DEFINE_FUNC(RAND_bytes, 2);
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h
new file mode 100644
index 000000000..284bb7364
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolve_p.h
@@ -0,0 +1,32 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QFFMPEGSYMBOLSRESOLVE_P_H
+#define QFFMPEGSYMBOLSRESOLVE_P_H
+
+#include "qnamespace.h"
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+QT_BEGIN_NAMESPACE
+
+inline void resolveSymbols()
+{
+#ifdef DYNAMIC_RESOLVE_OPENSSL_SYMBOLS
+    extern bool resolveOpenSsl();
+    resolveOpenSsl();
+#endif
+}
+
+QT_END_NAMESPACE
+
+#endif // QFFMPEGSYMBOLSRESOLVE_P_H
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils.cpp b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils.cpp
new file mode 100644
index 000000000..c4a4d9666
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils.cpp
@@ -0,0 +1,103 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qffmpegsymbolsresolveutils_p.h"
+
+#include <qdebug.h>
+#include <algorithm>
+#include <qloggingcategory.h>
+
+QT_BEGIN_NAMESPACE
+
+static Q_LOGGING_CATEGORY(qLcLibSymbolsRelolver, "qt.multimedia.ffmpeg.libsymbolsresolver");
+
+LibSymbolsResolver::LibSymbolsResolver(const char *libName, size_t symbolsCount,
+                                       LibsLoader libsLoader)
+    : m_libName(libName), m_libsLoader(libsLoader)
+{
+    Q_ASSERT(m_libName);
+    Q_ASSERT(m_libsLoader);
+    m_symbols.reserve(symbolsCount);
+}
+
+bool LibSymbolsResolver::resolve()
+{
+    if (m_state.testAndSetRelaxed(Initial, Requested)
+        || !m_state.testAndSetAcquire(Ready, Finished))
+        return false;
+
+    qCDebug(qLcLibSymbolsRelolver)
+            << "Start" << m_libName << "symbols resolving:" << m_symbols.size() << "symbols";
+
+    Q_ASSERT(m_symbols.size() == m_symbols.capacity());
+
+    auto cleanup = qScopeGuard([this]() { m_symbols = {}; });
+
+    auto libs = m_libsLoader();
+    if (libs.empty()) {
+        qCWarning(qLcLibSymbolsRelolver) << "Couldn't load" << m_libName << "library";
+        return false;
+    }
+
+    std::vector<QFunctionPointer> functions(m_symbols.size());
+
+    auto resolveElement = [&libs](const SymbolElement &element) {
+        return resolve(libs, element.name);
+    };
+
+    std::transform(m_symbols.begin(), m_symbols.end(), functions.begin(), resolveElement);
+
+    if (std::find(functions.begin(), functions.end(), nullptr) != functions.end()) {
+        unload(libs);
+        qCWarning(qLcLibSymbolsRelolver) << "Couldn't resolve" << m_libName << "symbols";
+        return false;
+    }
+
+    for (size_t i = 0; i < functions.size(); ++i)
+        m_symbols[i].setter(functions[i]);
+
+    qCDebug(qLcLibSymbolsRelolver) << m_libName << "symbols resolved";
+    return true;
+}
+
+void LibSymbolsResolver::registerSymbol(const char *name, FunctionSetter setter)
+{
+    Q_ASSERT(setter);
+    Q_ASSERT(m_symbols.size() < m_symbols.capacity());
+
+    m_symbols.push_back({ name, setter });
+
+    // handle the corner case: a user has initialized QtMM with global vars construction
+    // and it happened before the symbols initializing
+    if (m_symbols.size() == m_symbols.capacity() && !m_state.testAndSetRelease(Initial, Ready)
+        && m_state.testAndSetRelease(Requested, Ready))
+        resolve();
+}
+
+void LibSymbolsResolver::unload(const Libs &libs)
+{
+    for (auto &lib : libs)
+        lib->unload();
+}
+
+bool LibSymbolsResolver::tryLoad(const Libs &libs)
+{
+    auto load = [](auto &lib) { return lib->load(); };
+    if (std::all_of(libs.begin(), libs.end(), load))
+        return true;
+
+    unload(libs);
+    return false;
+}
+
+QFunctionPointer LibSymbolsResolver::resolve(const Libs &libs, const char *symbolName)
+{
+    for (auto &lib : libs)
+        if (auto pointer = lib->resolve(symbolName))
+            return pointer;
+
+    qWarning() << "Cannot resolve symbol" << symbolName;
+    return nullptr;
+}
+
+QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h
new file mode 100644
index 000000000..780f1adc1
--- /dev/null
+++ b/src/plugins/multimedia/ffmpeg/qffmpegsymbolsresolveutils_p.h
@@ -0,0 +1,129 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QFFMPEGSYMBOLSRESOLVEUTILS_P_H
+#define QFFMPEGSYMBOLSRESOLVEUTILS_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API. It exists purely as an
+// implementation detail. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include <QtCore/qlibrary.h>
+
+#include <qatomic.h>
+
+#include <vector>
+#include <memory>
+#include <tuple>
+
+QT_BEGIN_NAMESPACE
+
+using Libs = std::vector<std::unique_ptr<QLibrary>>;
+
+class LibSymbolsResolver
+{
+public:
+    using FunctionSetter = void (*)(QFunctionPointer);
+    using LibsLoader = Libs (*)();
+
+    LibSymbolsResolver(const char *libName, size_t symbolsCount, LibsLoader libsLoader);
+
+    bool resolve();
+
+    void registerSymbol(const char *name, FunctionSetter setter);
+
+    static void unload(const Libs &libs);
+
+    static bool tryLoad(const Libs &libs);
+
+private:
+    static QFunctionPointer resolve(const Libs &libs, const char *symbolName);
+
+private:
+    const char *const m_libName;
+    LibsLoader m_libsLoader;
+
+    struct SymbolElement
+    {
+        const char *name;
+        FunctionSetter setter;
+    };
+
+    std::vector<SymbolElement> m_symbols;
+
+    enum State { Initial, Requested, Ready, Finished };
+
+    QAtomicInteger<int> m_state = Initial;
+};
+
+QT_END_NAMESPACE
+
+template <typename T>
+struct DefaultReturn
+{
+    template <typename... Arg>
+    T operator()(Arg &&...) { return val; }
+    T val;
+};
+
+template <>
+struct DefaultReturn<void>
+{
+    DefaultReturn(int = 0){};
+    template <typename... Arg>
+    void operator()(Arg &&...) { }
+};
+
+template <typename...>
+struct FuncInfo;
+
+template <typename R, typename... A>
+struct FuncInfo<R(A...)>
+{
+    using Return = R;
+    using Args = std::tuple<A...>;
+};
+
+// clang-format off
+
+#define DEFINE_FUNC_IMPL(F, Vars, TypesWithVars, ReturnFunc) \
+    using F##_ReturnType = FuncInfo<decltype(F)>::Return; \
+    using q_##F##_Type = F##_ReturnType (*)(TypesWithVars(F)); \
+    static q_##F##_Type q_##F = []() { \
+        auto setter = [](QFunctionPointer ptr) { q_##F = (q_##F##_Type)ptr; }; \
+        resolver()->registerSymbol(#F, setter); \
+        return [](TypesWithVars(F)) { return ReturnFunc(Vars()); }; \
+    }(); \
+    extern "C" [[maybe_unused]] F##_ReturnType F(TypesWithVars(F)) { return q_##F(Vars()); }
+
+#define VAR(I) a##I
+#define VARS0()
+#define VARS1() VAR(0)
+#define VARS2() VARS1(), VAR(1)
+#define VARS3() VARS2(), VAR(2)
+#define VARS4() VARS3(), VAR(3)
+#define VARS5() VARS4(), VAR(4)
+
+#define TYPE_WITH_VAR(F, I) std::tuple_element_t<I, FuncInfo<decltype(F)>::Args> VAR(I)
+#define TYPES_WITH_VARS0(F)
+#define TYPES_WITH_VARS1(F) TYPE_WITH_VAR(F, 0)
+#define TYPES_WITH_VARS2(F) TYPES_WITH_VARS1(F), TYPE_WITH_VAR(F, 1)
+#define TYPES_WITH_VARS3(F) TYPES_WITH_VARS2(F), TYPE_WITH_VAR(F, 2)
+#define TYPES_WITH_VARS4(F) TYPES_WITH_VARS3(F), TYPE_WITH_VAR(F, 3)
+#define TYPES_WITH_VARS5(F) TYPES_WITH_VARS4(F), TYPE_WITH_VAR(F, 4)
+
+#define RET(F, ...) DefaultReturn<FuncInfo<decltype(F)>::Return>{__VA_ARGS__}
+
+#define DEFINE_FUNC(F, ArgsCount, /*Return value*/...) \
+    DEFINE_FUNC_IMPL(F, VARS##ArgsCount, TYPES_WITH_VARS##ArgsCount, RET(F, __VA_ARGS__));
+
+// clang-format on
+
+#endif // QFFMPEGSYMBOLSRESOLVEUTILS_P_H