update function argument of SSL_CTX_set_options

openssl3 uses uint64_t for the options argument in SSL_CTX_set_options,
older ones used long.
sizeof(long) is not the same on any platform as sizeof(uint64_t)

Fixes: QTBUG-105041
Change-Id: If148ffd883f50b58bc284c6f2609337d80fb5c58
Pick-to: 5.15 6.2 6.3 6.4
Reviewed-by: Dmitry Shachnev <mitya57@gmail.com>
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

4 files changed, 12 insertions, 7 deletions

diff --git a/src/plugins/tls/openssl/qsslcontext_openssl.cpp b/src/plugins/tls/openssl/qsslcontext_openssl.cpp
index 9e59477634..33f0d41a09 100644
--- a/src/plugins/tls/openssl/qsslcontext_openssl.cpp
+++ b/src/plugins/tls/openssl/qsslcontext_openssl.cpp
@@ -63,9 +63,9 @@ static inline QString msgErrorSettingEllipticCurves(const QString &why)
     return QSslSocket::tr("Error when setting the elliptic curves (%1)").arg(why);
 }
 
-long QSslContext::setupOpenSslOptions(QSsl::SslProtocol protocol, QSsl::SslOptions sslOptions)
+qssloptions QSslContext::setupOpenSslOptions(QSsl::SslProtocol protocol, QSsl::SslOptions sslOptions)
 {
-    long options;
+    qssloptions options;
     switch (protocol) {
 QT_WARNING_PUSH
 QT_WARNING_DISABLE_DEPRECATED
@@ -476,7 +476,7 @@ QT_WARNING_POP
     }
 
     // Enable bug workarounds.
-    const long options = setupOpenSslOptions(configuration.protocol(), configuration.d->sslOptions);
+    const qssloptions options = setupOpenSslOptions(configuration.protocol(), configuration.d->sslOptions);
     q_SSL_CTX_set_options(sslContext->ctx, options);
 
     // Tell OpenSSL to release memory early
diff --git a/src/plugins/tls/openssl/qsslcontext_openssl_p.h b/src/plugins/tls/openssl/qsslcontext_openssl_p.h
index 55af2af292..3bd39baf0c 100644
--- a/src/plugins/tls/openssl/qsslcontext_openssl_p.h
+++ b/src/plugins/tls/openssl/qsslcontext_openssl_p.h
@@ -37,7 +37,8 @@ public:
                                                                bool allowRootCertOnDemandLoading);
     static std::shared_ptr<QSslContext> sharedFromPrivateConfiguration(QSslSocket::SslMode mode, QSslConfigurationPrivate *privConfiguration,
                                                                       bool allowRootCertOnDemandLoading);
-    static long setupOpenSslOptions(QSsl::SslProtocol protocol, QSsl::SslOptions sslOptions);
+
+    static qssloptions setupOpenSslOptions(QSsl::SslProtocol protocol, QSsl::SslOptions sslOptions);
 
     QSslError::SslError error() const;
     QString errorString() const;
diff --git a/src/plugins/tls/openssl/qsslsocket_openssl_symbols.cpp b/src/plugins/tls/openssl/qsslsocket_openssl_symbols.cpp
index c3d05f134c..899a6c4819 100644
--- a/src/plugins/tls/openssl/qsslsocket_openssl_symbols.cpp
+++ b/src/plugins/tls/openssl/qsslsocket_openssl_symbols.cpp
@@ -123,7 +123,7 @@ DEFINEFUNC2(void, OPENSSL_sk_push, OPENSSL_STACK *a, a, void *b, b, return, DUMM
 DEFINEFUNC(void, OPENSSL_sk_free, OPENSSL_STACK *a, a, return, DUMMYARG)
 DEFINEFUNC2(void *, OPENSSL_sk_value, OPENSSL_STACK *a, a, int b, b, return nullptr, return)
 DEFINEFUNC(int, SSL_session_reused, SSL *a, a, return 0, return)
-DEFINEFUNC2(unsigned long, SSL_CTX_set_options, SSL_CTX *ctx, ctx, unsigned long op, op, return 0, return)
+DEFINEFUNC2(qssloptions, SSL_CTX_set_options, SSL_CTX *ctx, ctx, qssloptions op, op, return 0, return)
 using info_callback = void (*) (const SSL *ssl, int type, int val);
 DEFINEFUNC2(void, SSL_set_info_callback, SSL *ssl, ssl, info_callback cb, cb, return, return)
 DEFINEFUNC(const char *, SSL_alert_type_string, int value, value, return nullptr, return)
diff --git a/src/plugins/tls/openssl/qsslsocket_openssl_symbols_p.h b/src/plugins/tls/openssl/qsslsocket_openssl_symbols_p.h
index 97999d4c39..3c387ab96b 100644
--- a/src/plugins/tls/openssl/qsslsocket_openssl_symbols_p.h
+++ b/src/plugins/tls/openssl/qsslsocket_openssl_symbols_p.h
@@ -185,7 +185,11 @@ QT_BEGIN_NAMESPACE
 // **************** Static declarations ******************
 
 #endif // !defined QT_LINKED_OPENSSL
-
+#if defined(OPENSSL_VERSION_MAJOR) && OPENSSL_VERSION_MAJOR >= 3
+typedef uint64_t qssloptions;
+#else
+typedef unsigned long qssloptions;
+#endif
 // TODO: the following lines previously were a part of 1.1 - specific header.
 // To reduce the amount of the change, I'm directly copying and pasting the
 // content of the header here. Later, can be better sorted/split into groups,
@@ -209,7 +213,7 @@ void q_OPENSSL_sk_push(OPENSSL_STACK *st, void *data);
 void q_OPENSSL_sk_free(OPENSSL_STACK *a);
 void * q_OPENSSL_sk_value(OPENSSL_STACK *a, int b);
 int q_SSL_session_reused(SSL *a);
-unsigned long q_SSL_CTX_set_options(SSL_CTX *ctx, unsigned long op);
+qssloptions q_SSL_CTX_set_options(SSL_CTX *ctx, qssloptions op);
 int q_OPENSSL_init_ssl(uint64_t opts, const OPENSSL_INIT_SETTINGS *settings);
 size_t q_SSL_get_client_random(SSL *a, unsigned char *out, size_t outlen);
 size_t q_SSL_SESSION_get_master_key(const SSL_SESSION *session, unsigned char *out, size_t outlen);