TLS socket: make verification callback lock-free (OpenSSL)

When our QSslSocketBackendPrivate (OpenSSL backend) was developed,
the ancient versions of OpenSSL did not have an API needed to pass
an application-specific data into verification callback. Thus the
developers resorted to the use of global variables (a list with errors)
and locks. Some of our auto-tests use QNAM and in-process server.
Whenever the client (essentially qhttpthreadeddelegate) and the server
live in different threads, any use of 'https' is dead-lock prone,
which recent events demonstrated and which were previously observed
but not understood properly (rare occasions, not always easy to
reproduce). Now we fix this for good by removing locking.

There are two places (in 5.12) where these locks are needed:

1. Before calling SSL_connect/SSL_accept (handshake) - here
   we reuse the same trick we do in PSK callback ('SSL' has
   an external data set, and it's 'this', meaning an object
   of type QSslSocketBackendPrivate).

2. The static member function 'verify', here we do not have
   'SSL', but we have our temporary 'X509_STORE', to which
   we can directly attach an external data - a pointer to
   a vector to collect verification errors.

Note, this change assumes that OpenSSL Qt is build/linked
against is at least of version 1.0.1 - we set external data
on SSL unconditionally (no version checks).

Fixes: QTBUG-76157
Change-Id: I05c98e77dfd5fb0c2c260fb6c463732facf53ffc
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
 

1 files changed, 10 insertions, 0 deletions

diff --git a/src/network/ssl/qsslsocket_openssl_symbols.cpp b/src/network/ssl/qsslsocket_openssl_symbols.cpp
index 6a5ab91669..483a2cbad0 100644
--- a/src/network/ssl/qsslsocket_openssl_symbols.cpp
+++ b/src/network/ssl/qsslsocket_openssl_symbols.cpp
@@ -178,6 +178,8 @@ DEFINEFUNC(ASN1_TIME *, X509_getm_notAfter, X509 *a, a, return nullptr, return)
 DEFINEFUNC(long, X509_get_version, X509 *a, a, return -1, return)
 DEFINEFUNC(EVP_PKEY *, X509_get_pubkey, X509 *a, a, return nullptr, return)
 DEFINEFUNC2(void, X509_STORE_set_verify_cb, X509_STORE *a, a, X509_STORE_CTX_verify_cb verify_cb, verify_cb, return, DUMMYARG)
+DEFINEFUNC3(int, X509_STORE_set_ex_data, X509_STORE *a, a, int idx, idx, void *data, data, return 0, return)
+DEFINEFUNC2(void *, X509_STORE_get_ex_data, X509_STORE *r, r, int idx, idx, return nullptr, return)
 DEFINEFUNC(STACK_OF(X509) *, X509_STORE_CTX_get0_chain, X509_STORE_CTX *a, a, return nullptr, return)
 DEFINEFUNC3(void, CRYPTO_free, void *str, str, const char *file, file, int line, line, return, DUMMYARG)
 DEFINEFUNC(long, OpenSSL_version_num, void, DUMMYARG, return 0, return)
@@ -223,6 +225,8 @@ DEFINEFUNC(int, CRYPTO_num_locks, DUMMYARG, DUMMYARG, return 0, return)
 DEFINEFUNC(void, CRYPTO_set_locking_callback, void (*a)(int, int, const char *, int), a, return, DUMMYARG)
 DEFINEFUNC(void, CRYPTO_set_id_callback, unsigned long (*a)(), a, return, DUMMYARG)
 DEFINEFUNC(void, CRYPTO_free, void *a, a, return, DUMMYARG)
+DEFINEFUNC3(int, CRYPTO_set_ex_data, CRYPTO_EX_DATA *ad, ad, int idx, idx, void *val, val, return 0, return)
+DEFINEFUNC2(void *, CRYPTO_get_ex_data, const CRYPTO_EX_DATA *ad, ad, int idx, idx, return nullptr, return)
 DEFINEFUNC(unsigned long, ERR_peek_last_error, DUMMYARG, DUMMYARG, return 0, return)
 DEFINEFUNC(void, ERR_free_strings, void, DUMMYARG, return, DUMMYARG)
 DEFINEFUNC(void, EVP_CIPHER_CTX_cleanup, EVP_CIPHER_CTX *a, a, return, DUMMYARG)
@@ -533,6 +537,7 @@ DEFINEFUNC2(int, X509_STORE_CTX_set_purpose, X509_STORE_CTX *a, a, int b, b, ret
 DEFINEFUNC(int, X509_STORE_CTX_get_error, X509_STORE_CTX *a, a, return -1, return)
 DEFINEFUNC(int, X509_STORE_CTX_get_error_depth, X509_STORE_CTX *a, a, return -1, return)
 DEFINEFUNC(X509 *, X509_STORE_CTX_get_current_cert, X509_STORE_CTX *a, a, return nullptr, return)
+DEFINEFUNC(X509_STORE *, X509_STORE_CTX_get0_store, X509_STORE_CTX *ctx, ctx, return nullptr, return)
 DEFINEFUNC(X509_STORE_CTX *, X509_STORE_CTX_new, DUMMYARG, DUMMYARG, return nullptr, return)
 DEFINEFUNC2(void *, X509_STORE_CTX_get_ex_data, X509_STORE_CTX *ctx, ctx, int idx, idx, return nullptr, return)
 DEFINEFUNC(int, SSL_get_ex_data_X509_STORE_CTX_idx, DUMMYARG, DUMMYARG, return -1, return)
@@ -998,6 +1003,8 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(X509_get_version)
     RESOLVEFUNC(X509_get_pubkey)
     RESOLVEFUNC(X509_STORE_set_verify_cb)
+    RESOLVEFUNC(X509_STORE_set_ex_data)
+    RESOLVEFUNC(X509_STORE_get_ex_data)
     RESOLVEFUNC(CRYPTO_free)
     RESOLVEFUNC(OpenSSL_version_num)
     RESOLVEFUNC(OpenSSL_version)
@@ -1046,6 +1053,8 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(CRYPTO_num_locks)
     RESOLVEFUNC(CRYPTO_set_id_callback)
     RESOLVEFUNC(CRYPTO_set_locking_callback)
+    RESOLVEFUNC(CRYPTO_set_ex_data)
+    RESOLVEFUNC(CRYPTO_get_ex_data)
     RESOLVEFUNC(ERR_peek_last_error)
     RESOLVEFUNC(ERR_free_strings)
     RESOLVEFUNC(EVP_CIPHER_CTX_cleanup)
@@ -1305,6 +1314,7 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(X509_STORE_CTX_get_error)
     RESOLVEFUNC(X509_STORE_CTX_get_error_depth)
     RESOLVEFUNC(X509_STORE_CTX_get_current_cert)
+    RESOLVEFUNC(X509_STORE_CTX_get0_store)
     RESOLVEFUNC(X509_cmp)
     RESOLVEFUNC(X509_STORE_CTX_get_ex_data)