Workaround the broken 'OPENSSL_config' (Windows)

In old versions of OpenSSL this function can try to access a config file
sometimes resulting in EACCES. While handling ENOENT correctly, OPENSSL_config
unconditionally calls std::exit on EACCES, which is unacceptable, especially if we
have a Qt-app which is not using SSL at all (but, for example, is using QNAM).
To workaround this, we pre-test if this file can be opened and if not and the
last error is ERR_R_SYS_LIB we just skip q_OPENSSL_add_all_algorithms call.

Task-number: QTBUG-43843
Change-Id: I309172d3b5e7847f67a87ba33c406d4751bc60ca
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

3 files changed, 58 insertions, 1 deletions

diff --git a/src/network/ssl/qsslsocket_openssl.cpp b/src/network/ssl/qsslsocket_openssl.cpp
index aca7507d13..28576f1167 100644
--- a/src/network/ssl/qsslsocket_openssl.cpp
+++ b/src/network/ssl/qsslsocket_openssl.cpp
@@ -213,6 +213,48 @@ static unsigned int q_ssl_psk_server_callback(SSL *ssl,
 #endif
 } // extern "C"
 
+static void q_OpenSSL_add_all_algorithms_safe()
+{
+#ifdef Q_OS_WIN
+    // Prior to version 1.0.1m an attempt to call OpenSSL_add_all_algorithms on
+    // Windows could result in 'exit' call from OPENSSL_config (QTBUG-43843).
+    // We can predict this and avoid OPENSSL_add_all_algorithms call.
+    // From OpenSSL docs:
+    // "An application does not need to add algorithms to use them explicitly,
+    // for example by EVP_sha1(). It just needs to add them if it (or any of
+    // the functions it calls) needs to lookup algorithms.
+    // The cipher and digest lookup functions are used in many parts of the
+    // library. If the table is not initialized several functions will
+    // misbehave and complain they cannot find algorithms. This includes the
+    // PEM, PKCS#12, SSL and S/MIME libraries. This is a common query in
+    // the OpenSSL mailing lists."
+    //
+    // Anyway, as a result, we chose not to call this function if it would exit.
+
+    if (q_SSLeay() < 0x100010DFL)
+    {
+        // Now, before we try to call it, check if an attempt to open config file
+        // will result in exit:
+        if (char *confFileName = q_CONF_get1_default_config_file()) {
+            BIO *confFile = q_BIO_new_file(confFileName, "r");
+            const auto lastError = q_ERR_peek_last_error();
+            q_OPENSSL_free(confFileName);
+            if (confFile) {
+                q_BIO_free(confFile);
+            } else {
+                q_ERR_clear_error();
+                if (ERR_GET_REASON(lastError) == ERR_R_SYS_LIB) {
+                    qCWarning(lcSsl, "failed to open openssl.conf file");
+                    return;
+                }
+            }
+        }
+    }
+#endif // Q_OS_WIN
+
+    q_OpenSSL_add_all_algorithms();
+}
+
 QSslSocketBackendPrivate::QSslSocketBackendPrivate()
     : ssl(0),
       readBio(0),
@@ -504,7 +546,7 @@ bool QSslSocketPrivate::ensureLibraryLoaded()
         if (q_SSL_library_init() != 1)
             return false;
         q_SSL_load_error_strings();
-        q_OpenSSL_add_all_algorithms();
+        q_OpenSSL_add_all_algorithms_safe();
 
 #if OPENSSL_VERSION_NUMBER >= 0x10001000L
         if (q_SSLeay() >= 0x10001000L)
diff --git a/src/network/ssl/qsslsocket_openssl_symbols.cpp b/src/network/ssl/qsslsocket_openssl_symbols.cpp
index 2a0d746fde..02364b194b 100644
--- a/src/network/ssl/qsslsocket_openssl_symbols.cpp
+++ b/src/network/ssl/qsslsocket_openssl_symbols.cpp
@@ -144,6 +144,9 @@ DEFINEFUNC(unsigned char *, ASN1_STRING_data, ASN1_STRING *a, a, return 0, retur
 DEFINEFUNC(int, ASN1_STRING_length, ASN1_STRING *a, a, return 0, return)
 DEFINEFUNC2(int, ASN1_STRING_to_UTF8, unsigned char **a, a, ASN1_STRING *b, b, return 0, return);
 DEFINEFUNC4(long, BIO_ctrl, BIO *a, a, int b, b, long c, c, void *d, d, return -1, return)
+DEFINEFUNC2(BIO *, BIO_new_file, const char *filename, filename, const char *mode, mode, return 0, return)
+DEFINEFUNC(void, ERR_clear_error, DUMMYARG, DUMMYARG, return, DUMMYARG)
+DEFINEFUNC(void, OPENSSL_free, void *ptr, ptr, return, DUMMYARG)
 DEFINEFUNC(int, BIO_free, BIO *a, a, return 0, return)
 DEFINEFUNC(BIO *, BIO_new, BIO_METHOD *a, a, return 0, return)
 DEFINEFUNC2(BIO *, BIO_new_mem_buf, void *a, a, int b, b, return 0, return)
@@ -168,6 +171,7 @@ DEFINEFUNC(void, DSA_free, DSA *a, a, return, DUMMYARG)
 DEFINEFUNC3(X509 *, d2i_X509, X509 **a, a, const unsigned char **b, b, long c, c, return 0, return)
 DEFINEFUNC2(char *, ERR_error_string, unsigned long a, a, char *b, b, return 0, return)
 DEFINEFUNC(unsigned long, ERR_get_error, DUMMYARG, DUMMYARG, return 0, return)
+DEFINEFUNC(unsigned long, ERR_peek_last_error, DUMMYARG, DUMMYARG, return 0, return)
 DEFINEFUNC(void, ERR_free_strings, void, DUMMYARG, return, DUMMYARG)
 DEFINEFUNC(void, EVP_CIPHER_CTX_cleanup, EVP_CIPHER_CTX *a, a, return, DUMMYARG)
 DEFINEFUNC(void, EVP_CIPHER_CTX_init, EVP_CIPHER_CTX *a, a, return, DUMMYARG)
@@ -407,6 +411,7 @@ DEFINEFUNC3(DSA *, d2i_DSAPrivateKey, DSA **a, a, unsigned char **b, b, long c,
 DEFINEFUNC3(EC_KEY *, d2i_ECPrivateKey, EC_KEY **a, a, unsigned char **b, b, long c, c, return 0, return)
 #endif
 #endif
+DEFINEFUNC(char *, CONF_get1_default_config_file, DUMMYARG, DUMMYARG, return 0, return)
 DEFINEFUNC(void, OPENSSL_add_all_algorithms_noconf, void, DUMMYARG, return, DUMMYARG)
 DEFINEFUNC(void, OPENSSL_add_all_algorithms_conf, void, DUMMYARG, return, DUMMYARG)
 DEFINEFUNC3(int, SSL_CTX_load_verify_locations, SSL_CTX *ctx, ctx, const char *CAfile, CAfile, const char *CApath, CApath, return 0, return)
@@ -780,6 +785,9 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(ASN1_STRING_length)
     RESOLVEFUNC(ASN1_STRING_to_UTF8)
     RESOLVEFUNC(BIO_ctrl)
+    RESOLVEFUNC(BIO_new_file)
+    RESOLVEFUNC(ERR_clear_error)
+    RESOLVEFUNC(OPENSSL_free)
     RESOLVEFUNC(BIO_free)
     RESOLVEFUNC(BIO_new)
     RESOLVEFUNC(BIO_new_mem_buf)
@@ -803,6 +811,7 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(DSA_free)
     RESOLVEFUNC(ERR_error_string)
     RESOLVEFUNC(ERR_get_error)
+    RESOLVEFUNC(ERR_peek_last_error)
     RESOLVEFUNC(ERR_free_strings)
     RESOLVEFUNC(EVP_CIPHER_CTX_cleanup)
     RESOLVEFUNC(EVP_CIPHER_CTX_init)
@@ -992,6 +1001,7 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(d2i_DSAPrivateKey)
     RESOLVEFUNC(d2i_RSAPrivateKey)
 #endif
+    RESOLVEFUNC(CONF_get1_default_config_file)
     RESOLVEFUNC(OPENSSL_add_all_algorithms_noconf)
     RESOLVEFUNC(OPENSSL_add_all_algorithms_conf)
     RESOLVEFUNC(SSL_CTX_load_verify_locations)
diff --git a/src/network/ssl/qsslsocket_openssl_symbols_p.h b/src/network/ssl/qsslsocket_openssl_symbols_p.h
index b35a895d38..68dc6da811 100644
--- a/src/network/ssl/qsslsocket_openssl_symbols_p.h
+++ b/src/network/ssl/qsslsocket_openssl_symbols_p.h
@@ -221,6 +221,9 @@ unsigned char * q_ASN1_STRING_data(ASN1_STRING *a);
 int q_ASN1_STRING_length(ASN1_STRING *a);
 int q_ASN1_STRING_to_UTF8(unsigned char **a, ASN1_STRING *b);
 long q_BIO_ctrl(BIO *a, int b, long c, void *d);
+BIO *q_BIO_new_file(const char *filename, const char *mode);
+void q_ERR_clear_error();
+void q_OPENSSL_free(void *ptr);
 Q_AUTOTEST_EXPORT int q_BIO_free(BIO *a);
 Q_AUTOTEST_EXPORT BIO *q_BIO_new(BIO_METHOD *a);
 BIO *q_BIO_new_mem_buf(void *a, int b);
@@ -256,6 +259,7 @@ void q_DSA_free(DSA *a);
 X509 *q_d2i_X509(X509 **a, const unsigned char **b, long c);
 char *q_ERR_error_string(unsigned long a, char *b);
 unsigned long q_ERR_get_error();
+unsigned long q_ERR_peek_last_error();
 void q_ERR_free_strings();
 void q_EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a);
 void q_EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a);
@@ -565,6 +569,7 @@ DSA *q_d2i_DSAPrivateKey(DSA **a, unsigned char **pp, long length);
 #define q_EVP_PKEY_assign_DSA(pkey,dsa) q_EVP_PKEY_assign((pkey),EVP_PKEY_DSA,\
                                         (char *)(dsa))
 #define q_OpenSSL_add_all_algorithms() q_OPENSSL_add_all_algorithms_conf()
+char *q_CONF_get1_default_config_file();
 void q_OPENSSL_add_all_algorithms_noconf();
 void q_OPENSSL_add_all_algorithms_conf();
 int q_SSL_CTX_load_verify_locations(SSL_CTX *ctx, const char *CAfile, const char *CApath);