Add support for OpenSSL 1.1

The AM can cope dynamically with either OpenSSL 1.0 or 1.1 at runtime.
(Qt itself in contrast has to be compiled for a specific version).

This means we automatically and transparently will use the OpenSSL version
that Qt was compiled against.

Change-Id: I699bbfea132cefad64504415915657cca0453e4f
Reviewed-by: Dominik Holland <dominik.holland@pelagicore.com>

3 files changed, 38 insertions, 25 deletions

diff --git a/src/crypto-lib/libcryptofunction.cpp b/src/crypto-lib/libcryptofunction.cpp
index 2c11cda5..c5a5a0be 100644
--- a/src/crypto-lib/libcryptofunction.cpp
+++ b/src/crypto-lib/libcryptofunction.cpp
@@ -45,11 +45,9 @@
 
 #include "libcryptofunction.h"
 
-// we want at least openssl 1.0.1
-#define AM_MINIMUM_OPENSSL_VERSION 0x1000100fL
-// we want at most openssl 1.0.255
-#define AM_MAXIMUM_OPENSSL_VERSION 0x100ff00fL
-
+// we want at least openssl 1.0.1 or 1.1.0
+#define AM_MINIMUM_OPENSSL10_VERSION 0x1000100fL
+#define AM_MINIMUM_OPENSSL11_VERSION 0x1010000fL
 
 QT_BEGIN_NAMESPACE_AM
 
@@ -58,8 +56,11 @@ static AM_LIBCRYPTO_FUNCTION(SSLeay, unsigned long(*)(), 0);
 static AM_LIBCRYPTO_FUNCTION(OPENSSL_add_all_algorithms_noconf, void(*)());
 static AM_LIBCRYPTO_FUNCTION(ERR_load_crypto_strings, void(*)());
 
+static AM_LIBCRYPTO_FUNCTION(OpenSSL_version_num, unsigned long(*)(), 0);
+static AM_LIBCRYPTO_FUNCTION(OPENSSL_init_crypto, int(*)(uint64_t, void *), 0);
 
 QLibrary *Cryptography::LibCryptoFunctionBase::s_library = nullptr;
+bool Cryptography::LibCryptoFunctionBase::s_isOpenSSL11 = false;
 
 bool Cryptography::LibCryptoFunctionBase::initialize()
 {
@@ -76,7 +77,8 @@ bool Cryptography::LibCryptoFunctionBase::initialize()
     // Loading libcrypto is a mess, since distros are not creating links for libcrypto.so.1
     // anymore. In order to not duplicate a lot of bad hacks, we simply let QtNetwork do the
     // dirty work.
-    QSslSocket::supportsSsl();
+    if (!QSslSocket::supportsSsl())
+        return false;
 
     s_library = new QLibrary(QString::fromLatin1(libname), 1);
     bool ok = s_library->load();
@@ -85,23 +87,30 @@ bool Cryptography::LibCryptoFunctionBase::initialize()
         ok = s_library->load();
     }
     if (ok) {
-        if (am_SSLeay.functionPointer()) {
-            auto version = am_SSLeay();
-            if (version >= AM_MINIMUM_OPENSSL_VERSION) {
-                if (version <= AM_MAXIMUM_OPENSSL_VERSION) {
-                    am_OPENSSL_add_all_algorithms_noconf();
-                    am_ERR_load_crypto_strings();
-                    return true;
-                } else {
-                    qCritical("Loaded libcrypto (%s), but the version is too new: 0x%08lx (maximum supported version is: 0x%08lx)",
-                              qPrintable(s_library->fileName()), version, AM_MAXIMUM_OPENSSL_VERSION);
-                }
+        unsigned long version = 0;
+
+        if (am_OpenSSL_version_num.functionPointer())
+            version = am_OpenSSL_version_num();  // 1.1
+        else if (am_SSLeay.functionPointer())
+            version = am_SSLeay(); // 1.0
+
+        if (version > 0) {
+            if (version >= AM_MINIMUM_OPENSSL11_VERSION) {
+                s_isOpenSSL11 = true;
+                return (am_OPENSSL_init_crypto(4 /*OPENSSL_INIT_ADD_ALL_CIPHERS*/
+                                               | 8 /*OPENSSL_INIT_ADD_ALL_DIGESTS*/
+                                               | 2 /*OPENSSL_INIT_LOAD_CRYPTO_STRINGS*/,
+                                               nullptr) == 1);
+            } else if (version >= AM_MINIMUM_OPENSSL10_VERSION) {
+                am_OPENSSL_add_all_algorithms_noconf();
+                am_ERR_load_crypto_strings();
+                return true;
             } else {
                 qCritical("Loaded libcrypto (%s), but the version is too old: 0x%08lx (minimum supported version is: 0x%08lx)",
-                          qPrintable(s_library->fileName()), version, AM_MINIMUM_OPENSSL_VERSION);
+                          qPrintable(s_library->fileName()), version, AM_MINIMUM_OPENSSL10_VERSION);
             }
         } else {
-            qCritical("Could not get version information from libcrypto: symbol 'SSLeay' was not found");
+            qCritical("Could not get version information from libcrypto: neither of the symbols 'SSLeay' or 'OpenSSL_version_num' were found");
         }
         s_library->unload();
     } else {
diff --git a/src/crypto-lib/libcryptofunction.h b/src/crypto-lib/libcryptofunction.h
index 6aae0332..d750fb11 100644
--- a/src/crypto-lib/libcryptofunction.h
+++ b/src/crypto-lib/libcryptofunction.h
@@ -80,6 +80,7 @@ class LibCryptoFunctionBase
 {
 public:
     static bool initialize();
+    static inline bool isOpenSSL11() { return s_isOpenSSL11; }
 
 protected:
     LibCryptoFunctionBase(const char *symbol);
@@ -91,6 +92,7 @@ protected:
 
 private:
     static QLibrary *s_library;
+    static bool s_isOpenSSL11;
     bool m_tried = false;
 };
 
diff --git a/src/crypto-lib/signature_openssl.cpp b/src/crypto-lib/signature_openssl.cpp
index e21885f5..e91702d9 100644
--- a/src/crypto-lib/signature_openssl.cpp
+++ b/src/crypto-lib/signature_openssl.cpp
@@ -69,7 +69,8 @@ static AM_LIBCRYPTO_FUNCTION(PKCS7_free, void(*)(PKCS7 *));
 static AM_LIBCRYPTO_FUNCTION(EVP_PKEY_free, void(*)(EVP_PKEY *));
 static AM_LIBCRYPTO_FUNCTION(PKCS12_free, void(*)(PKCS12 *));
 static AM_LIBCRYPTO_FUNCTION(X509_STORE_free, void(*)(X509_STORE *));
-static AM_LIBCRYPTO_FUNCTION(sk_pop_free, void(*)(STACK_OF_X509 *, void(*)(void *)));
+static AM_LIBCRYPTO_FUNCTION(sk_pop_free, void(*)(STACK_OF_X509 *, void(*)(void *)));         // 1.0
+static AM_LIBCRYPTO_FUNCTION(OPENSSL_sk_pop_free, void(*)(STACK_OF_X509 *, void(*)(void *))); // 1.1
 
 // error handling
 static AM_LIBCRYPTO_FUNCTION(ERR_get_error, unsigned long(*)(), 4|64 /*ERR_R_INTERNAL_ERROR*/);
@@ -81,7 +82,6 @@ static AM_LIBCRYPTO_FUNCTION(PKCS12_parse, int (*)(PKCS12 *, const char *, EVP_P
 static AM_LIBCRYPTO_FUNCTION(PKCS7_sign, PKCS7 *(*)(X509 *, EVP_PKEY *, STACK_OF_X509 *, BIO *, int), nullptr);
 static AM_LIBCRYPTO_FUNCTION(BIO_new, BIO *(*)(BIO_METHOD *), nullptr);
 static AM_LIBCRYPTO_FUNCTION(BIO_s_mem, BIO_METHOD *(*)(), nullptr);
-static AM_LIBCRYPTO_FUNCTION(i2d_PKCS7, int(*)(PKCS7 *, unsigned char **out), 0);
 static AM_LIBCRYPTO_FUNCTION(PEM_ASN1_write_bio, int (*)(i2d_of_void *, const char *, BIO *, void *, const EVP_CIPHER *, unsigned char *, int, pem_password_cb *, void *), 0);
 static AM_LIBCRYPTO_FUNCTION(i2d_PKCS7_bio, int (*)(BIO *, PKCS7 *), 0);
 static AM_LIBCRYPTO_FUNCTION(d2i_PKCS7_bio, PKCS7 *(*)(BIO *, PKCS7 **), nullptr);
@@ -89,8 +89,8 @@ static AM_LIBCRYPTO_FUNCTION(d2i_PKCS7_bio, PKCS7 *(*)(BIO *, PKCS7 **), nullptr
 // verify
 static AM_LIBCRYPTO_FUNCTION(BIO_new_mem_buf, BIO *(*)(const void *, int), nullptr);
 static AM_LIBCRYPTO_FUNCTION(PEM_ASN1_read_bio, void *(*)(d2i_of_void *, const char *, BIO *, void **, pem_password_cb *, void *), nullptr);
-static AM_LIBCRYPTO_FUNCTION(d2i_PKCS7, PKCS7 *(*)(PKCS7 **, const unsigned char **, long), nullptr);
-static AM_LIBCRYPTO_FUNCTION(d2i_X509, X509 *(*)(X509 **, const unsigned char **, long), nullptr);
+static AM_LIBCRYPTO_FUNCTION(d2i_X509, X509 *(*)(X509 **, const unsigned char **, long), nullptr);     // 1.0
+static AM_LIBCRYPTO_FUNCTION(d2i_X509_AUX, X509 *(*)(X509 **, const unsigned char **, long), nullptr); // 1.1
 static AM_LIBCRYPTO_FUNCTION(X509_STORE_new, X509_STORE *(*)(), nullptr);
 static AM_LIBCRYPTO_FUNCTION(X509_STORE_add_cert, int (*)(X509_STORE *, X509 *), 0);
 static AM_LIBCRYPTO_FUNCTION(PKCS7_verify, int (*)(PKCS7 *, STACK_OF_X509 *, X509_STORE *, BIO *, BIO *, int), 0);
@@ -109,7 +109,9 @@ struct OpenSslDeleter {
     static inline void cleanup(X509_STORE *x509Store)
     { am_X509_STORE_free(x509Store); }
     static inline void cleanup(STACK_OF_X509 *stackOfX509)
-    { am_sk_pop_free(stackOfX509, (void(*)(void *)) am_X509_free.functionPointer()); }
+    { (Cryptography::LibCryptoFunctionBase::isOpenSSL11() ? am_OPENSSL_sk_pop_free
+                                                          : am_sk_pop_free)
+                (stackOfX509, (void(*)(void *)) am_X509_free.functionPointer()); }
 };
 
 template <typename T> using OpenSslPointer = QScopedPointer<T, OpenSslDeleter>;
@@ -216,7 +218,7 @@ bool SignaturePrivate::verify(const QByteArray &signaturePkcs7,
         // BIO_eof(b) == (int)BIO_ctrl(b,BIO_CTRL_EOF,0,NULL)
         while (!am_BIO_ctrl(bioCert.data(), 2 /*BIO_CTRL_EOF*/, 0, nullptr)) {
             //OpenSslPointer<X509> cert(PEM_read_bio_X509(bioCert.data(), 0, 0, 0));
-            OpenSslPointer<X509> cert((X509 *) am_PEM_ASN1_read_bio((d2i_of_void *) am_d2i_X509.functionPointer(),
+            OpenSslPointer<X509> cert((X509 *) am_PEM_ASN1_read_bio((d2i_of_void *) (Cryptography::LibCryptoFunctionBase::isOpenSSL11() ? am_d2i_X509_AUX : am_d2i_X509).functionPointer(),
                                                                     "CERTIFICATE" /*PEM_STRING_X509*/, bioCert.data(),
                                                                     nullptr, nullptr, nullptr));
             if (!cert)