Fix build if openssl is configured with no-des or no-rc2

A custom build of openssl can disable DES or RC2. This
allows to build Qt against those builds.

Change-Id: I9b91c943fab4d217a791381e81a7d87a9ff5031a
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

3 files changed, 37 insertions, 16 deletions

diff --git a/src/network/ssl/qsslkey_openssl.cpp b/src/network/ssl/qsslkey_openssl.cpp
index aa81b735b9..58df544a0e 100644
--- a/src/network/ssl/qsslkey_openssl.cpp
+++ b/src/network/ssl/qsslkey_openssl.cpp
@@ -193,6 +193,16 @@ QByteArray QSslKeyPrivate::toPem(const QByteArray &passPhrase) const
     if (!QSslSocket::supportsSsl() || isNull || algorithm == QSsl::Opaque)
         return QByteArray();
 
+    // ### the cipher should be selectable in the API:
+    const EVP_CIPHER *cipher = nullptr;
+    if (type == QSsl::PrivateKey && !passPhrase.isEmpty()) {
+#ifndef OPENSSL_NO_DES
+        cipher = q_EVP_des_ede3_cbc();
+#else
+        return QByteArray();
+#endif
+    }
+
     BIO *bio = q_BIO_new(q_BIO_s_mem());
     if (!bio)
         return QByteArray();
@@ -205,9 +215,7 @@ QByteArray QSslKeyPrivate::toPem(const QByteArray &passPhrase) const
                 fail = true;
         } else {
             if (!q_PEM_write_bio_RSAPrivateKey(
-                    bio, rsa,
-                    // ### the cipher should be selectable in the API:
-                    passPhrase.isEmpty() ? (const EVP_CIPHER *)0 : q_EVP_des_ede3_cbc(),
+                    bio, rsa, cipher,
                     const_cast<uchar *>((const uchar *)passPhrase.data()), passPhrase.size(), 0, 0)) {
                 fail = true;
             }
@@ -218,9 +226,7 @@ QByteArray QSslKeyPrivate::toPem(const QByteArray &passPhrase) const
                 fail = true;
         } else {
             if (!q_PEM_write_bio_DSAPrivateKey(
-                    bio, dsa,
-                    // ### the cipher should be selectable in the API:
-                    passPhrase.isEmpty() ? (const EVP_CIPHER *)0 : q_EVP_des_ede3_cbc(),
+                    bio, dsa, cipher,
                     const_cast<uchar *>((const uchar *)passPhrase.data()), passPhrase.size(), 0, 0)) {
                 fail = true;
             }
@@ -232,9 +238,7 @@ QByteArray QSslKeyPrivate::toPem(const QByteArray &passPhrase) const
                 fail = true;
         } else {
             if (!q_PEM_write_bio_ECPrivateKey(
-                    bio, ec,
-                    // ### the cipher should be selectable in the API:
-                    passPhrase.isEmpty() ? (const EVP_CIPHER *)0 : q_EVP_des_ede3_cbc(),
+                    bio, ec, cipher,
                     const_cast<uchar *>((const uchar *)passPhrase.data()), passPhrase.size(), 0, 0)) {
                 fail = true;
             }
@@ -274,34 +278,39 @@ Qt::HANDLE QSslKeyPrivate::handle() const
 
 static QByteArray doCrypt(QSslKeyPrivate::Cipher cipher, const QByteArray &data, const QByteArray &key, const QByteArray &iv, int enc)
 {
-#if QT_CONFIG(opensslv11)
-    EVP_CIPHER_CTX *ctx = q_EVP_CIPHER_CTX_new();
-#else
-    EVP_CIPHER_CTX evpCipherContext;
-    EVP_CIPHER_CTX *ctx = &evpCipherContext;
-#endif
-
     const EVP_CIPHER* type = 0;
     int i = 0, len = 0;
 
     switch (cipher) {
     case QSslKeyPrivate::DesCbc:
+#ifndef OPENSSL_NO_DES
         type = q_EVP_des_cbc();
+#endif
         break;
     case QSslKeyPrivate::DesEde3Cbc:
+#ifndef OPENSSL_NO_DES
         type = q_EVP_des_ede3_cbc();
+#endif
         break;
     case QSslKeyPrivate::Rc2Cbc:
+#ifndef OPENSSL_NO_RC2
         type = q_EVP_rc2_cbc();
+#endif
         break;
     }
 
+    if (type == nullptr)
+        return QByteArray();
+
     QByteArray output;
     output.resize(data.size() + EVP_MAX_BLOCK_LENGTH);
 
 #if QT_CONFIG(opensslv11)
+    EVP_CIPHER_CTX *ctx = q_EVP_CIPHER_CTX_new();
     q_EVP_CIPHER_CTX_reset(ctx);
 #else
+    EVP_CIPHER_CTX evpCipherContext;
+    EVP_CIPHER_CTX *ctx = &evpCipherContext;
     q_EVP_CIPHER_CTX_init(ctx);
 #endif
 
diff --git a/src/network/ssl/qsslsocket_openssl_symbols.cpp b/src/network/ssl/qsslsocket_openssl_symbols.cpp
index 82ff5e9e3a..466eba0bd0 100644
--- a/src/network/ssl/qsslsocket_openssl_symbols.cpp
+++ b/src/network/ssl/qsslsocket_openssl_symbols.cpp
@@ -322,9 +322,13 @@ DEFINEFUNC5(int, EVP_CipherInit, EVP_CIPHER_CTX *ctx, ctx, const EVP_CIPHER *typ
 DEFINEFUNC6(int, EVP_CipherInit_ex, EVP_CIPHER_CTX *ctx, ctx, const EVP_CIPHER *cipher, cipher, ENGINE *impl, impl, const unsigned char *key, key, const unsigned char *iv, iv, int enc, enc, return 0, return)
 DEFINEFUNC5(int, EVP_CipherUpdate, EVP_CIPHER_CTX *ctx, ctx, unsigned char *out, out, int *outl, outl, const unsigned char *in, in, int inl, inl, return 0, return)
 DEFINEFUNC3(int, EVP_CipherFinal, EVP_CIPHER_CTX *ctx, ctx, unsigned char *out, out, int *outl, outl, return 0, return)
+#ifndef OPENSSL_NO_DES
 DEFINEFUNC(const EVP_CIPHER *, EVP_des_cbc, DUMMYARG, DUMMYARG, return 0, return)
 DEFINEFUNC(const EVP_CIPHER *, EVP_des_ede3_cbc, DUMMYARG, DUMMYARG, return 0, return)
+#endif
+#ifndef OPENSSL_NO_RC2
 DEFINEFUNC(const EVP_CIPHER *, EVP_rc2_cbc, DUMMYARG, DUMMYARG, return 0, return)
+#endif
 DEFINEFUNC(const EVP_MD *, EVP_sha1, DUMMYARG, DUMMYARG, return 0, return)
 DEFINEFUNC3(int, EVP_PKEY_assign, EVP_PKEY *a, a, int b, b, char *c, c, return -1, return)
 DEFINEFUNC2(int, EVP_PKEY_set1_RSA, EVP_PKEY *a, a, RSA *b, b, return -1, return)
@@ -1042,9 +1046,13 @@ bool q_resolveOpenSslSymbols()
     RESOLVEFUNC(EVP_CipherInit_ex)
     RESOLVEFUNC(EVP_CipherUpdate)
     RESOLVEFUNC(EVP_CipherFinal)
+#ifndef OPENSSL_NO_DES
     RESOLVEFUNC(EVP_des_cbc)
     RESOLVEFUNC(EVP_des_ede3_cbc)
+#endif
+#ifndef OPENSSL_NO_RC2
     RESOLVEFUNC(EVP_rc2_cbc)
+#endif
     RESOLVEFUNC(EVP_sha1)
     RESOLVEFUNC(EVP_PKEY_assign)
     RESOLVEFUNC(EVP_PKEY_set1_RSA)
diff --git a/src/network/ssl/qsslsocket_openssl_symbols_p.h b/src/network/ssl/qsslsocket_openssl_symbols_p.h
index 4fb8f26cf6..68b519d74e 100644
--- a/src/network/ssl/qsslsocket_openssl_symbols_p.h
+++ b/src/network/ssl/qsslsocket_openssl_symbols_p.h
@@ -266,9 +266,13 @@ int q_EVP_CipherInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, const unsigned
 int q_EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv, int enc);
 int q_EVP_CipherUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl, const unsigned char *in, int inl);
 int q_EVP_CipherFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl);
+#ifndef OPENSSL_NO_DES
 const EVP_CIPHER *q_EVP_des_cbc();
 const EVP_CIPHER *q_EVP_des_ede3_cbc();
+#endif
+#ifndef OPENSSL_NO_RC2
 const EVP_CIPHER *q_EVP_rc2_cbc();
+#endif
 const EVP_MD *q_EVP_sha1();
 int q_EVP_PKEY_assign(EVP_PKEY *a, int b, char *c);
 Q_AUTOTEST_EXPORT int q_EVP_PKEY_set1_RSA(EVP_PKEY *a, RSA *b);