Fix infinite recursion caused by the RDSEED feature

We made qRandomCpu() call qCpuHasFeature(), which needs to test that
RDRND works and does so by calling qRandomCpu(). So disentangle that by
making sure we don't recurse.

And by making sure that we fill the buffer with RDRND data, not with
RDSEED.

Fixes: QTBUG-79162
Change-Id: Ib5d667bf77a740c28d2efffd15cc583a9afcb97a
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

1 files changed, 63 insertions, 40 deletions

diff --git a/src/corelib/tools/qsimd.cpp b/src/corelib/tools/qsimd.cpp
index 6e3b0f9faf..d7c1d8c4a9 100644
--- a/src/corelib/tools/qsimd.cpp
+++ b/src/corelib/tools/qsimd.cpp
@@ -1,7 +1,7 @@
 /****************************************************************************
 **
 ** Copyright (C) 2016 The Qt Company Ltd.
-** Copyright (C) 2018 Intel Corporation.
+** Copyright (C) 2019 Intel Corporation.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtCore module of the Qt Toolkit.
@@ -188,6 +188,8 @@ static inline quint64 detectProcessorFeatures()
 # define PICreg "%%rbx"
 #endif
 
+static bool checkRdrndWorks() noexcept;
+
 static int maxBasicCpuidSupported()
 {
 #if defined(Q_CC_EMSCRIPTEN)
@@ -376,37 +378,8 @@ static quint64 detectProcessorFeatures()
         features &= ~AllAVX512;
     }
 
-#if defined(Q_PROCESSOR_X86) && QT_COMPILER_SUPPORTS_HERE(RDRND)
-    /**
-     * Some AMD CPUs (e.g. AMD A4-6250J and AMD Ryzen 3000-series) have a
-     * failing random generation instruction, which always returns
-     * 0xffffffff, even when generation was "successful".
-     *
-     * This code checks if hardware random generator generates four consecutive
-     * equal numbers. If it does, then we probably have a failing one and
-     * should disable it completely.
-     *
-     * https://bugreports.qt.io/browse/QTBUG-69423
-     */
-    if (features & CpuFeatureRDRND) {
-        const qsizetype testBufferSize = 4;
-        unsigned testBuffer[4] = {};
-
-        const qsizetype generated = qRandomCpu(testBuffer, testBufferSize);
-
-        if (Q_UNLIKELY(generated == testBufferSize &&
-            testBuffer[0] == testBuffer[1] &&
-            testBuffer[1] == testBuffer[2] &&
-            testBuffer[2] == testBuffer[3])) {
-
-            fprintf(stderr, "WARNING: CPU random generator seem to be failing, disable hardware random number generation\n");
-            fprintf(stderr, "WARNING: RDRND generated: 0x%x 0x%x 0x%x 0x%x\n",
-                    testBuffer[0], testBuffer[1], testBuffer[2], testBuffer[3]);
-
-            features &= ~CpuFeatureRDRND;
-        }
-    }
-#endif
+    if (features & CpuFeatureRDRND && !checkRdrndWorks())
+        features &= ~(CpuFeatureRDRND | CpuFeatureRDSEED);
 
     return features;
 }
@@ -662,15 +635,9 @@ static unsigned *qt_random_rdseed(unsigned *ptr, unsigned *)
 }
 #  endif
 
-QT_FUNCTION_TARGET(RDRND) qsizetype qRandomCpu(void *buffer, qsizetype count) noexcept
+static QT_FUNCTION_TARGET(RDRND) unsigned *qt_random_rdrnd(unsigned *ptr, unsigned *end) noexcept
 {
-    unsigned *ptr = reinterpret_cast<unsigned *>(buffer);
-    unsigned *end = ptr + count;
     int retries = 10;
-
-    if (qCpuHasFeature(RDSEED))
-        ptr = qt_random_rdseed(ptr, end);
-
     while (ptr + sizeof(qregisteruint)/sizeof(*ptr) <= end) {
         if (_rdrandXX_step(reinterpret_cast<qregisteruint *>(ptr)))
             ptr += sizeof(qregisteruint)/sizeof(*ptr);
@@ -688,8 +655,64 @@ QT_FUNCTION_TARGET(RDRND) qsizetype qRandomCpu(void *buffer, qsizetype count) no
     }
 
 out:
+    return ptr;
+}
+
+static QT_FUNCTION_TARGET(RDRND) Q_DECL_COLD_FUNCTION bool checkRdrndWorks() noexcept
+{
+    /*
+     * Some AMD CPUs (e.g. AMD A4-6250J and AMD Ryzen 3000-series) have a
+     * failing random generation instruction, which always returns
+     * 0xffffffff, even when generation was "successful".
+     *
+     * This code checks if hardware random generator generates four consecutive
+     * equal numbers. If it does, then we probably have a failing one and
+     * should disable it completely.
+     *
+     * https://bugreports.qt.io/browse/QTBUG-69423
+     */
+    constexpr qsizetype TestBufferSize = 4;
+    unsigned testBuffer[TestBufferSize] = {};
+
+    unsigned *end = qt_random_rdrnd(testBuffer, testBuffer + TestBufferSize);
+    if (end < testBuffer + 3) {
+        // Random generation didn't produce enough data for us to make a
+        // determination whether it's working or not. Assume it isn't, but
+        // don't print a warning.
+        return false;
+    }
+
+    // Check the results for equality
+    if (testBuffer[0] == testBuffer[1]
+        && testBuffer[0] == testBuffer[2]
+        && end == testBuffer + TestBufferSize && testBuffer[0] == testBuffer[3]) {
+        fprintf(stderr, "WARNING: CPU random generator seem to be failing, "
+                        "disabling hardware random number generation\n"
+                        "WARNING: RDRND generated:");
+        for (unsigned *ptr = testBuffer; ptr < end; ++ptr)
+            fprintf(stderr, " 0x%x", *ptr);
+        fprintf(stderr, "\n");
+        return false;
+    }
+
+    // We're good
+    return true;
+}
+
+QT_FUNCTION_TARGET(RDRND) qsizetype qRandomCpu(void *buffer, qsizetype count) noexcept
+{
+    unsigned *ptr = reinterpret_cast<unsigned *>(buffer);
+    unsigned *end = ptr + count;
+
+    if (qCpuHasFeature(RDSEED))
+        ptr = qt_random_rdseed(ptr, end);
+
+    // fill the buffer with RDRND if RDSEED didn't
+    ptr = qt_random_rdrnd(ptr, end);
     return ptr - reinterpret_cast<unsigned *>(buffer);
 }
-#endif
+#elif defined(Q_PROCESSOR_X86) && !defined(Q_OS_NACL) && !defined(Q_PROCESSOR_ARM)
+static bool checkRdrndWorks() noexcept { return false; }
+#endif // Q_PROCESSOR_X86 && RDRND
 
 QT_END_NAMESPACE