Implement precheck of ffmpeg hw devices and codecs

This is a preparation for including nvenc/nvdec to linux and
maybe windows. The related commit includes nvdec/nvenc to
linux build, windows is to be investigated.
codereview.qt-project.org/c/qt/qt5/+/496374

In some cases ffmpeg doesn't filter not available hw accelerated
codecs.
We need our own filter to enshure the best codec selection and
not expose non-working functionality.

Some implementation details:
* Hw devices are tested on the first initialization.
* Ffmpeg logging should be disabled on the testing stage in order
  not to spam with errors.
* Codecs for invalid hw devices are filtered.

Task-number: QTBUG-115471
Change-Id: I0992f9d4ff37b59a5625ffea4bd208455fc9f1ac
Reviewed-by: Lars Knoll <lars@knoll.priv.no>
(cherry picked from commit 1e4b642fa5ba9250803092b855024ed5bb335072)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

4 files changed, 168 insertions, 60 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpeg.cpp b/src/plugins/multimedia/ffmpeg/qffmpeg.cpp
index eb16bf431..8c0327f06 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeg.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeg.cpp
@@ -167,27 +167,30 @@ static void dumpCodecInfo(const AVCodec *codec)
     }
 }
 
-static bool isCodecValid(const AVCodec *encoder,
+static bool isCodecValid(const AVCodec *codec,
                          const std::vector<AVHWDeviceType> &availableHwDeviceTypes)
 {
-    if (encoder->type != AVMEDIA_TYPE_VIDEO)
+    if (codec->type != AVMEDIA_TYPE_VIDEO)
         return true;
 
-    if (!encoder->pix_fmts)
+    const auto pixFmts = codec->pix_fmts;
+
+    if (!pixFmts)
         return true; // To be investigated. This happens for RAW_VIDEO, that is supposed to be OK,
-                     // and with v4l2m2m codec, that is suspicious.
+                     // and with v4l2m2m codecs, that is suspicious.
+
+    if (findAVFormat(pixFmts, &isHwPixelFormat) == AV_PIX_FMT_NONE)
+        return true;
 
-    auto checkFormat = [&](AVPixelFormat pixelFormat) {
-        if (isSwPixelFormat(pixelFormat))
-            return true; // If a codec supports sw pixel formats, it can be used without hw accel
+    if ((codec->capabilities & AV_CODEC_CAP_HARDWARE) == 0)
+        return true;
 
-        return std::any_of(availableHwDeviceTypes.begin(), availableHwDeviceTypes.end(),
-                           [&pixelFormat](AVHWDeviceType type) {
-                               return pixelFormatForHwDevice(type) == pixelFormat;
-                           });
+    auto checkDeviceType = [pixFmts](AVHWDeviceType type) {
+        return hasAVFormat(pixFmts, pixelFormatForHwDevice(type));
     };
 
-    return findAVFormat(encoder->pix_fmts, checkFormat) != AV_PIX_FMT_NONE;
+    return std::any_of(availableHwDeviceTypes.begin(), availableHwDeviceTypes.end(),
+                       checkDeviceType);
 }
 
 const CodecsStorage &codecsStorage(CodecStorageType codecsType)
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
index 90da93f46..a2230ca03 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp
@@ -12,22 +12,27 @@
 #endif
 #if QT_CONFIG(wmf)
 #include "qffmpeghwaccel_d3d11_p.h"
+#include <QtCore/private/qsystemlibrary_p.h>
+
 #endif
 #ifdef Q_OS_ANDROID
 #    include "qffmpeghwaccel_mediacodec_p.h"
 #endif
 #include "qffmpeg_p.h"
 #include "qffmpegvideobuffer_p.h"
+#include "qscopedvaluerollback.h"
+#include "QtCore/qfile.h"
 
 #include <rhi/qrhi.h>
 #include <qloggingcategory.h>
-#include <set>
+#include <unordered_set>
 
 /* Infrastructure for HW acceleration goes into this file. */
 
 QT_BEGIN_NAMESPACE
 
 static Q_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel");
+extern bool thread_local FFmpegLogsEnabledInThread;
 
 namespace QFFmpeg {
 
@@ -36,8 +41,11 @@ static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators
     AV_HWDEVICE_TYPE_MEDIACODEC,
 #elif defined(Q_OS_LINUX)
     AV_HWDEVICE_TYPE_VAAPI,
-    AV_HWDEVICE_TYPE_VDPAU,
     AV_HWDEVICE_TYPE_CUDA,
+    // TODO: investigate VDPAU advantages.
+    // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU
+    // if it's included into the ffmpeg build and vdpau drivers are installed.
+    // AV_HWDEVICE_TYPE_VDPAU
 #elif defined (Q_OS_WIN)
     AV_HWDEVICE_TYPE_D3D11VA,
 #elif defined (Q_OS_DARWIN)
@@ -45,52 +53,134 @@ static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators
 #endif
 };
 
-static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
+static AVBufferUPtr loadHWContext(AVHWDeviceType type)
 {
-    std::vector<AVHWDeviceType> result;
+    AVBufferRef *hwContext = nullptr;
+    qCDebug(qLHWAccel) << "    Checking HW context:" << av_hwdevice_get_type_name(type);
+    int ret = av_hwdevice_ctx_create(&hwContext, type, nullptr, nullptr, 0);
 
-    const auto definedDeviceTypes = qgetenv(envVarName);
-    if (!definedDeviceTypes.isNull()) {
-        const auto definedDeviceTypesString = QString::fromUtf8(definedDeviceTypes).toLower();
-        for (const auto &deviceType : definedDeviceTypesString.split(',')) {
-            if (!deviceType.isEmpty()) {
-                const auto foundType = av_hwdevice_find_type_by_name(deviceType.toUtf8().data());
-                if (foundType == AV_HWDEVICE_TYPE_NONE)
-                    qWarning() << "Unknown hw device type" << deviceType;
-                else
-                    result.emplace_back(foundType);
-            }
+    if (ret == 0) {
+        qCDebug(qLHWAccel) << "    Using above hw context.";
+        return AVBufferUPtr(hwContext);
+    }
+    qCDebug(qLHWAccel) << "    Could not create hw context:" << ret << strerror(-ret);
+    return nullptr;
+}
+
+// FFmpeg might crash on loading non-existing hw devices.
+// Let's roughly precheck drivers/libraries.
+static bool precheckDriver(AVHWDeviceType type)
+{
+    // precheckings might need some improvements
+#if defined(Q_OS_LINUX)
+    if (type == AV_HWDEVICE_TYPE_CUDA)
+        return QFile::exists(QLatin1String("/proc/driver/nvidia/version"));
+#elif defined(Q_OS_WINDOWS)
+    if (type == AV_HWDEVICE_TYPE_D3D11VA)
+        return QSystemLibrary(QLatin1String("d3d11.dll")).load();
+
+    if (type == AV_HWDEVICE_TYPE_DXVA2)
+        return QSystemLibrary(QLatin1String("d3d9.dll")).load();
+
+    // TODO: check nvenc/nvdec and revisit the checking
+    if (type == AV_HWDEVICE_TYPE_CUDA)
+        return QSystemLibrary(QLatin1String("nvml.dll")).load();
+#else
+     Q_UNUSED(type);
+#endif
+
+    return true;
+}
+
+static bool checkHwType(AVHWDeviceType type)
+{
+    const auto deviceName = av_hwdevice_get_type_name(type);
+    if (!deviceName) {
+        qWarning() << "Internal ffmpeg error, unknow hw type:" << type;
+        return false;
+    }
+
+    if (!precheckDriver(type)) {
+        qCDebug(qLHWAccel) << "Drivers for hw device" << deviceName << "is not installed";
+        return false;
+    }
+
+    if (type == AV_HWDEVICE_TYPE_MEDIACODEC ||
+        type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
+        type == AV_HWDEVICE_TYPE_D3D11VA ||
+        type == AV_HWDEVICE_TYPE_DXVA2)
+        return true; // Don't waste time; it's expected to work fine of the precheck is OK
+
+
+    QScopedValueRollback rollback(FFmpegLogsEnabledInThread);
+    FFmpegLogsEnabledInThread = false;
+
+    return loadHWContext(type) != nullptr;
+}
+
+static const std::vector<AVHWDeviceType> &deviceTypes()
+{
+    static const auto types = []() {
+        qCDebug(qLHWAccel) << "Check device types";
+        QElapsedTimer timer;
+        timer.start();
+
+        // gather hw pix formats
+        std::unordered_set<AVPixelFormat> hwPixFormats;
+        void *opaque = nullptr;
+        while (auto codec = av_codec_iterate(&opaque)) {
+            if (auto pixFmt = codec->pix_fmts)
+                for (; *pixFmt != AV_PIX_FMT_NONE; ++pixFmt)
+                    if (isHwPixelFormat(*pixFmt))
+                        hwPixFormats.insert(*pixFmt);
         }
 
-        return result;
-    } else {
-        std::set<AVHWDeviceType> deviceTypesSet;
+        // create a device types list
+        std::vector<AVHWDeviceType> result;
         AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
         while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
-            deviceTypesSet.insert(type);
+            if (hwPixFormats.count(pixelFormatForHwDevice(type)) && checkHwType(type))
+                result.push_back(type);
+        result.shrink_to_fit();
+
+        // reorder the list accordingly preferredHardwareAccelerators
+        auto it = result.begin();
+        for (const auto preffered : preferredHardwareAccelerators) {
+            auto found = std::find(it, result.end(), preffered);
+            if (found != result.end())
+                std::rotate(it++, found, std::next(found));
+        }
 
-        for (const auto preffered : preferredHardwareAccelerators)
-            if (deviceTypesSet.erase(preffered))
-                result.push_back(preffered);
+        using namespace std::chrono;
+        qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(timer.durationElapsed());
 
-        result.insert(result.end(), deviceTypesSet.begin(), deviceTypesSet.end());
-    }
+        return result;
+    }();
 
-    result.shrink_to_fit();
-    return result;
+    return types;
 }
 
-static AVBufferRef *loadHWContext(const AVHWDeviceType type)
+static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
 {
-    AVBufferRef *hwContext = nullptr;
-    int ret = av_hwdevice_ctx_create(&hwContext, type, nullptr, nullptr, 0);
-    qCDebug(qLHWAccel) << "    Checking HW context:" << av_hwdevice_get_type_name(type);
-    if (ret == 0) {
-        qCDebug(qLHWAccel) << "    Using above hw context.";
-        return hwContext;
+    const auto definedDeviceTypes = qgetenv(envVarName);
+
+    if (definedDeviceTypes.isNull())
+        return deviceTypes();
+
+    std::vector<AVHWDeviceType> result;
+    const auto definedDeviceTypesString = QString::fromUtf8(definedDeviceTypes).toLower();
+    for (const auto &deviceType : definedDeviceTypesString.split(',')) {
+        if (!deviceType.isEmpty()) {
+            const auto foundType = av_hwdevice_find_type_by_name(deviceType.toUtf8().data());
+            if (foundType == AV_HWDEVICE_TYPE_NONE)
+                qWarning() << "Unknown hw device type" << deviceType;
+            else
+                result.emplace_back(foundType);
+        }
     }
-    qCDebug(qLHWAccel) << "    Could not create hw context:" << ret << strerror(-ret);
-    return nullptr;
+
+    result.shrink_to_fit();
+    return result;
 }
 
 template<typename CodecFinder>
@@ -215,8 +305,8 @@ HWAccel::~HWAccel() = default;
 
 std::unique_ptr<HWAccel> HWAccel::create(AVHWDeviceType deviceType)
 {
-    if (auto *ctx = loadHWContext(deviceType))
-        return std::unique_ptr<HWAccel>(new HWAccel(ctx));
+    if (auto ctx = loadHWContext(deviceType))
+        return std::unique_ptr<HWAccel>(new HWAccel(std::move(ctx)));
     else
         return {};
 }
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
index 42ddc139d..bf5b5cc3d 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h
@@ -120,9 +120,7 @@ public:
     static const std::vector<AVHWDeviceType> &decodingDeviceTypes();
 
 private:
-    HWAccel(AVBufferRef *hwDeviceContext, AVBufferRef *hwFrameContext = nullptr)
-        : m_hwDeviceContext(hwDeviceContext), m_hwFramesContext(hwFrameContext)
-    {}
+    HWAccel(AVBufferUPtr hwDeviceContext) : m_hwDeviceContext(std::move(hwDeviceContext)) { }
 };
 
 }
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
index db3e9f82c..adf15d973 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp
@@ -76,9 +76,16 @@ public:
     }
 };
 
+bool thread_local FFmpegLogsEnabledInThread = true;
+static bool UseCustomFFmpegLogger = false;
+
 static void qffmpegLogCallback(void *ptr, int level, const char *fmt, va_list vl)
 {
-    Q_UNUSED(ptr)
+    if (!FFmpegLogsEnabledInThread)
+        return;
+
+    if (!UseCustomFFmpegLogger)
+        return av_log_default_callback(ptr, level, fmt, vl);
 
     // filter logs above the chosen level and AV_LOG_QUIET (negative level)
     if (level < 0 || level > av_log_get_level())
@@ -98,6 +105,16 @@ static void qffmpegLogCallback(void *ptr, int level, const char *fmt, va_list vl
         qCritical() << message;
 }
 
+static void setupFFmpegLogger()
+{
+    if (qEnvironmentVariableIsSet("QT_FFMPEG_DEBUG")) {
+        av_log_set_level(AV_LOG_DEBUG);
+        UseCustomFFmpegLogger = true;
+    }
+
+    av_log_set_callback(&qffmpegLogCallback);
+}
+
 static QPlatformSurfaceCapture *createWindowCaptureByBackend(QString backend) {
     if (backend == QLatin1String("grabwindow"))
         return new QGrabWindowSurfaceCapture(QPlatformSurfaceCapture::WindowSource{});
@@ -121,6 +138,8 @@ static QPlatformSurfaceCapture *createWindowCaptureByBackend(QString backend) {
 
 QFFmpegMediaIntegration::QFFmpegMediaIntegration()
 {
+    setupFFmpegLogger();
+
     m_formatsInfo = new QFFmpegMediaFormatInfo();
 
 #if defined(Q_OS_ANDROID)
@@ -142,15 +161,13 @@ QFFmpegMediaIntegration::QFFmpegMediaIntegration()
     m_capturableWindows = std::make_unique<QWinCapturableWindows>();
 #endif
 
-    if (qEnvironmentVariableIsSet("QT_FFMPEG_DEBUG")) {
-        av_log_set_level(AV_LOG_DEBUG);
-        av_log_set_callback(&qffmpegLogCallback);
-    }
-
 #ifndef QT_NO_DEBUG
     qDebug() << "Available HW decoding frameworks:";
-    AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE;
-    while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE)
+    for (auto type : QFFmpeg::HWAccel::decodingDeviceTypes())
+        qDebug() << "    " << av_hwdevice_get_type_name(type);
+
+    qDebug() << "Available HW encoding frameworks:";
+    for (auto type : QFFmpeg::HWAccel::encodingDeviceTypes())
         qDebug() << "    " << av_hwdevice_get_type_name(type);
 #endif
 }