Fix bumpy video playback on Windows with FFmpeg using synced textures

We have been facing issues with bumpy video playback when attempting to
keep the decoded textures on the GPU to avoid copying decoded textures
to the CPU. This is seen as stuttering in the video playback, and it
looks like we are switching between new and old frames during playback.

A root cause for this can be that we used texture sharing without
properly synchronizing access to it. This patch fixes this issue by
using synchronized surface sharing as recommended by Microsoft
documentation. This patch also reduces the number of per-frame texture
allocations.

This way we get flicker free rendering during video playback.

Task-number: QTBUG-111815
Task-number: QTBUG-115308
Task-number: QTBUG-111459
Task-number: QTBUG-109213
Pick-to: 6.5
Change-Id: I6ed34051f138fc845d60b4208fdbef2ae7d68189
Reviewed-by: Artem Dyomin <artem.dyomin@qt.io>
(cherry picked from commit 3363496a7f0a0690bc3a2e50172cf6563f198701)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
(cherry picked from commit a3bf450b212625d1626bbd549b3681fc8ef60a37)

2 files changed, 241 insertions, 69 deletions

diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp
index 4e0b47a45..46605cbe8 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp
@@ -16,13 +16,157 @@
 #include <qloggingcategory.h>
 
 #include <libavutil/hwcontext_d3d11va.h>
+#include <d3d11_1.h>
+#include <dxgi1_2.h>
+#include <private/quniquehandle_p.h>
+
 
 QT_BEGIN_NAMESPACE
 
-static Q_LOGGING_CATEGORY(qLcMediaFFmpegHWAccel, "qt.multimedia.hwaccel")
+static Q_LOGGING_CATEGORY(qLcMediaFFmpegHWAccel, "qt.multimedia.hwaccel");
+
+namespace {
+
+ComPtr<ID3D11Device1> GetD3DDevice(QRhi *rhi)
+{
+    const auto native = static_cast<const QRhiD3D11NativeHandles *>(rhi->nativeHandles());
+    if (!native)
+        return {};
+
+    const ComPtr<ID3D11Device> rhiDevice = static_cast<ID3D11Device *>(native->dev);
 
+    ComPtr<ID3D11Device1> dev1;
+    if (rhiDevice.As(&dev1) != S_OK)
+        return nullptr;
+
+    return dev1;
+}
+
+}
 namespace QFFmpeg {
 
+
+bool TextureBridge::copyToSharedTex(ID3D11Device *dev, ID3D11DeviceContext *ctx,
+                                    const ComPtr<ID3D11Texture2D> &tex, UINT index)
+{
+    if (!ensureSrcTex(dev, tex))
+        return false;
+
+    // Flush to ensure that texture is fully updated before we share it.
+    ctx->Flush();
+
+    if (m_srcMutex->AcquireSync(m_srcKey, INFINITE) != S_OK)
+        return false;
+
+    ctx->CopySubresourceRegion(m_srcTex.Get(), 0, 0, 0, 0, tex.Get(), index, nullptr);
+
+    m_srcMutex->ReleaseSync(m_destKey);
+    return true;
+}
+
+ComPtr<ID3D11Texture2D> TextureBridge::copyFromSharedTex(const ComPtr<ID3D11Device1> &dev,
+                                                         const ComPtr<ID3D11DeviceContext> &ctx)
+{
+    if (!ensureDestTex(dev))
+        return {};
+
+    if (m_destMutex->AcquireSync(m_destKey, INFINITE) != S_OK)
+        return {};
+
+    ctx->CopySubresourceRegion(m_outputTex.Get(), 0, 0, 0, 0, m_destTex.Get(), 0, nullptr);
+
+    m_destMutex->ReleaseSync(m_srcKey);
+
+    return m_outputTex;
+}
+
+bool TextureBridge::ensureDestTex(const ComPtr<ID3D11Device1> &dev)
+{
+    if (m_destTex)
+        return true;
+
+    if (dev->OpenSharedResource1(m_sharedHandle.get(), IID_PPV_ARGS(&m_destTex)) != S_OK)
+        return false;
+
+    CD3D11_TEXTURE2D_DESC desc{};
+    m_destTex->GetDesc(&desc);
+
+    desc.MiscFlags = 0;
+    desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
+
+    if (dev->CreateTexture2D(&desc, nullptr, m_outputTex.ReleaseAndGetAddressOf()) != S_OK)
+        return false;
+
+    if (m_destTex.As(&m_destMutex) != S_OK)
+        return false;
+
+    return true;
+}
+
+bool TextureBridge::ensureSrcTex(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex)
+{
+    if (!isSrcInitialized(dev, tex))
+        return recreateSrc(dev, tex);
+
+    return true;
+}
+
+bool TextureBridge::isSrcInitialized(const ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex)
+{
+    if (!m_srcTex)
+        return false;
+
+    // Check if device has changed
+    ComPtr<ID3D11Device> texDevice;
+    tex->GetDevice(texDevice.GetAddressOf());
+    if (dev != texDevice.Get())
+        return false;
+
+    // Check if shared texture has correct size and format
+    CD3D11_TEXTURE2D_DESC inputDesc{};
+    tex->GetDesc(&inputDesc);
+
+    CD3D11_TEXTURE2D_DESC currentDesc{};
+    m_srcTex->GetDesc(&currentDesc);
+
+    if (inputDesc.Format != currentDesc.Format)
+        return false;
+
+    if (inputDesc.Width != currentDesc.Width || inputDesc.Height != currentDesc.Height)
+        return false;
+
+    return true;
+}
+
+bool TextureBridge::recreateSrc(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex)
+{
+    CD3D11_TEXTURE2D_DESC desc{};
+    tex->GetDesc(&desc);
+
+    CD3D11_TEXTURE2D_DESC texDesc{ desc.Format, desc.Width, desc.Height };
+    texDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX | D3D11_RESOURCE_MISC_SHARED_NTHANDLE;
+
+    if (dev->CreateTexture2D(&texDesc, nullptr, m_srcTex.ReleaseAndGetAddressOf()) != S_OK)
+        return false;
+
+    ComPtr<IDXGIResource1> res;
+    if (m_srcTex.As(&res) != S_OK)
+        return false;
+
+    const HRESULT hr =
+            res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ, nullptr, &m_sharedHandle);
+
+    if (hr != S_OK || !m_sharedHandle)
+        return false;
+
+    if (m_srcTex.As(&m_srcMutex) != S_OK || !m_srcMutex)
+        return false;
+
+    m_destTex = nullptr;
+    m_destMutex = nullptr;
+    return true;
+}
+
 class D3D11TextureSet : public TextureSet
 {
 public:
@@ -39,91 +183,59 @@ private:
     ComPtr<ID3D11Texture2D> m_tex;
 };
 
-
 D3D11TextureConverter::D3D11TextureConverter(QRhi *rhi)
-    : TextureConverterBackend(rhi)
-{
-}
-
-static ComPtr<ID3D11Texture2D> getSharedTextureForDevice(ID3D11Device *dev, ID3D11Texture2D *tex)
-{
-    ComPtr<IDXGIResource> dxgiResource;
-    HRESULT hr = tex->QueryInterface(__uuidof(IDXGIResource), reinterpret_cast<void **>(dxgiResource.GetAddressOf()));
-    if (FAILED(hr)) {
-        qCDebug(qLcMediaFFmpegHWAccel) << "Failed to obtain resource handle from FFMpeg texture" << hr;
-        return {};
-    }
-    HANDLE shared = nullptr;
-    hr = dxgiResource->GetSharedHandle(&shared);
-    if (FAILED(hr)) {
-        qCDebug(qLcMediaFFmpegHWAccel) << "Failed to obtain shared handle for FFmpeg texture" << hr;
-        return {};
-    }
-
-    ComPtr<ID3D11Texture2D> sharedTex;
-    hr = dev->OpenSharedResource(shared, __uuidof(ID3D11Texture2D), reinterpret_cast<void **>(sharedTex.GetAddressOf()));
-    if (FAILED(hr))
-        qCDebug(qLcMediaFFmpegHWAccel) << "Failed to share FFmpeg texture" << hr;
-    return sharedTex;
-}
-
-static ComPtr<ID3D11Texture2D> copyTextureFromArray(ID3D11Device *dev, ID3D11Texture2D *array, int index)
+    : TextureConverterBackend(rhi), m_rhiDevice{ GetD3DDevice(rhi) }
 {
-    D3D11_TEXTURE2D_DESC arrayDesc = {};
-    array->GetDesc(&arrayDesc);
-
-    D3D11_TEXTURE2D_DESC texDesc = {};
-    texDesc.Width = arrayDesc.Width;
-    texDesc.Height = arrayDesc.Height;
-    texDesc.Format = arrayDesc.Format;
-    texDesc.ArraySize = 1;
-    texDesc.MipLevels = 1;
-    texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
-    texDesc.MiscFlags = 0;
-    texDesc.SampleDesc = { 1, 0};
-
-    ComPtr<ID3D11Texture2D> texCopy;
-    HRESULT hr = dev->CreateTexture2D(&texDesc, nullptr, texCopy.GetAddressOf());
-    if (FAILED(hr)) {
-        qCDebug(qLcMediaFFmpegHWAccel) << "Failed to create texture" << hr;
-        return {};
-    }
-
-    ComPtr<ID3D11DeviceContext> ctx;
-    dev->GetImmediateContext(ctx.GetAddressOf());
-    ctx->CopySubresourceRegion(texCopy.Get(), 0, 0, 0, 0, array, index, nullptr);
+    if (!m_rhiDevice)
+        return;
 
-    return texCopy;
+    m_rhiDevice->GetImmediateContext(m_rhiCtx.GetAddressOf());
 }
 
 TextureSet *D3D11TextureConverter::getTextures(AVFrame *frame)
 {
-    if (!frame || !frame->hw_frames_ctx || frame->format != AV_PIX_FMT_D3D11)
+    if (!m_rhiDevice)
         return nullptr;
 
-    auto *fCtx = (AVHWFramesContext *)frame->hw_frames_ctx->data;
-    auto *ctx = fCtx->device_ctx;
-    if (!ctx || ctx->type != AV_HWDEVICE_TYPE_D3D11VA)
+    if (!frame || !frame->hw_frames_ctx || frame->format != AV_PIX_FMT_D3D11)
         return nullptr;
 
-    auto nh = static_cast<const QRhiD3D11NativeHandles *>(rhi->nativeHandles());
-    if (!nh)
+    const auto *fCtx = reinterpret_cast<AVHWFramesContext*>(frame->hw_frames_ctx->data);
+    const auto *ctx = fCtx->device_ctx;
+
+    if (!ctx || ctx->type != AV_HWDEVICE_TYPE_D3D11VA)
         return nullptr;
 
-    auto ffmpegTex = (ID3D11Texture2D *)frame->data[0];
-    int index = (intptr_t)frame->data[1];
+    const ComPtr<ID3D11Texture2D> ffmpegTex = reinterpret_cast<ID3D11Texture2D*>(frame->data[0]);
+    const int index = static_cast<int>(reinterpret_cast<intptr_t>(frame->data[1]));
 
     if (rhi->backend() == QRhi::D3D11) {
-        auto dev = reinterpret_cast<ID3D11Device *>(nh->dev);
-        if (!dev)
-            return nullptr;
-        auto sharedTex = getSharedTextureForDevice(dev, ffmpegTex);
-        if (sharedTex) {
-            auto tex = copyTextureFromArray(dev, sharedTex.Get(), index);
-            if (tex) {
-                return new D3D11TextureSet(std::move(tex));
+        {
+            const auto *avDeviceCtx = static_cast<AVD3D11VADeviceContext *>(ctx->hwctx);
+
+            if (!avDeviceCtx)
+                return nullptr;
+
+            // Lock the FFmpeg device context while we copy from FFmpeg's
+            // frame pool into a shared texture because the underlying ID3D11DeviceContext
+            // is not thread safe.
+            avDeviceCtx->lock(avDeviceCtx->lock_ctx);
+            QScopeGuard autoUnlock([&] { avDeviceCtx->unlock(avDeviceCtx->lock_ctx); });
+
+            // Populate the shared texture with one slice from the frame pool
+            if (!m_bridge.copyToSharedTex(avDeviceCtx->device, avDeviceCtx->device_context,
+                                          ffmpegTex, index)) {
+                return nullptr;
             }
         }
+
+        // Get a copy of the texture on the RHI device
+        ComPtr<ID3D11Texture2D> output = m_bridge.copyFromSharedTex(m_rhiDevice, m_rhiCtx);
+
+        if (!output)
+            return nullptr;
+
+        return new D3D11TextureSet(std::move(output));
     }
 
     return nullptr;
diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h
index 2e9c77f5b..4d6ddb926 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h
@@ -15,6 +15,12 @@
 //
 
 #include "qffmpeghwaccel_p.h"
+#include <private/quniquehandle_p.h>
+#include <private/qcomptr_p.h>
+#include <qt_windows.h>
+
+#include <d3d11.h>
+#include <d3d11_1.h>
 
 #if QT_CONFIG(wmf)
 
@@ -24,6 +30,55 @@ class QRhi;
 
 namespace QFFmpeg {
 
+struct SharedTextureHandleTraits
+{
+    using Type = HANDLE;
+    static Type invalidValue() { return nullptr; }
+    static bool close(Type handle) { return CloseHandle(handle) != 0; }
+};
+
+using SharedTextureHandle = QUniqueHandle<SharedTextureHandleTraits>;
+
+/*! \internal Utility class for synchronized transfer of a texture between two D3D devices
+ *
+ * This class is used to copy a texture from one device to another device. This
+ * is implemented using a shared texture, along with keyed mutexes to synchronize
+ * access to the texture.
+ *
+ * This is needed because we need to copy data from FFmpeg to RHI. FFmpeg and RHI
+ * uses different D3D devices.
+ */
+class TextureBridge final
+{
+public:
+    /** Copy a texture slice at position 'index' belonging to device 'dev'
+     * into a shared texture */
+    bool copyToSharedTex(ID3D11Device *dev, ID3D11DeviceContext *ctx,
+                         const ComPtr<ID3D11Texture2D> &tex, UINT index);
+
+    /** Obtain a copy of the texture on a second device 'dev' */
+    ComPtr<ID3D11Texture2D> copyFromSharedTex(const ComPtr<ID3D11Device1> &dev,
+                                              const ComPtr<ID3D11DeviceContext> &ctx);
+
+private:
+    bool ensureDestTex(const ComPtr<ID3D11Device1> &dev);
+    bool ensureSrcTex(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex);
+    bool isSrcInitialized(const ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex);
+    bool recreateSrc(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex);
+
+    SharedTextureHandle m_sharedHandle{};
+
+    const UINT m_srcKey = 0;
+    ComPtr<ID3D11Texture2D> m_srcTex;
+    ComPtr<IDXGIKeyedMutex> m_srcMutex;
+
+    const UINT m_destKey = 1;
+    ComPtr<ID3D11Texture2D> m_destTex;
+    ComPtr<IDXGIKeyedMutex> m_destMutex;
+
+    ComPtr<ID3D11Texture2D> m_outputTex;
+};
+
 class D3D11TextureConverter : public TextureConverterBackend
 {
 public:
@@ -32,6 +87,11 @@ public:
     TextureSet *getTextures(AVFrame *frame) override;
 
     static void SetupDecoderTextures(AVCodecContext *s);
+
+private:
+    ComPtr<ID3D11Device1> m_rhiDevice;
+    ComPtr<ID3D11DeviceContext> m_rhiCtx;
+    TextureBridge m_bridge;
 };
 
 }