From 129b06ba77e451c08778badcd54cbaf193d195bc Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Wed, 29 Jul 2015 13:00:03 +0300 Subject: winrt: Add camera video probe controls [ChangLog][multimedia][winrt] The WinRT backend now supports QVideoProbes on camera objects. Task-number: QTBUG-46228 Change-Id: I7850c5ec6f61e5824064d4be8afc8a0b55d05806 Reviewed-by: Andrew Knight --- src/plugins/winrt/qwinrtcameracontrol.cpp | 1 + src/plugins/winrt/qwinrtcameraservice.cpp | 10 +- .../winrt/qwinrtcameravideorenderercontrol.cpp | 152 +++++++++++++++++++-- .../winrt/qwinrtcameravideorenderercontrol.h | 6 + src/plugins/winrt/qwinrtvideoprobecontrol.cpp | 55 ++++++++ src/plugins/winrt/qwinrtvideoprobecontrol.h | 54 ++++++++ src/plugins/winrt/winrt.pro | 6 +- 7 files changed, 271 insertions(+), 13 deletions(-) create mode 100644 src/plugins/winrt/qwinrtvideoprobecontrol.cpp create mode 100644 src/plugins/winrt/qwinrtvideoprobecontrol.h diff --git a/src/plugins/winrt/qwinrtcameracontrol.cpp b/src/plugins/winrt/qwinrtcameracontrol.cpp index 833054c6e..847afa1e8 100644 --- a/src/plugins/winrt/qwinrtcameracontrol.cpp +++ b/src/plugins/winrt/qwinrtcameracontrol.cpp @@ -630,6 +630,7 @@ void QWinRTCameraControl::setState(QCamera::State state) emit stateChanged(d->state); d->status = QCamera::ActiveStatus; emit statusChanged(d->status); + d->mediaSink->RequestSample(); break; } case QCamera::LoadedState: { diff --git a/src/plugins/winrt/qwinrtcameraservice.cpp b/src/plugins/winrt/qwinrtcameraservice.cpp index a4d292242..977acdcab 100644 --- a/src/plugins/winrt/qwinrtcameraservice.cpp +++ b/src/plugins/winrt/qwinrtcameraservice.cpp @@ -37,6 +37,8 @@ #include "qwinrtcameraservice.h" #include "qwinrtcameracontrol.h" #include "qwinrtcamerainfocontrol.h" +#include "qwinrtvideoprobecontrol.h" +#include "qwinrtcameravideorenderercontrol.h" #include #include @@ -47,6 +49,7 @@ #include #include #include +#include QT_BEGIN_NAMESPACE @@ -98,12 +101,17 @@ QMediaControl *QWinRTCameraService::requestControl(const char *name) if (qstrcmp(name, QCameraLocksControl_iid) == 0) return d->cameraControl->cameraLocksControl(); + if (qstrcmp(name, QMediaVideoProbeControl_iid) == 0) + return new QWinRTVideoProbeControl(qobject_cast(d->cameraControl->videoRenderer())); + return nullptr; } void QWinRTCameraService::releaseControl(QMediaControl *control) { - Q_UNUSED(control); + Q_ASSERT(control); + if (QWinRTVideoProbeControl *videoProbe = qobject_cast(control)) + videoProbe->deleteLater(); } QT_END_NAMESPACE diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp index 4878c55c9..6c5575a17 100644 --- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp +++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,60 @@ using namespace Microsoft::WRL; QT_BEGIN_NAMESPACE +class QWinRTCameraVideoBuffer : public QAbstractVideoBuffer +{ +public: + QWinRTCameraVideoBuffer(IMF2DBuffer *buffer, int size) + : QAbstractVideoBuffer(NoHandle) + , currentMode(NotMapped) + , buffer(buffer) + , size(size) + { + } + + ~QWinRTCameraVideoBuffer() + { + unmap(); + } + + MapMode mapMode() const Q_DECL_OVERRIDE + { + return currentMode; + } + + uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) Q_DECL_OVERRIDE + { + if (currentMode != NotMapped || mode == NotMapped) + return nullptr; + + BYTE *bytes; + LONG stride; + HRESULT hr = buffer->Lock2D(&bytes, &stride); + RETURN_IF_FAILED("Failed to lock camera frame buffer", nullptr); + + if (bytesPerLine) + *bytesPerLine = stride; + if (numBytes) + *numBytes = size; + currentMode = mode; + return bytes; + } + + void unmap() Q_DECL_OVERRIDE + { + if (currentMode == NotMapped) + return; + HRESULT hr = buffer->Unlock2D(); + RETURN_VOID_IF_FAILED("Failed to unlock camera frame buffer"); + currentMode = NotMapped; + } + +private: + ComPtr buffer; + MapMode currentMode; + int size; +}; + class D3DVideoBlitter { public: @@ -143,11 +198,52 @@ public: ComPtr buffers[CAMERA_SAMPLE_QUEUE_SIZE]; QAtomicInteger writeIndex; QAtomicInteger readIndex; + QVideoFrame::PixelFormat cameraSampleformat; + int cameraSampleSize; + uint videoProbesCounter; + bool getCameraSampleInfo(const ComPtr &buffer); + ComPtr dequeueBuffer(); }; +bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr &buffer) +{ + ComPtr sourceTexture; + ComPtr dxgiBuffer; + HRESULT hr = buffer.As(&dxgiBuffer); + Q_ASSERT_SUCCEEDED(hr); + hr = dxgiBuffer->GetResource(IID_PPV_ARGS(&sourceTexture)); + if (FAILED(hr)) { + qErrnoWarning(hr, "The video frame does not support texture output"); + cameraSampleformat = QVideoFrame::Format_Invalid; + return false; + } + D3D11_TEXTURE2D_DESC desc; + sourceTexture->GetDesc(&desc); + switch (desc.Format) { + case DXGI_FORMAT_R8G8B8A8_TYPELESS: + cameraSampleformat = QVideoFrame::Format_ARGB32; + break; + case DXGI_FORMAT_NV12: + cameraSampleformat = QVideoFrame::Format_NV12; + break; + default: + cameraSampleformat = QVideoFrame::Format_Invalid; + qErrnoWarning("Unsupported camera probe format."); + return false; + } + DWORD pcbLength; + hr = buffer->GetContiguousLength(&pcbLength); + Q_ASSERT_SUCCEEDED(hr); + cameraSampleSize = pcbLength; + return true; +} + QWinRTCameraVideoRendererControl::QWinRTCameraVideoRendererControl(const QSize &size, QObject *parent) : QWinRTAbstractVideoRendererControl(size, parent), d_ptr(new QWinRTCameraVideoRendererControlPrivate) { + Q_D(QWinRTCameraVideoRendererControl); + d->cameraSampleformat = QVideoFrame::Format_User; + d->videoProbesCounter = 0; } QWinRTCameraVideoRendererControl::~QWinRTCameraVideoRendererControl() @@ -158,22 +254,15 @@ QWinRTCameraVideoRendererControl::~QWinRTCameraVideoRendererControl() bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target) { Q_D(QWinRTCameraVideoRendererControl); - - const quint16 readIndex = d->readIndex; - if (readIndex == d->writeIndex) { + ComPtr buffer = d->dequeueBuffer(); + if (!buffer) { emit bufferRequested(); return false; } - HRESULT hr; - ComPtr buffer = d->buffers[readIndex]; - Q_ASSERT(buffer); - d->buffers[readIndex].Reset(); - d->readIndex = (readIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE; - ComPtr sourceTexture; ComPtr dxgiBuffer; - hr = buffer.As(&dxgiBuffer); + HRESULT hr = buffer.As(&dxgiBuffer); Q_ASSERT_SUCCEEDED(hr); hr = dxgiBuffer->GetResource(IID_PPV_ARGS(&sourceTexture)); if (FAILED(hr)) { @@ -196,11 +285,41 @@ void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer) { Q_D(QWinRTCameraVideoRendererControl); Q_ASSERT(buffer); + + if (d->videoProbesCounter > 0) { + if (d->cameraSampleformat == QVideoFrame::Format_User) + d->getCameraSampleInfo(buffer); + + if (d->cameraSampleformat != QVideoFrame::Format_Invalid) { + QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize); + QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat); + emit videoFrameProbed(frame); + } + } + const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE; if (d->readIndex == writeIndex) // Drop new sample if queue is full return; d->buffers[d->writeIndex] = buffer; d->writeIndex = writeIndex; + + if (!surface()) { + d->dequeueBuffer(); + emit bufferRequested(); + } +} + +ComPtr QWinRTCameraVideoRendererControlPrivate::dequeueBuffer() +{ + const quint16 currentReadIndex = readIndex; + if (currentReadIndex == writeIndex) + return nullptr; + + ComPtr buffer = buffers[currentReadIndex]; + Q_ASSERT(buffer); + buffers[currentReadIndex].Reset(); + readIndex = (currentReadIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE; + return buffer; } void QWinRTCameraVideoRendererControl::discardBuffers() @@ -211,4 +330,17 @@ void QWinRTCameraVideoRendererControl::discardBuffers() buffer.Reset(); } +void QWinRTCameraVideoRendererControl::incrementProbe() +{ + Q_D(QWinRTCameraVideoRendererControl); + ++d->videoProbesCounter; +} + +void QWinRTCameraVideoRendererControl::decrementProbe() +{ + Q_D(QWinRTCameraVideoRendererControl); + Q_ASSERT(d->videoProbesCounter > 0); + --d->videoProbesCounter; +} + QT_END_NAMESPACE diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.h b/src/plugins/winrt/qwinrtcameravideorenderercontrol.h index 68e9bd9f8..122418de3 100644 --- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.h +++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.h @@ -39,10 +39,13 @@ #include "qwinrtabstractvideorenderercontrol.h" +#include + struct IMF2DBuffer; QT_BEGIN_NAMESPACE +class QWinRTVideoProbeControl; class QVideoSurfaceFormat; class QWinRTCameraVideoRendererControlPrivate; class QWinRTCameraVideoRendererControl : public QWinRTAbstractVideoRendererControl @@ -55,9 +58,12 @@ public: bool render(ID3D11Texture2D *texture) Q_DECL_OVERRIDE; void queueBuffer(IMF2DBuffer *buffer); void discardBuffers(); + void incrementProbe(); + void decrementProbe(); signals: void bufferRequested(); + void videoFrameProbed(const QVideoFrame &frame); private: QScopedPointer d_ptr; diff --git a/src/plugins/winrt/qwinrtvideoprobecontrol.cpp b/src/plugins/winrt/qwinrtvideoprobecontrol.cpp new file mode 100644 index 000000000..7242efe60 --- /dev/null +++ b/src/plugins/winrt/qwinrtvideoprobecontrol.cpp @@ -0,0 +1,55 @@ +/**************************************************************************** +** +** Copyright (C) 2015 The Qt Company Ltd and/or its subsidiary(-ies). +** Contact: http://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL3$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see http://www.qt.io/terms-conditions. For further +** information use the contact form at http://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPLv3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or later as published by the Free +** Software Foundation and appearing in the file LICENSE.GPL included in +** the packaging of this file. Please review the following information to +** ensure the GNU General Public License version 2.0 requirements will be +** met: http://www.gnu.org/licenses/gpl-2.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#include "qwinrtvideoprobecontrol.h" +#include "qwinrtcameravideorenderercontrol.h" + +QT_BEGIN_NAMESPACE + +QWinRTVideoProbeControl::QWinRTVideoProbeControl(QWinRTCameraVideoRendererControl *parent) + : QMediaVideoProbeControl(parent) +{ + QObject::connect(parent, &QWinRTCameraVideoRendererControl::videoFrameProbed, + this, &QMediaVideoProbeControl::videoFrameProbed, Qt::QueuedConnection); + parent->incrementProbe(); +} + +QWinRTVideoProbeControl::~QWinRTVideoProbeControl() +{ + if (QWinRTCameraVideoRendererControl *renderer = qobject_cast(parent())) + renderer->decrementProbe(); +} + +QT_END_NAMESPACE diff --git a/src/plugins/winrt/qwinrtvideoprobecontrol.h b/src/plugins/winrt/qwinrtvideoprobecontrol.h new file mode 100644 index 000000000..dc9a8392e --- /dev/null +++ b/src/plugins/winrt/qwinrtvideoprobecontrol.h @@ -0,0 +1,54 @@ +/**************************************************************************** +** +** Copyright (C) 2015 The Qt Company Ltd and/or its subsidiary(-ies). +** Contact: http://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL3$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see http://www.qt.io/terms-conditions. For further +** information use the contact form at http://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPLv3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or later as published by the Free +** Software Foundation and appearing in the file LICENSE.GPL included in +** the packaging of this file. Please review the following information to +** ensure the GNU General Public License version 2.0 requirements will be +** met: http://www.gnu.org/licenses/gpl-2.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ +#ifndef QWINRTVIDEOPROBECONTROL_H +#define QWINRTVIDEOPROBECONTROL_H + +#include + +QT_BEGIN_NAMESPACE + +class QWinRTCameraVideoRendererControl; +class QWinRTVideoProbeControl : public QMediaVideoProbeControl +{ + Q_OBJECT +public: + explicit QWinRTVideoProbeControl(QWinRTCameraVideoRendererControl *parent); + ~QWinRTVideoProbeControl(); +}; + +QT_END_NAMESPACE + +#endif // QWINRTVIDEOPROBECONTROL_H diff --git a/src/plugins/winrt/winrt.pro b/src/plugins/winrt/winrt.pro index 3ccce818a..2f87ea8ff 100644 --- a/src/plugins/winrt/winrt.pro +++ b/src/plugins/winrt/winrt.pro @@ -21,7 +21,8 @@ HEADERS += \ qwinrtmediaplayerservice.h \ qwinrtplayerrenderercontrol.h \ qwinrtserviceplugin.h \ - qwinrtvideodeviceselectorcontrol.h + qwinrtvideodeviceselectorcontrol.h \ + qwinrtvideoprobecontrol.h SOURCES += \ qwinrtabstractvideorenderercontrol.cpp \ @@ -37,7 +38,8 @@ SOURCES += \ qwinrtmediaplayerservice.cpp \ qwinrtplayerrenderercontrol.cpp \ qwinrtserviceplugin.cpp \ - qwinrtvideodeviceselectorcontrol.cpp + qwinrtvideodeviceselectorcontrol.cpp \ + qwinrtvideoprobecontrol.cpp OTHER_FILES += \ winrt.json -- cgit v1.2.3