diff options
Diffstat (limited to 'src/multimedia/video/qabstractvideofilter.cpp')
| -rw-r--r-- | src/multimedia/video/qabstractvideofilter.cpp | 319 |
1 files changed, 319 insertions, 0 deletions
diff --git a/src/multimedia/video/qabstractvideofilter.cpp b/src/multimedia/video/qabstractvideofilter.cpp new file mode 100644 index 000000000..ccfe9ccdd --- /dev/null +++ b/src/multimedia/video/qabstractvideofilter.cpp @@ -0,0 +1,319 @@ +/**************************************************************************** +** +** Copyright (C) 2015 The Qt Company Ltd. +** Contact: http://www.qt.io/licensing/ +** +** This file is part of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL21$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see http://www.qt.io/terms-conditions. For further +** information use the contact form at http://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 or version 3 as published by the Free +** Software Foundation and appearing in the file LICENSE.LGPLv21 and +** LICENSE.LGPLv3 included in the packaging of this file. Please review the +** following information to ensure the GNU Lesser General Public License +** requirements will be met: https://www.gnu.org/licenses/lgpl.html and +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** As a special exception, The Qt Company gives you certain additional +** rights. These rights are described in The Qt Company LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qabstractvideofilter.h" + +QT_BEGIN_NAMESPACE + +/*! + \class QAbstractVideoFilter + \since 5.5 + \brief The QAbstractVideoFilter class represents a filter that is applied to the video frames + received by a VideoOutput type. + \inmodule QtMultimedia + + \ingroup multimedia + \ingroup multimedia_video + + QAbstractVideoFilter provides a convenient way for applications to run image + processing, computer vision algorithms or any generic transformation or + calculation on the output of a VideoOutput type, regardless of the source + (video or camera). By providing a simple interface it allows applications and + third parties to easily develop QML types that provide image processing + algorithms using popular frameworks like \l{http://opencv.org}{OpenCV}. Due to + the close integration with the final stages of the Qt Multimedia video + pipeline, accelerated and possibly zero-copy solutions are feasible too: for + instance, a plugin providing OpenCL-based algorithms can use OpenCL's OpenGL + interop to use the OpenGL textures created by a hardware accelerated video + decoder, without additional readbacks and copies. + + \note QAbstractVideoFilter is not always the best choice. To apply effects or + transformations using OpenGL shaders to the image shown on screen, the + standard Qt Quick approach of using ShaderEffect items in combination with + VideoOutput should be used. VideoFilter is not a replacement for this. It is + rather targeted for performing computations (that do not necessarily change + the image shown on screen) and computer vision algorithms provided by + external frameworks. + + QAbstractVideoFilter is meant to be subclassed. The subclasses are then registered to + the QML engine, so they can be used as a QML type. The list of filters are + assigned to a VideoOutput type via its \l{QtMultimedia::VideoOutput::filters}{filters} + property. + + A single filter represents one transformation or processing step on + a video frame. The output is a modified video frame, some arbitrary data or + both. For example, image transformations will result in a different image, + whereas an algorithm for detecting objects on an image will likely provide + a list of rectangles. + + Arbitrary data can be represented as properties on the QAbstractVideoFilter subclass + and on the QObject or QJSValue instances passed to its signals. What exactly + these properties and signals are, is up to the individual video + filters. Completion of the operations can be indicated by + signals. Computations that do not result in a modified image will pass the + input image through so that subsequent filters can be placed after them. + + Properties set on QAbstractVideoFilter serve as input to the computation, similarly + to how uniform values are specified in ShaderEffect types. The changed + property values are taken into use when the next video frame is processed. + + The typical usage is to subclass QAbstractVideoFilter and QVideoFilterRunnable: + + \badcode + class MyFilterRunnable : public QVideoFilterRunnable { + public: + QVideoFrame run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags) { ... } + }; + + class MyFilter : public QAbstractVideoFilter { + public: + QVideoFilterRunnable *createFilterRunnable() { return new MyFilterRunnable; } + signals: + void finished(QObject *result); + }; + + int main(int argc, char **argv) { + ... + qmlRegisterType<MyFilter>("my.uri", 1, 0, "MyFilter"); + ... + } + \endcode + + MyFilter is thus accessible from QML: + + \badcode + import my.uri 1.0 + + Camera { + id: camera + } + MyFilter { + id: filter + // set properties, they can also be animated + onFinished: console.log("results of the computation: " + result) + } + VideoOutput { + source: camera + filters: [ filter ] + anchors.fill: parent + } + \endcode + + This also allows providing filters in QML plugins, separately from the application. + + \sa VideoOutput, Camera, MediaPlayer, QVideoFilterRunnable +*/ + +/*! + \class QVideoFilterRunnable + \since 5.5 + \brief The QVideoFilterRunnable class represents the implementation of a filter + that owns all graphics and computational resources, and performs the actual filtering + or calculations. + \inmodule QtMultimedia + + \ingroup multimedia + \ingroup multimedia_video + + Video filters are split into QAbstractVideoFilter and corresponding QVideoFilterRunnable + instances, similar to QQuickItem and QSGNode. This is necessary to support + threaded rendering scenarios. When using the threaded render loop of the Qt + Quick scene graph, all rendering happens on a dedicated thread. + QVideoFilterRunnable instances always live on this thread and all its functions, + run(), the constructor, and the destructor, are guaranteed to be invoked on + that thread with the OpenGL context bound. QAbstractVideoFilter instances live on + the main (GUI) thread, like any other QObject and QQuickItem instances + created from QML. + + Once created, QVideoFilterRunnable instances are managed by Qt Multimedia and + will be automatically destroyed and recreated when necessary, for example + when the scene graph is invalidated or the QQuickWindow changes or is closed. + Creation happens via the QAbstractVideoFilter::createFilterRunnable() factory function. + + \sa QAbstractVideoFilter + */ + +/*! + \fn QVideoFrame QVideoFilterRunnable::run(QVideoFrame *input, const QVideoSurfaceFormat &surfaceFormat, RunFlags flags) + + Reimplement this function to perform filtering or computation on the \a + input video frame. Like the constructor and destructor, this function is + always called on the render thread with the OpenGL context bound. + + Implementations that do not modify the video frame can simply return \a input. + + It is safe to access properties of the associated QAbstractVideoFilter instance from + this function. + + \a input will not be mapped, it is up to this function to call QVideoFrame::map() + and QVideoFrame::unmap() as necessary. + + \a surfaceFormat provides additional information, for example it can be used + to determine which way is up in the input image as that is important for + filters to operate on multiple platforms with multiple cameras. + + \a flags contains additional information about the filter's invocation. For + example the LastInChain flag indicates that the filter is the last in a + VideoOutput's associated filter list. This can be very useful in cases where + multiple filters are chained together and the work is performed on image data + in some custom format (for example a format specific to some computer vision + framework). To avoid conversion on every filter in the chain, all + intermediate filters can return a QVideoFrame hosting data in the custom + format. Only the last, where the flag is set, returns a QVideoFrame in a + format compatible with Qt. + + Filters that want to expose the results of their computation to Javascript + code in QML can declare their own custom signals in the QAbstractVideoFilter + subclass to indicate the completion of the operation. For filters that only + calculate some results and do not modify the video frame, it is also possible + to operate asynchronously. They can queue the necessary operations using the + compute API and return from this function without emitting any signals. The + signal indicating the completion is then emitted only when the compute API + indicates that the operations were done and the results are available. Note + that it is strongly recommended to represent the filter's output data as a + separate instance of QJSValue or a QObject-derived class which is passed as a + parameter to the signal and becomes exposed to the Javascript engine. In case + of QObject the ownership of this object is controlled by the standard QML + rules: if it has no parent, ownership is transferred to the Javascript engine, + otherwise it stays with the emitter. Note that the signal connection may be + queued,for example when using the threaded render loop of Qt Quick, and so the + object must stay valid for a longer time, destroying it right after calling + this function is not safe. Using a dedicated results object is guaranteed to + be safe even when using threaded rendering. The same is not necessarily true + for properties on the QAbstractVideoFilter instance itself: properties can + safely be read in run() since the gui thread is blocked during that time but + writing may become problematic. + + \note Avoid time consuming operations in this function as they block the + entire rendering of the application. + + \note The handleType() and pixelFormat() of \a input is completely up to the + video decoding backend on the platform in use. On some platforms different + forms of input are used depending on the graphics stack. For example, when + playing back videos on Windows with the WMF backend, QVideoFrame contains + OpenGL-wrapped Direct3D textures in case of using ANGLE, but regular pixel + data when using desktop OpenGL (opengl32.dll). Similarly, the video file + format will often decide if the data is RGB or YUV, but this may also depend + on the decoder and the configuration in use. The returned video frame does + not have to be in the same format as the input, for example a filter with an + input of a QVideoFrame backed by system memory can output a QVideoFrame with + an OpenGL texture handle. + + \sa QVideoFrame, QVideoSurfaceFormat + */ + +/*! + \enum QVideoFilterRunnable::RunFlag + + \value LastInChain Indicates that the filter runnable's associated QAbstractVideoFilter + is the last in the corresponding VideoOutput type's filters list, meaning + that the returned frame is the one that is going to be presented to the scene + graph without invoking any further filters. + */ + +class QAbstractVideoFilterPrivate +{ +public: + QAbstractVideoFilterPrivate() : + active(true) + { } + + bool active; +}; + +/*! + \internal + */ +QVideoFilterRunnable::~QVideoFilterRunnable() +{ +} + +/*! + Constructs a new QAbstractVideoFilter instance with parent object \a parent. + */ +QAbstractVideoFilter::QAbstractVideoFilter(QObject *parent) : + QObject(parent), + d_ptr(new QAbstractVideoFilterPrivate) +{ +} + +/*! + \internal + */ +QAbstractVideoFilter::~QAbstractVideoFilter() +{ + delete d_ptr; +} + +/*! + \return \c true if the filter is active. + + By default filters are active. When set to \c false, the filter will be + ignored by the VideoOutput type. + */ +bool QAbstractVideoFilter::isActive() const +{ + Q_D(const QAbstractVideoFilter); + return d->active; +} + +/*! + \internal + */ +void QAbstractVideoFilter::setActive(bool v) +{ + Q_D(QAbstractVideoFilter); + if (d->active != v) { + d->active = v; + emit activeChanged(); + } +} + +/*! + \fn QVideoFilterRunnable *QAbstractVideoFilter::createFilterRunnable() + + Factory function to create a new instance of a QVideoFilterRunnable subclass + corresponding to this filter. + + This function is called on the thread on which the Qt Quick scene graph + performs rendering, with the OpenGL context bound. Ownership of the returned + instance is transferred: the returned instance will live on the render thread + and will be destroyed automatically when necessary. + + Typically, implementations of the function will simply construct a new + QVideoFilterRunnable instance, passing \c this to the constructor as the + filter runnables must know their associated QAbstractVideoFilter instance to + access dynamic properties and optionally emit signals. + */ + +QT_END_NAMESPACE |