diff options
Diffstat (limited to 'src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp')
| -rw-r--r-- | src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp | 464 |
1 files changed, 301 insertions, 163 deletions
diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp index 91e48e9c7..451637b9b 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp @@ -1,68 +1,110 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamerimagecapture_p.h" -#include <private/qplatformcamera_p.h> -#include <private/qplatformimagecapture_p.h> -#include <qgstvideobuffer_p.h> -#include <qgstutils_p.h> -#include <qgstreamermetadata_p.h> -#include <qvideoframeformat.h> -#include <private/qmediastoragelocation_p.h> -#include <QtCore/QDebug> -#include <QtCore/QDir> -#include <qstandardpaths.h> +#include <QtMultimedia/private/qplatformcamera_p.h> +#include <QtMultimedia/private/qplatformimagecapture_p.h> +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/private/qmediastoragelocation_p.h> +#include <QtCore/qdebug.h> +#include <QtCore/qdir.h> +#include <QtCore/qstandardpaths.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qloggingcategory.h> -#include <qloggingcategory.h> +#include <common/qgstreamermetadata_p.h> +#include <common/qgstvideobuffer_p.h> +#include <common/qgstutils_p.h> + +#include <utility> QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcImageCapture, "qt.multimedia.imageCapture") +namespace { +Q_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture") + +struct ThreadPoolSingleton +{ + QObject m_context; + QMutex m_poolMutex; + QThreadPool *m_instance{}; + bool m_appUnderDestruction = false; + + QThreadPool *get(const QMutexLocker<QMutex> &) + { + if (m_instance) + return m_instance; + if (m_appUnderDestruction || !qApp) + return nullptr; + + using namespace std::chrono; + + m_instance = new QThreadPool(qApp); + m_instance->setMaxThreadCount(1); // 1 thread; + static constexpr auto expiryTimeout = minutes(5); + m_instance->setExpiryTimeout(round<milliseconds>(expiryTimeout).count()); + + QObject::connect(qApp, &QCoreApplication::aboutToQuit, &m_context, [&] { + // we need to make sure that thread-local QRhi is destroyed before the application to + // prevent QTBUG-124189 + QMutexLocker guard(&m_poolMutex); + delete m_instance; + m_instance = {}; + m_appUnderDestruction = true; + }); + + QObject::connect(qApp, &QCoreApplication::destroyed, &m_context, [&] { + m_appUnderDestruction = false; + }); + return m_instance; + } + + template <typename Functor> + QFuture<void> run(Functor &&f) + { + QMutexLocker guard(&m_poolMutex); + QThreadPool *pool = get(guard); + if (!pool) + return QFuture<void>{}; + + return QtConcurrent::run(pool, std::forward<Functor>(f)); + } +}; + +ThreadPoolSingleton s_threadPoolSingleton; + +}; // namespace -QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) - : QPlatformImageCapture(parent), - QGstreamerBufferProbe(ProbeBuffers) +QMaybe<QPlatformImageCapture *> QGstreamerImageCapture::create(QImageCapture *parent) { - bin = QGstBin("imageCaptureBin"); + QGstElement videoconvert = + QGstElement::createFromFactory("videoconvert", "imageCaptureConvert"); + if (!videoconvert) + return errorMessageCannotFindElement("videoconvert"); - queue = QGstElement("queue", "imageCaptureQueue"); + QGstElement jpegenc = QGstElement::createFromFactory("jpegenc", "jpegEncoder"); + if (!jpegenc) + return errorMessageCannotFindElement("jpegenc"); + + QGstElement jifmux = QGstElement::createFromFactory("jifmux", "jpegMuxer"); + if (!jifmux) + return errorMessageCannotFindElement("jifmux"); + + return new QGstreamerImageCapture(videoconvert, jpegenc, jifmux, parent); +} + +QGstreamerImageCapture::QGstreamerImageCapture(QGstElement videoconvert, QGstElement jpegenc, + QGstElement jifmux, QImageCapture *parent) + : QPlatformImageCapture(parent), + QGstreamerBufferProbe(ProbeBuffers), + videoConvert(std::move(videoconvert)), + encoder(std::move(jpegenc)), + muxer(std::move(jifmux)) +{ + bin = QGstBin::create("imageCaptureBin"); + + queue = QGstElement::createFromFactory("queue", "imageCaptureQueue"); // configures the queue to be fast, lightweight and non blocking queue.set("leaky", 2 /*downstream*/); queue.set("silent", true); @@ -70,37 +112,47 @@ QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) queue.set("max-size-bytes", uint(0)); queue.set("max-size-time", quint64(0)); - videoConvert = QGstElement("videoconvert", "imageCaptureConvert"); - encoder = QGstElement("jpegenc", "jpegEncoder"); - muxer = QGstElement("jifmux", "jpegMuxer"); - sink = QGstElement("fakesink","imageCaptureSink"); + sink = QGstElement::createFromFactory("fakesink", "imageCaptureSink"); + filter = QGstElement::createFromFactory("capsfilter", "filter"); // imageCaptureSink do not wait for a preroll buffer when going READY -> PAUSED // as no buffer will arrive until capture() is called sink.set("async", false); - bin.add(queue, videoConvert, encoder, muxer, sink); - queue.link(videoConvert, encoder, muxer, sink); + bin.add(queue, filter, videoConvert, encoder, muxer, sink); + qLinkGstElements(queue, filter, videoConvert, encoder, muxer, sink); bin.addGhostPad(queue, "sink"); addProbeToPad(queue.staticPad("src").pad(), false); sink.set("signal-handoffs", true); - g_signal_connect(sink.object(), "handoff", G_CALLBACK(&QGstreamerImageCapture::saveImageFilter), this); + m_handoffConnection = sink.connect("handoff", G_CALLBACK(&saveImageFilter), this); } QGstreamerImageCapture::~QGstreamerImageCapture() { bin.setStateSync(GST_STATE_NULL); + + // wait for pending futures + auto pendingFutures = [&] { + QMutexLocker guard(&m_mutex); + return std::move(m_pendingFutures); + }(); + + for (QFuture<void> &pendingImage : pendingFutures) + pendingImage.waitForFinished(); } bool QGstreamerImageCapture::isReadyForCapture() const { + QMutexLocker guard(&m_mutex); return m_session && !passImage && cameraActive; } int QGstreamerImageCapture::capture(const QString &fileName) { - QString path = QMediaStorageLocation::generateFileName(fileName, QStandardPaths::PicturesLocation, QLatin1String("jpg")); + using namespace Qt::Literals; + QString path = QMediaStorageLocation::generateFileName( + fileName, QStandardPaths::PicturesLocation, u"jpg"_s); return doCapture(path); } @@ -111,100 +163,154 @@ int QGstreamerImageCapture::captureToBuffer() int QGstreamerImageCapture::doCapture(const QString &fileName) { - qCDebug(qLcImageCapture) << "do capture"; - if (!m_session) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::ResourceError), - Q_ARG(QString, QPlatformImageCapture::msgImageCaptureNotSet())); - - qCDebug(qLcImageCapture) << "error 1"; - return -1; - } - if (!m_session->camera()) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::ResourceError), - Q_ARG(QString,tr("No camera available."))); - - qCDebug(qLcImageCapture) << "error 2"; - return -1; - } - if (passImage) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::NotReadyError), - Q_ARG(QString, QPlatformImageCapture::msgCameraNotReady())); - - qCDebug(qLcImageCapture) << "error 3"; - return -1; - } - m_lastId++; + qCDebug(qLcImageCaptureGst) << "do capture"; + + { + QMutexLocker guard(&m_mutex); + if (!m_session) { + invokeDeferred([this] { + emit error(-1, QImageCapture::ResourceError, + QPlatformImageCapture::msgImageCaptureNotSet()); + }); + + qCDebug(qLcImageCaptureGst) << "error 1"; + return -1; + } + if (!m_session->camera()) { + invokeDeferred([this] { + emit error(-1, QImageCapture::ResourceError, tr("No camera available.")); + }); - pendingImages.enqueue({m_lastId, fileName, QMediaMetaData{}}); - // let one image pass the pipeline - passImage = true; + qCDebug(qLcImageCaptureGst) << "error 2"; + return -1; + } + if (passImage) { + invokeDeferred([this] { + emit error(-1, QImageCapture::NotReadyError, + QPlatformImageCapture::msgCameraNotReady()); + }); + + qCDebug(qLcImageCaptureGst) << "error 3"; + return -1; + } + m_lastId++; + + pendingImages.enqueue({ m_lastId, fileName, QMediaMetaData{} }); + // let one image pass the pipeline + passImage = true; + } emit readyForCaptureChanged(false); return m_lastId; } +void QGstreamerImageCapture::setResolution(const QSize &resolution) +{ + QGstCaps padCaps = bin.staticPad("sink").currentCaps(); + if (padCaps.isNull()) { + qDebug() << "Camera not ready"; + return; + } + QGstCaps caps = padCaps.copy(); + if (caps.isNull()) + return; + + gst_caps_set_simple(caps.caps(), "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT, + resolution.height(), nullptr); + filter.set("caps", caps); +} + +// HACK: gcc-10 and earlier reject [=,this] when building with c++17 +#if __cplusplus >= 202002L +# define EQ_THIS_CAPTURE =, this +#else +# define EQ_THIS_CAPTURE = +#endif + bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer) { + QMutexLocker guard(&m_mutex); + if (!passImage) return false; - qCDebug(qLcImageCapture) << "probe buffer"; + qCDebug(qLcImageCaptureGst) << "probe buffer"; + + QGstBufferHandle bufferHandle{ + buffer, + QGstBufferHandle::NeedsRef, + }; passImage = false; - emit readyForCaptureChanged(isReadyForCapture()); - - QGstCaps caps = gst_pad_get_current_caps(bin.staticPad("sink").pad()); - GstVideoInfo previewInfo; - gst_video_info_from_caps(&previewInfo, caps.get()); + bool ready = isReadyForCapture(); + invokeDeferred([this, ready] { + emit readyForCaptureChanged(ready); + }); + QGstCaps caps = bin.staticPad("sink").currentCaps(); auto memoryFormat = caps.memoryFormat(); - auto fmt = QGstCaps(caps).formatForCaps(&previewInfo); - auto *sink = m_session->gstreamerVideoSink(); - auto *gstBuffer = new QGstVideoBuffer(buffer, previewInfo, sink, fmt, memoryFormat); - QVideoFrame frame(gstBuffer, fmt); - QImage img = frame.toImage(); - if (img.isNull()) { - qDebug() << "received a null image"; - return true; - } - - auto &imageData = pendingImages.head(); - emit imageExposed(imageData.id); - - qCDebug(qLcImageCapture) << "Image available!"; - emit imageAvailable(imageData.id, frame); + GstVideoInfo previewInfo; + QVideoFrameFormat fmt; + auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo(); + if (optionalFormatAndVideoInfo) + std::tie(fmt, previewInfo) = std::move(*optionalFormatAndVideoInfo); + + int futureId = futureIDAllocator += 1; + + // ensure QVideoFrame::toImage is executed on a worker thread that is joined before the + // qApplication is destroyed + QFuture<void> future = s_threadPoolSingleton.run([EQ_THIS_CAPTURE]() mutable { + QMutexLocker guard(&m_mutex); + auto scopeExit = qScopeGuard([&] { + m_pendingFutures.remove(futureId); + }); + + if (!m_session) { + qDebug() << "QGstreamerImageCapture::probeBuffer: no session"; + return; + } - emit imageCaptured(imageData.id, img); + auto *sink = m_session->gstreamerVideoSink(); + auto *gstBuffer = new QGstVideoBuffer{ + std::move(bufferHandle), previewInfo, sink, fmt, memoryFormat, + }; + QVideoFrame frame(gstBuffer, fmt); - QMediaMetaData metaData = this->metaData(); - metaData.insert(QMediaMetaData::Date, QDateTime::currentDateTime()); - metaData.insert(QMediaMetaData::Resolution, frame.size()); - imageData.metaData = metaData; + QImage img = frame.toImage(); + if (img.isNull()) { + qDebug() << "received a null image"; + return; + } - // ensure taginject injects this metaData - const auto &md = static_cast<const QGstreamerMetaData &>(metaData); - md.setMetaData(muxer.element()); + QMediaMetaData imageMetaData = metaData(); + imageMetaData.insert(QMediaMetaData::Resolution, frame.size()); + pendingImages.head().metaData = std::move(imageMetaData); + PendingImage pendingImage = pendingImages.head(); + + invokeDeferred([this, pendingImage = std::move(pendingImage), frame = std::move(frame), + img = std::move(img)]() mutable { + emit imageExposed(pendingImage.id); + qCDebug(qLcImageCaptureGst) << "Image available!"; + emit imageAvailable(pendingImage.id, frame); + emit imageCaptured(pendingImage.id, img); + emit imageMetadataAvailable(pendingImage.id, pendingImage.metaData); + }); + }); + + if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable + return true; - emit imageMetadataAvailable(imageData.id, metaData); + m_pendingFutures.insert(futureId, future); return true; } +#undef EQ_THIS_CAPTURE + void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session) { + QMutexLocker guard(&m_mutex); QGstreamerMediaCapture *captureSession = static_cast<QGstreamerMediaCapture *>(session); if (m_session == captureSession) return; @@ -225,71 +331,98 @@ void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *ses return; } - connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, &QGstreamerImageCapture::onCameraChanged); + connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, + &QGstreamerImageCapture::onCameraChanged); onCameraChanged(); } +void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m) +{ + { + QMutexLocker guard(&m_mutex); + QPlatformImageCapture::setMetaData(m); + } + + // ensure taginject injects this metaData + applyMetaDataToTagSetter(m, muxer); +} + void QGstreamerImageCapture::cameraActiveChanged(bool active) { - qCDebug(qLcImageCapture) << "cameraActiveChanged" << cameraActive << active; + qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active; if (cameraActive == active) return; cameraActive = active; - qCDebug(qLcImageCapture) << "isReady" << isReadyForCapture(); + qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture(); emit readyForCaptureChanged(isReadyForCapture()); } void QGstreamerImageCapture::onCameraChanged() { + QMutexLocker guard(&m_mutex); if (m_session->camera()) { cameraActiveChanged(m_session->camera()->isActive()); - connect(m_session->camera(), &QPlatformCamera::activeChanged, this, &QGstreamerImageCapture::cameraActiveChanged); + connect(m_session->camera(), &QPlatformCamera::activeChanged, this, + &QGstreamerImageCapture::cameraActiveChanged); } else { cameraActiveChanged(false); } } -gboolean QGstreamerImageCapture::saveImageFilter(GstElement *element, - GstBuffer *buffer, - GstPad *pad, - void *appdata) +gboolean QGstreamerImageCapture::saveImageFilter(GstElement *, GstBuffer *buffer, GstPad *, + QGstreamerImageCapture *capture) { - Q_UNUSED(element); - Q_UNUSED(pad); - QGstreamerImageCapture *capture = static_cast<QGstreamerImageCapture *>(appdata); + capture->saveBufferToImage(buffer); + return true; +} - capture->passImage = false; +void QGstreamerImageCapture::saveBufferToImage(GstBuffer *buffer) +{ + QMutexLocker guard(&m_mutex); + passImage = false; - if (capture->pendingImages.isEmpty()) { - return true; - } + if (pendingImages.isEmpty()) + return; - auto imageData = capture->pendingImages.dequeue(); - if (imageData.filename.isEmpty()) { - return true; - } + PendingImage imageData = pendingImages.dequeue(); + if (imageData.filename.isEmpty()) + return; - qCDebug(qLcImageCapture) << "saving image as" << imageData.filename; + int id = futureIDAllocator++; + QGstBufferHandle bufferHandle{ + buffer, + QGstBufferHandle::NeedsRef, + }; + + QFuture<void> saveImageFuture = QtConcurrent::run([this, imageData, bufferHandle, + id]() mutable { + auto cleanup = qScopeGuard([&] { + QMutexLocker guard(&m_mutex); + m_pendingFutures.remove(id); + }); + + qCDebug(qLcImageCaptureGst) << "saving image as" << imageData.filename; + + QFile f(imageData.filename); + if (!f.open(QFile::WriteOnly)) { + qCDebug(qLcImageCaptureGst) << " could not open image file for writing"; + return; + } - QFile f(imageData.filename); - if (f.open(QFile::WriteOnly)) { GstMapInfo info; + GstBuffer *buffer = bufferHandle.get(); if (gst_buffer_map(buffer, &info, GST_MAP_READ)) { f.write(reinterpret_cast<const char *>(info.data), info.size); gst_buffer_unmap(buffer, &info); } f.close(); - static QMetaMethod savedSignal = QMetaMethod::fromSignal(&QGstreamerImageCapture::imageSaved); - savedSignal.invoke(capture, - Qt::QueuedConnection, - Q_ARG(int, imageData.id), - Q_ARG(QString, imageData.filename)); - } else { - qCDebug(qLcImageCapture) << " could not open image file for writing"; - } + QMetaObject::invokeMethod(this, [this, imageData = std::move(imageData)]() mutable { + imageSaved(imageData.id, imageData.filename); + }); + }); - return TRUE; + m_pendingFutures.insert(id, saveImageFuture); } QImageEncoderSettings QGstreamerImageCapture::imageSettings() const @@ -300,9 +433,14 @@ QImageEncoderSettings QGstreamerImageCapture::imageSettings() const void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings) { if (m_settings != settings) { + QSize resolution = settings.resolution(); + if (m_settings.resolution() != resolution && !resolution.isEmpty()) + setResolution(resolution); + m_settings = settings; - // ### } } QT_END_NAMESPACE + +#include "moc_qgstreamerimagecapture_p.cpp" |